From 0fd54c2e91ff45a223ab15927c21d4fb11c1fcd1 Mon Sep 17 00:00:00 2001 From: "deepin-community-bot[bot]" <156989552+deepin-community-bot[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 10:32:19 +0000 Subject: [PATCH] feat: update python-urllib3 to 2.3.0-3 --- .gitignore | 11 + CHANGES.rst | 486 ++++-- LICENSE.txt | 2 +- MANIFEST.in | 5 - PKG-INFO | 1510 +---------------- README.md | 114 ++ README.rst | 120 -- debian/.gitignore | 1 + debian/changelog | 89 + debian/control | 31 +- debian/copyright | 62 +- ...-implement-missing-importlib-methods.patch | 47 - .../01_do-not-use-embedded-python-six.patch | 602 ------- debian/patches/CVE-2025-50181.patch | 279 +++ debian/patches/CVE-2025-50182.patch | 121 ++ debian/patches/httpx-0.28.patch | 55 + debian/patches/openssl-3.4.0.patch | 31 + debian/patches/series | 7 +- ..._blocked_per_thread-requires_network.patch | 23 + debian/pybuild.testfiles | 3 + debian/rules | 28 +- debian/tests/control | 14 +- debian/tests/python3-urllib3 | 18 +- debian/vendor/hypercorn/__init__.py | 5 + debian/vendor/hypercorn/__main__.py | 291 ++++ debian/vendor/hypercorn/app_wrappers.py | 151 ++ debian/vendor/hypercorn/asyncio/__init__.py | 46 + debian/vendor/hypercorn/asyncio/lifespan.py | 106 ++ debian/vendor/hypercorn/asyncio/run.py | 222 +++ debian/vendor/hypercorn/asyncio/statsd.py | 26 + debian/vendor/hypercorn/asyncio/task_group.py | 74 + debian/vendor/hypercorn/asyncio/tcp_server.py | 148 ++ debian/vendor/hypercorn/asyncio/udp_server.py | 60 + .../hypercorn/asyncio/worker_context.py | 38 + debian/vendor/hypercorn/config.py | 401 +++++ debian/vendor/hypercorn/events.py | 25 + debian/vendor/hypercorn/logging.py | 202 +++ .../vendor/hypercorn/middleware/__init__.py | 12 + .../vendor/hypercorn/middleware/dispatcher.py | 108 ++ .../hypercorn/middleware/http_to_https.py | 67 + debian/vendor/hypercorn/middleware/wsgi.py | 49 + debian/vendor/hypercorn/protocol/__init__.py | 94 + debian/vendor/hypercorn/protocol/events.py | 58 + debian/vendor/hypercorn/protocol/h11.py | 317 ++++ debian/vendor/hypercorn/protocol/h2.py | 385 +++++ debian/vendor/hypercorn/protocol/h3.py | 148 ++ .../vendor/hypercorn/protocol/http_stream.py | 215 +++ debian/vendor/hypercorn/protocol/quic.py | 135 ++ debian/vendor/hypercorn/protocol/ws_stream.py | 375 ++++ debian/vendor/hypercorn/py.typed | 1 + debian/vendor/hypercorn/run.py | 115 ++ debian/vendor/hypercorn/statsd.py | 95 ++ debian/vendor/hypercorn/trio/__init__.py | 52 + debian/vendor/hypercorn/trio/lifespan.py | 97 ++ debian/vendor/hypercorn/trio/run.py | 122 ++ debian/vendor/hypercorn/trio/statsd.py | 16 + debian/vendor/hypercorn/trio/task_group.py | 78 + debian/vendor/hypercorn/trio/tcp_server.py | 159 ++ debian/vendor/hypercorn/trio/udp_server.py | 46 + .../vendor/hypercorn/trio/worker_context.py | 39 + debian/vendor/hypercorn/typing.py | 338 ++++ debian/vendor/hypercorn/utils.py | 223 +++ dev-requirements.txt | 40 +- docs/_static/banner_github.svg | 13 + docs/advanced-usage.rst | 428 +++-- docs/changelog.rst | 5 + docs/conf.py | 72 +- docs/contributing.rst | 66 +- docs/index.rst | 22 +- docs/reference/contrib/appengine.rst | 7 - docs/reference/contrib/emscripten.rst | 89 + docs/reference/contrib/index.rst | 4 +- docs/reference/contrib/ntlmpool.rst | 7 - docs/reference/contrib/pyopenssl.rst | 3 - docs/reference/contrib/securetransport.rst | 28 - docs/reference/index.rst | 2 +- docs/reference/urllib3.connection.rst | 6 + docs/reference/urllib3.connectionpool.rst | 4 + docs/reference/urllib3.exceptions.rst | 6 +- docs/reference/urllib3.poolmanager.rst | 6 + docs/reference/urllib3.request.rst | 9 +- docs/reference/urllib3.response.rst | 11 + docs/reference/urllib3.util.rst | 4 +- docs/requirements.txt | 6 +- docs/sponsors.rst | 41 +- docs/user-guide.rst | 699 ++++---- docs/v2-migration-guide.rst | 400 +++++ docs/v2-roadmap.rst | 177 -- dummyserver/app.py | 476 ++++++ dummyserver/asgi_proxy.py | 110 ++ dummyserver/certs/README.rst | 2 +- dummyserver/handlers.py | 339 ---- dummyserver/hypercornserver.py | 172 ++ dummyserver/proxy.py | 147 -- dummyserver/{server.py => socketserver.py} | 118 +- dummyserver/testcase.py | 334 ++-- pyproject.toml | 136 ++ setup.cfg | 37 - setup.py | 104 -- src/urllib3.egg-info/PKG-INFO | 1492 ---------------- src/urllib3.egg-info/SOURCES.txt | 136 -- src/urllib3.egg-info/dependency_links.txt | 1 - src/urllib3.egg-info/requires.txt | 27 - src/urllib3.egg-info/top_level.txt | 1 - src/urllib3/__init__.py | 155 +- src/urllib3/_base_connection.py | 165 ++ src/urllib3/_collections.py | 414 +++-- .../{request.py => _request_methods.py} | 187 +- src/urllib3/_version.py | 18 +- src/urllib3/connection.py | 1120 ++++++++---- src/urllib3/connectionpool.py | 777 +++++---- src/urllib3/contrib/_appengine_environ.py | 36 - .../contrib/_securetransport/bindings.py | 519 ------ .../contrib/_securetransport/low_level.py | 397 ----- src/urllib3/contrib/appengine.py | 314 ---- src/urllib3/contrib/emscripten/__init__.py | 16 + src/urllib3/contrib/emscripten/connection.py | 255 +++ .../emscripten/emscripten_fetch_worker.js | 110 ++ src/urllib3/contrib/emscripten/fetch.py | 708 ++++++++ src/urllib3/contrib/emscripten/request.py | 22 + src/urllib3/contrib/emscripten/response.py | 285 ++++ src/urllib3/contrib/ntlmpool.py | 130 -- src/urllib3/contrib/pyopenssl.py | 400 +++-- src/urllib3/contrib/securetransport.py | 920 ---------- src/urllib3/contrib/socks.py | 84 +- src/urllib3/exceptions.py | 196 +-- src/urllib3/fields.py | 257 +-- src/urllib3/filepost.py | 65 +- src/urllib3/http2/__init__.py | 53 + src/urllib3/http2/connection.py | 356 ++++ src/urllib3/http2/probe.py | 87 + src/urllib3/packages/__init__.py | 0 src/urllib3/packages/backports/__init__.py | 0 src/urllib3/packages/backports/makefile.py | 51 - .../packages/backports/weakref_finalize.py | 155 -- src/urllib3/packages/six.py | 1076 ------------ src/urllib3/poolmanager.py | 343 ++-- src/urllib3/py.typed | 2 + src/urllib3/response.py | 1051 ++++++++---- src/urllib3/util/__init__.py | 19 +- src/urllib3/util/connection.py | 80 +- src/urllib3/util/proxy.py | 38 +- src/urllib3/util/queue.py | 22 - src/urllib3/util/request.py | 172 +- src/urllib3/util/response.py | 82 +- src/urllib3/util/retry.py | 363 ++-- src/urllib3/util/ssl_.py | 565 +++--- src/urllib3/util/ssl_match_hostname.py | 94 +- src/urllib3/util/ssltransport.py | 162 +- src/urllib3/util/timeout.py | 118 +- src/urllib3/util/url.py | 394 +++-- src/urllib3/util/util.py | 42 + src/urllib3/util/wait.py | 90 +- test/__init__.py | 343 ++-- test/appengine/__init__.py | 0 test/appengine/conftest.py | 78 - test/appengine/test_gae_manager.py | 178 -- test/appengine/test_urlfetch.py | 66 - test/benchmark.py | 76 - test/conftest.py | 302 +++- .../contrib/emscripten}/__init__.py | 0 test/contrib/emscripten/conftest.py | 278 +++ .../emscripten/templates/pyodide-console.html | 271 +++ test/contrib/emscripten/test_emscripten.py | 1191 +++++++++++++ test/contrib/test_pyopenssl.py | 42 +- test/contrib/test_pyopenssl_dependencies.py | 18 +- test/contrib/test_securetransport.py | 67 - test/contrib/test_socks.py | 262 +-- test/port_helpers.py | 39 +- test/socketpair_helper.py | 63 - test/test_collections.py | 326 ++-- test/test_compatibility.py | 60 +- test/test_connection.py | 318 +++- test/test_connectionpool.py | 305 ++-- test/test_exceptions.py | 48 +- test/test_fields.py | 114 +- test/test_filepost.py | 62 +- test/test_http2_connection.py | 360 ++++ test/test_no_ssl.py | 65 +- test/test_poolmanager.py | 201 ++- test/test_proxymanager.py | 28 +- test/test_queue_monkeypatch.py | 13 +- test/test_request.py | 26 - test/test_response.py | 984 ++++++++--- test/test_retry.py | 212 ++- test/test_retry_deprecated.py | 485 ------ test/test_ssl.py | 390 +++-- test/test_ssltransport.py | 297 ++-- test/test_util.py | 624 ++++--- test/test_wait.py | 52 +- test/tz_stub.py | 23 +- .../with_dummyserver/test_chunked_transfer.py | 156 +- test/with_dummyserver/test_connection.py | 140 ++ test/with_dummyserver/test_connectionpool.py | 548 +++--- test/with_dummyserver/test_https.py | 1110 ++++++++---- test/with_dummyserver/test_no_ssl.py | 29 +- test/with_dummyserver/test_poolmanager.py | 516 ++++-- .../test_proxy_poolmanager.py | 759 +++++---- test/with_dummyserver/test_socketlevel.py | 1434 ++++++++++++---- 199 files changed, 24032 insertions(+), 16477 deletions(-) create mode 100644 .gitignore delete mode 100644 MANIFEST.in create mode 100644 README.md delete mode 100644 README.rst create mode 100644 debian/.gitignore delete mode 100644 debian/patches/0002-implement-missing-importlib-methods.patch delete mode 100644 debian/patches/01_do-not-use-embedded-python-six.patch create mode 100644 debian/patches/CVE-2025-50181.patch create mode 100644 debian/patches/CVE-2025-50182.patch create mode 100644 debian/patches/httpx-0.28.patch create mode 100644 debian/patches/openssl-3.4.0.patch create mode 100644 debian/patches/test_http2_probe_blocked_per_thread-requires_network.patch create mode 100644 debian/pybuild.testfiles create mode 100644 debian/vendor/hypercorn/__init__.py create mode 100644 debian/vendor/hypercorn/__main__.py create mode 100644 debian/vendor/hypercorn/app_wrappers.py create mode 100644 debian/vendor/hypercorn/asyncio/__init__.py create mode 100644 debian/vendor/hypercorn/asyncio/lifespan.py create mode 100644 debian/vendor/hypercorn/asyncio/run.py create mode 100644 debian/vendor/hypercorn/asyncio/statsd.py create mode 100644 debian/vendor/hypercorn/asyncio/task_group.py create mode 100644 debian/vendor/hypercorn/asyncio/tcp_server.py create mode 100644 debian/vendor/hypercorn/asyncio/udp_server.py create mode 100644 debian/vendor/hypercorn/asyncio/worker_context.py create mode 100644 debian/vendor/hypercorn/config.py create mode 100644 debian/vendor/hypercorn/events.py create mode 100644 debian/vendor/hypercorn/logging.py create mode 100644 debian/vendor/hypercorn/middleware/__init__.py create mode 100644 debian/vendor/hypercorn/middleware/dispatcher.py create mode 100644 debian/vendor/hypercorn/middleware/http_to_https.py create mode 100644 debian/vendor/hypercorn/middleware/wsgi.py create mode 100755 debian/vendor/hypercorn/protocol/__init__.py create mode 100644 debian/vendor/hypercorn/protocol/events.py create mode 100755 debian/vendor/hypercorn/protocol/h11.py create mode 100755 debian/vendor/hypercorn/protocol/h2.py create mode 100644 debian/vendor/hypercorn/protocol/h3.py create mode 100644 debian/vendor/hypercorn/protocol/http_stream.py create mode 100644 debian/vendor/hypercorn/protocol/quic.py create mode 100644 debian/vendor/hypercorn/protocol/ws_stream.py create mode 100644 debian/vendor/hypercorn/py.typed create mode 100644 debian/vendor/hypercorn/run.py create mode 100644 debian/vendor/hypercorn/statsd.py create mode 100644 debian/vendor/hypercorn/trio/__init__.py create mode 100644 debian/vendor/hypercorn/trio/lifespan.py create mode 100644 debian/vendor/hypercorn/trio/run.py create mode 100644 debian/vendor/hypercorn/trio/statsd.py create mode 100644 debian/vendor/hypercorn/trio/task_group.py create mode 100644 debian/vendor/hypercorn/trio/tcp_server.py create mode 100644 debian/vendor/hypercorn/trio/udp_server.py create mode 100644 debian/vendor/hypercorn/trio/worker_context.py create mode 100644 debian/vendor/hypercorn/typing.py create mode 100644 debian/vendor/hypercorn/utils.py create mode 100644 docs/_static/banner_github.svg create mode 100644 docs/changelog.rst delete mode 100644 docs/reference/contrib/appengine.rst create mode 100644 docs/reference/contrib/emscripten.rst delete mode 100644 docs/reference/contrib/ntlmpool.rst delete mode 100644 docs/reference/contrib/securetransport.rst create mode 100644 docs/v2-migration-guide.rst delete mode 100644 docs/v2-roadmap.rst create mode 100644 dummyserver/app.py create mode 100755 dummyserver/asgi_proxy.py delete mode 100644 dummyserver/handlers.py create mode 100644 dummyserver/hypercornserver.py delete mode 100755 dummyserver/proxy.py rename dummyserver/{server.py => socketserver.py} (62%) create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100755 setup.py delete mode 100644 src/urllib3.egg-info/PKG-INFO delete mode 100644 src/urllib3.egg-info/SOURCES.txt delete mode 100644 src/urllib3.egg-info/dependency_links.txt delete mode 100644 src/urllib3.egg-info/requires.txt delete mode 100644 src/urllib3.egg-info/top_level.txt create mode 100644 src/urllib3/_base_connection.py rename src/urllib3/{request.py => _request_methods.py} (50%) delete mode 100644 src/urllib3/contrib/_appengine_environ.py delete mode 100644 src/urllib3/contrib/_securetransport/bindings.py delete mode 100644 src/urllib3/contrib/_securetransport/low_level.py delete mode 100644 src/urllib3/contrib/appengine.py create mode 100644 src/urllib3/contrib/emscripten/__init__.py create mode 100644 src/urllib3/contrib/emscripten/connection.py create mode 100644 src/urllib3/contrib/emscripten/emscripten_fetch_worker.js create mode 100644 src/urllib3/contrib/emscripten/fetch.py create mode 100644 src/urllib3/contrib/emscripten/request.py create mode 100644 src/urllib3/contrib/emscripten/response.py delete mode 100644 src/urllib3/contrib/ntlmpool.py delete mode 100644 src/urllib3/contrib/securetransport.py create mode 100644 src/urllib3/http2/__init__.py create mode 100644 src/urllib3/http2/connection.py create mode 100644 src/urllib3/http2/probe.py delete mode 100644 src/urllib3/packages/__init__.py delete mode 100644 src/urllib3/packages/backports/__init__.py delete mode 100644 src/urllib3/packages/backports/makefile.py delete mode 100644 src/urllib3/packages/backports/weakref_finalize.py delete mode 100644 src/urllib3/packages/six.py create mode 100644 src/urllib3/py.typed delete mode 100644 src/urllib3/util/queue.py create mode 100644 src/urllib3/util/util.py delete mode 100644 test/appengine/__init__.py delete mode 100644 test/appengine/conftest.py delete mode 100644 test/appengine/test_gae_manager.py delete mode 100644 test/appengine/test_urlfetch.py delete mode 100644 test/benchmark.py rename {src/urllib3/contrib/_securetransport => test/contrib/emscripten}/__init__.py (100%) create mode 100644 test/contrib/emscripten/conftest.py create mode 100644 test/contrib/emscripten/templates/pyodide-console.html create mode 100644 test/contrib/emscripten/test_emscripten.py delete mode 100644 test/contrib/test_securetransport.py delete mode 100644 test/socketpair_helper.py create mode 100644 test/test_http2_connection.py delete mode 100644 test/test_request.py delete mode 100644 test/test_retry_deprecated.py create mode 100644 test/with_dummyserver/test_connection.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..77a3bb2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +.* +!/.github/ +*.py[co] +*.egg +*.egg-info +*.log +/dist +/build +/docs/_build +coverage.xml +src/urllib3/_version.py diff --git a/CHANGES.rst b/CHANGES.rst index 3a0a4f0..1cb2703 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,27 +1,328 @@ -Changes -======= +2.3.0 (2024-12-22) +================== -1.26.18 (2023-10-17) --------------------- +Features +-------- + +- Added ``HTTPResponse.shutdown()`` to stop any ongoing or future reads for a specific response. It calls ``shutdown(SHUT_RD)`` on the underlying socket. This feature was `sponsored by LaunchDarkly `__. (`#2868 `__) +- Added support for JavaScript Promise Integration on Emscripten. This enables more efficient WebAssembly + requests and streaming, and makes it possible to use in Node.js if you launch it as ``node --experimental-wasm-stack-switching``. (`#3400 `__) +- Added the ``proxy_is_tunneling`` property to ``HTTPConnection`` and ``HTTPSConnection``. (`#3285 `__) +- Added pickling support to ``NewConnectionError`` and ``NameResolutionError``. (`#3480 `__) + + +Bugfixes +-------- + +- Fixed an issue in debug logs where the HTTP version was rendering as "HTTP/11" instead of "HTTP/1.1". (`#3489 `__) + + +Deprecations and Removals +------------------------- + +- Removed support for Python 3.8. (`#3492 `__) + + +2.2.3 (2024-09-12) +================== + +Features +-------- + +- Added support for Python 3.13. (`#3473 `__) + +Bugfixes +-------- + +- Fixed the default encoding of chunked request bodies to be UTF-8 instead of ISO-8859-1. + All other methods of supplying a request body already use UTF-8 starting in urllib3 v2.0. (`#3053 `__) +- Fixed ResourceWarning on CONNECT with Python < 3.11.4 by backporting https://github.com/python/cpython/issues/103472. (`#3252 `__) +- Adjust tolerance for floating-point comparison on Windows to avoid flakiness in CI (`#3413 `__) +- Fixed a crash where certain standard library hash functions were absent in restricted environments. (`#3432 `__) +- Fixed mypy error when adding to ``HTTPConnection.default_socket_options``. (`#3448 `__) + +HTTP/2 (experimental) +--------------------- + +HTTP/2 support is still in early development. + +- Excluded Transfer-Encoding: chunked from HTTP/2 request body (`#3425 `__) +- Added version checking for ``h2`` (https://pypi.org/project/h2/) usage. + + Now only accepting supported h2 major version 4.x.x. (`#3290 `__) +- Added a probing mechanism for determining whether a given target origin + supports HTTP/2 via ALPN. (`#3301 `__) +- Add support for sending a request body with HTTP/2 (`#3302 `__) + + +Deprecations and Removals +------------------------- + +- Note for downstream distributors: the ``_version.py`` file has been removed and is now created at build time by hatch-vcs. (`#3412 `__) +- Drop support for end-of-life PyPy3.8 and PyPy3.9. (`#3475 `__) + + +2.2.2 (2024-06-17) +================== + +- Added the ``Proxy-Authorization`` header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via ``Retry.remove_headers_on_redirect``. +- Allowed passing negative integers as ``amt`` to read methods of ``http.client.HTTPResponse`` as an alternative to ``None``. (`#3122 `__) +- Fixed return types representing copying actions to use ``typing.Self``. (`#3363 `__) + +2.2.1 (2024-02-16) +================== + +- Fixed issue where ``InsecureRequestWarning`` was emitted for HTTPS connections when using Emscripten. (`#3331 `__) +- Fixed ``HTTPConnectionPool.urlopen`` to stop automatically casting non-proxy headers to ``HTTPHeaderDict``. This change was premature as it did not apply to proxy headers and ``HTTPHeaderDict`` does not handle byte header values correctly yet. (`#3343 `__) +- Changed ``InvalidChunkLength`` to ``ProtocolError`` when response terminates before the chunk length is sent. (`#2860 `__) +- Changed ``ProtocolError`` to be more verbose on incomplete reads with excess content. (`#3261 `__) + + +2.2.0 (2024-01-30) +================== + +- Added support for `Emscripten and Pyodide `__, including streaming support in cross-origin isolated browser environments where threading is enabled. (`#2951 `__) +- Added support for ``HTTPResponse.read1()`` method. (`#3186 `__) +- Added rudimentary support for HTTP/2. (`#3284 `__) +- Fixed issue where requests against urls with trailing dots were failing due to SSL errors + when using proxy. (`#2244 `__) +- Fixed ``HTTPConnection.proxy_is_verified`` and ``HTTPSConnection.proxy_is_verified`` + to be always set to a boolean after connecting to a proxy. It could be + ``None`` in some cases previously. (`#3130 `__) +- Fixed an issue where ``headers`` passed in a request with ``json=`` would be mutated (`#3203 `__) +- Fixed ``HTTPSConnection.is_verified`` to be set to ``False`` when connecting + from a HTTPS proxy to an HTTP target. It was set to ``True`` previously. (`#3267 `__) +- Fixed handling of new error message from OpenSSL 3.2.0 when configuring an HTTP proxy as HTTPS (`#3268 `__) +- Fixed TLS 1.3 post-handshake auth when the server certificate validation is disabled (`#3325 `__) +- Note for downstream distributors: To run integration tests, you now need to run the tests a second + time with the ``--integration`` pytest flag. (`#3181 `__) + + +2.1.0 (2023-11-13) +================== + +- Removed support for the deprecated urllib3[secure] extra. (`#2680 `__) +- Removed support for the deprecated SecureTransport TLS implementation. (`#2681 `__) +- Removed support for the end-of-life Python 3.7. (`#3143 `__) +- Allowed loading CA certificates from memory for proxies. (`#3065 `__) +- Fixed decoding Gzip-encoded responses which specified ``x-gzip`` content-encoding. (`#3174 `__) + + +2.0.7 (2023-10-17) +================== * Made body stripped from HTTP requests changing the request method to GET after HTTP 303 "See Other" redirect responses. -1.26.17 (2023-10-02) --------------------- +2.0.6 (2023-10-02) +================== * Added the ``Cookie`` header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via ``Retry.remove_headers_on_redirect``. +2.0.5 (2023-09-20) +================== + +- Allowed pyOpenSSL third-party module without any deprecation warning. (`#3126 `__) +- Fixed default ``blocksize`` of ``HTTPConnection`` classes to match high-level classes. Previously was 8KiB, now 16KiB. (`#3066 `__) + + +2.0.4 (2023-07-19) +================== + +- Added support for union operators to ``HTTPHeaderDict`` (`#2254 `__) +- Added ``BaseHTTPResponse`` to ``urllib3.__all__`` (`#3078 `__) +- Fixed ``urllib3.connection.HTTPConnection`` to raise the ``http.client.connect`` audit event to have the same behavior as the standard library HTTP client (`#2757 `__) +- Relied on the standard library for checking hostnames in supported PyPy releases (`#3087 `__) + + +2.0.3 (2023-06-07) +================== + +- Allowed alternative SSL libraries such as LibreSSL, while still issuing a warning as we cannot help users facing issues with implementations other than OpenSSL. (`#3020 `__) +- Deprecated URLs which don't have an explicit scheme (`#2950 `_) +- Fixed response decoding with Zstandard when compressed data is made of several frames. (`#3008 `__) +- Fixed ``assert_hostname=False`` to correctly skip hostname check. (`#3051 `__) + + +2.0.2 (2023-05-03) +================== + +- Fixed ``HTTPResponse.stream()`` to continue yielding bytes if buffered decompressed data + was still available to be read even if the underlying socket is closed. This prevents + a compressed response from being truncated. (`#3009 `__) + + +2.0.1 (2023-04-30) +================== + +- Fixed a socket leak when fingerprint or hostname verifications fail. (`#2991 `__) +- Fixed an error when ``HTTPResponse.read(0)`` was the first ``read`` call or when the internal response body buffer was otherwise empty. (`#2998 `__) + + +2.0.0 (2023-04-26) +================== + +Read the `v2.0 migration guide `__ for help upgrading to the latest version of urllib3. + +Removed +------- + +* Removed support for Python 2.7, 3.5, and 3.6 (`#883 `__, `#2336 `__). +* Removed fallback on certificate ``commonName`` in ``match_hostname()`` function. + This behavior was deprecated in May 2000 in RFC 2818. Instead only ``subjectAltName`` + is used to verify the hostname by default. To enable verifying the hostname against + ``commonName`` use ``SSLContext.hostname_checks_common_name = True`` (`#2113 `__). +* Removed support for Python with an ``ssl`` module compiled with LibreSSL, CiscoSSL, + wolfSSL, and all other OpenSSL alternatives. Python is moving to require OpenSSL with PEP 644 (`#2168 `__). +* Removed support for OpenSSL versions earlier than 1.1.1 or that don't have SNI support. + When an incompatible OpenSSL version is detected an ``ImportError`` is raised (`#2168 `__). +* Removed the list of default ciphers for OpenSSL 1.1.1+ and SecureTransport as their own defaults are already secure (`#2082 `__). +* Removed ``urllib3.contrib.appengine.AppEngineManager`` and support for Google App Engine Standard Environment (`#2044 `__). +* Removed deprecated ``Retry`` options ``method_whitelist``, ``DEFAULT_REDIRECT_HEADERS_BLACKLIST`` (`#2086 `__). +* Removed ``urllib3.HTTPResponse.from_httplib`` (`#2648 `__). +* Removed default value of ``None`` for the ``request_context`` parameter of ``urllib3.PoolManager.connection_from_pool_key``. This change should have no effect on users as the default value of ``None`` was an invalid option and was never used (`#1897 `__). +* Removed the ``urllib3.request`` module. ``urllib3.request.RequestMethods`` has been made a private API. + This change was made to ensure that ``from urllib3 import request`` imported the top-level ``request()`` + function instead of the ``urllib3.request`` module (`#2269 `__). +* Removed support for SSLv3.0 from the ``urllib3.contrib.pyopenssl`` even when support is available from the compiled OpenSSL library (`#2233 `__). +* Removed the deprecated ``urllib3.contrib.ntlmpool`` module (`#2339 `__). +* Removed ``DEFAULT_CIPHERS``, ``HAS_SNI``, ``USE_DEFAULT_SSLCONTEXT_CIPHERS``, from the private module ``urllib3.util.ssl_`` (`#2168 `__). +* Removed ``urllib3.exceptions.SNIMissingWarning`` (`#2168 `__). +* Removed the ``_prepare_conn`` method from ``HTTPConnectionPool``. Previously this was only used to call ``HTTPSConnection.set_cert()`` by ``HTTPSConnectionPool`` (`#1985 `__). +* Removed ``tls_in_tls_required`` property from ``HTTPSConnection``. This is now determined from the ``scheme`` parameter in ``HTTPConnection.set_tunnel()`` (`#1985 `__). +* Removed the ``strict`` parameter/attribute from ``HTTPConnection``, ``HTTPSConnection``, ``HTTPConnectionPool``, ``HTTPSConnectionPool``, and ``HTTPResponse`` (`#2064 `__). + +Deprecated +---------- + +* Deprecated ``HTTPResponse.getheaders()`` and ``HTTPResponse.getheader()`` which will be removed in urllib3 v2.1.0. Instead use ``HTTPResponse.headers`` and ``HTTPResponse.headers.get(name, default)``. (`#1543 `__, `#2814 `__). +* Deprecated ``urllib3.contrib.pyopenssl`` module which will be removed in urllib3 v2.1.0 (`#2691 `__). +* Deprecated ``urllib3.contrib.securetransport`` module which will be removed in urllib3 v2.1.0 (`#2692 `__). +* Deprecated ``ssl_version`` option in favor of ``ssl_minimum_version``. ``ssl_version`` will be removed in urllib3 v2.1.0 (`#2110 `__). +* Deprecated the ``strict`` parameter of ``PoolManager.connection_from_context()`` as it's not longer needed in Python 3.x. It will be removed in urllib3 v2.1.0 (`#2267 `__) +* Deprecated the ``NewConnectionError.pool`` attribute which will be removed in urllib3 v2.1.0 (`#2271 `__). +* Deprecated ``format_header_param_html5`` and ``format_header_param`` in favor of ``format_multipart_header_param`` (`#2257 `__). +* Deprecated ``RequestField.header_formatter`` parameter which will be removed in urllib3 v2.1.0 (`#2257 `__). +* Deprecated ``HTTPSConnection.set_cert()`` method. Instead pass parameters to the ``HTTPSConnection`` constructor (`#1985 `__). +* Deprecated ``HTTPConnection.request_chunked()`` method which will be removed in urllib3 v2.1.0. Instead pass ``chunked=True`` to ``HTTPConnection.request()`` (`#1985 `__). + +Added +----- + +* Added top-level ``urllib3.request`` function which uses a preconfigured module-global ``PoolManager`` instance (`#2150 `__). +* Added the ``json`` parameter to ``urllib3.request()``, ``PoolManager.request()``, and ``ConnectionPool.request()`` methods to send JSON bodies in requests. Using this parameter will set the header ``Content-Type: application/json`` if ``Content-Type`` isn't already defined. + Added support for parsing JSON response bodies with ``HTTPResponse.json()`` method (`#2243 `__). +* Added type hints to the ``urllib3`` module (`#1897 `__). +* Added ``ssl_minimum_version`` and ``ssl_maximum_version`` options which set + ``SSLContext.minimum_version`` and ``SSLContext.maximum_version`` (`#2110 `__). +* Added support for Zstandard (RFC 8878) when ``zstandard`` 1.18.0 or later is installed. + Added the ``zstd`` extra which installs the ``zstandard`` package (`#1992 `__). +* Added ``urllib3.response.BaseHTTPResponse`` class. All future response classes will be subclasses of ``BaseHTTPResponse`` (`#2083 `__). +* Added ``FullPoolError`` which is raised when ``PoolManager(block=True)`` and a connection is returned to a full pool (`#2197 `__). +* Added ``HTTPHeaderDict`` to the top-level ``urllib3`` namespace (`#2216 `__). +* Added support for configuring header merging behavior with HTTPHeaderDict + When using a ``HTTPHeaderDict`` to provide headers for a request, by default duplicate + header values will be repeated. But if ``combine=True`` is passed into a call to + ``HTTPHeaderDict.add``, then the added header value will be merged in with an existing + value into a comma-separated list (``X-My-Header: foo, bar``) (`#2242 `__). +* Added ``NameResolutionError`` exception when a DNS error occurs (`#2305 `__). +* Added ``proxy_assert_hostname`` and ``proxy_assert_fingerprint`` kwargs to ``ProxyManager`` (`#2409 `__). +* Added a configurable ``backoff_max`` parameter to the ``Retry`` class. + If a custom ``backoff_max`` is provided to the ``Retry`` class, it + will replace the ``Retry.DEFAULT_BACKOFF_MAX`` (`#2494 `__). +* Added the ``authority`` property to the Url class as per RFC 3986 3.2. This property should be used in place of ``netloc`` for users who want to include the userinfo (auth) component of the URI (`#2520 `__). +* Added the ``scheme`` parameter to ``HTTPConnection.set_tunnel`` to configure the scheme of the origin being tunnelled to (`#1985 `__). +* Added the ``is_closed``, ``is_connected`` and ``has_connected_to_proxy`` properties to ``HTTPConnection`` (`#1985 `__). +* Added optional ``backoff_jitter`` parameter to ``Retry``. (`#2952 `__) + +Changed +------- + +* Changed ``urllib3.response.HTTPResponse.read`` to respect the semantics of ``io.BufferedIOBase`` regardless of compression. Specifically, this method: + + * Only returns an empty bytes object to indicate EOF (that is, the response has been fully consumed). + * Never returns more bytes than requested. + * Can issue any number of system calls: zero, one or multiple. + + If you want each ``urllib3.response.HTTPResponse.read`` call to issue a single system call, you need to disable decompression by setting ``decode_content=False`` (`#2128 `__). +* Changed ``urllib3.HTTPConnection.getresponse`` to return an instance of ``urllib3.HTTPResponse`` instead of ``http.client.HTTPResponse`` (`#2648 `__). +* Changed ``ssl_version`` to instead set the corresponding ``SSLContext.minimum_version`` + and ``SSLContext.maximum_version`` values. Regardless of ``ssl_version`` passed + ``SSLContext`` objects are now constructed using ``ssl.PROTOCOL_TLS_CLIENT`` (`#2110 `__). +* Changed default ``SSLContext.minimum_version`` to be ``TLSVersion.TLSv1_2`` in line with Python 3.10 (`#2373 `__). +* Changed ``ProxyError`` to wrap any connection error (timeout, TLS, DNS) that occurs when connecting to the proxy (`#2482 `__). +* Changed ``urllib3.util.create_urllib3_context`` to not override the system cipher suites with + a default value. The new default will be cipher suites configured by the operating system (`#2168 `__). +* Changed ``multipart/form-data`` header parameter formatting matches the WHATWG HTML Standard as of 2021-06-10. Control characters in filenames are no longer percent encoded (`#2257 `__). +* Changed the error raised when connecting via HTTPS when the ``ssl`` module isn't available from ``SSLError`` to ``ImportError`` (`#2589 `__). +* Changed ``HTTPConnection.request()`` to always use lowercase chunk boundaries when sending requests with ``Transfer-Encoding: chunked`` (`#2515 `__). +* Changed ``enforce_content_length`` default to True, preventing silent data loss when reading streamed responses (`#2514 `__). +* Changed internal implementation of ``HTTPHeaderDict`` to use ``dict`` instead of ``collections.OrderedDict`` for better performance (`#2080 `__). +* Changed the ``urllib3.contrib.pyopenssl`` module to wrap ``OpenSSL.SSL.Error`` with ``ssl.SSLError`` in ``PyOpenSSLContext.load_cert_chain`` (`#2628 `__). +* Changed usage of the deprecated ``socket.error`` to ``OSError`` (`#2120 `__). +* Changed all parameters in the ``HTTPConnection`` and ``HTTPSConnection`` constructors to be keyword-only except ``host`` and ``port`` (`#1985 `__). +* Changed ``HTTPConnection.getresponse()`` to set the socket timeout from ``HTTPConnection.timeout`` value before reading + data from the socket. This previously was done manually by the ``HTTPConnectionPool`` calling ``HTTPConnection.sock.settimeout(...)`` (`#1985 `__). +* Changed the ``_proxy_host`` property to ``_tunnel_host`` in ``HTTPConnectionPool`` to more closely match how the property is used (value in ``HTTPConnection.set_tunnel()``) (`#1985 `__). +* Changed name of ``Retry.BACK0FF_MAX`` to be ``Retry.DEFAULT_BACKOFF_MAX``. +* Changed TLS handshakes to use ``SSLContext.check_hostname`` when possible (`#2452 `__). +* Changed ``server_hostname`` to behave like other parameters only used by ``HTTPSConnectionPool`` (`#2537 `__). +* Changed the default ``blocksize`` to 16KB to match OpenSSL's default read amounts (`#2348 `__). +* Changed ``HTTPResponse.read()`` to raise an error when calling with ``decode_content=False`` after using ``decode_content=True`` to prevent data loss (`#2800 `__). + +Fixed +----- + +* Fixed thread-safety issue where accessing a ``PoolManager`` with many distinct origins would cause connection pools to be closed while requests are in progress (`#1252 `__). +* Fixed an issue where an ``HTTPConnection`` instance would erroneously reuse the socket read timeout value from reading the previous response instead of a newly configured connect timeout. + Instead now if ``HTTPConnection.timeout`` is updated before sending the next request the new timeout value will be used (`#2645 `__). +* Fixed ``socket.error.errno`` when raised from pyOpenSSL's ``OpenSSL.SSL.SysCallError`` (`#2118 `__). +* Fixed the default value of ``HTTPSConnection.socket_options`` to match ``HTTPConnection`` (`#2213 `__). +* Fixed a bug where ``headers`` would be modified by the ``remove_headers_on_redirect`` feature (`#2272 `__). +* Fixed a reference cycle bug in ``urllib3.util.connection.create_connection()`` (`#2277 `__). +* Fixed a socket leak if ``HTTPConnection.connect()`` fails (`#2571 `__). +* Fixed ``urllib3.contrib.pyopenssl.WrappedSocket`` and ``urllib3.contrib.securetransport.WrappedSocket`` close methods (`#2970 `__) + +1.26.20 (2024-08-29) +==================== + +* Fixed a crash where certain standard library hash functions were absent in + FIPS-compliant environments. + (`#3432 `__) +* Replaced deprecated dash-separated setuptools entries in ``setup.cfg``. + (`#3461 `__) +* Took into account macOS setting ``ECONNRESET`` instead of ``EPROTOTYPE`` in + its newer versions. + (`#3416 `__) +* Backported changes to our tests and CI configuration from v2.x to support + testing with CPython 3.12 and 3.13. + (`#3436 `__) + +1.26.19 (2024-06-17) +==================== + +* Added the ``Proxy-Authorization`` header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via ``Retry.remove_headers_on_redirect``. +* Fixed handling of OpenSSL 3.2.0 new error message for misconfiguring an HTTP proxy as HTTPS. (`#3405 `__) + +1.26.18 (2023-10-17) +==================== + +* Made body stripped from HTTP requests changing the request method to GET after HTTP 303 "See Other" redirect responses. + +1.26.17 (2023-10-02) +==================== + +* Added the ``Cookie`` header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via ``Retry.remove_headers_on_redirect``. (`#3139 `_) + 1.26.16 (2023-05-23) --------------------- +==================== * Fixed thread-safety issue where accessing a ``PoolManager`` with many distinct origins would cause connection pools to be closed while requests are in progress (`#2954 `_) - 1.26.15 (2023-03-10) --------------------- +==================== * Fix socket timeout value when ``HTTPConnection`` is reused (`#2645 `__) * Remove "!" character from the unreserved characters in IPv6 Zone ID parsing @@ -29,13 +330,13 @@ Changes * Fix IDNA handling of '\x80' byte (`#2901 `__) 1.26.14 (2023-01-11) --------------------- +==================== * Fixed parsing of port 0 (zero) returning None, instead of 0. (`#2850 `__) -* Removed deprecated getheaders() calls in contrib module. +* Removed deprecated getheaders() calls in contrib module. Fixed the type hint of ``PoolKey.key_retries`` by adding ``bool`` to the union. (`#2865 `__) 1.26.13 (2022-11-23) --------------------- +==================== * Deprecated the ``HTTPResponse.getheaders()`` and ``HTTPResponse.getheader()`` methods. * Fixed an issue where parsing a URL with leading zeroes in the port would be rejected @@ -43,32 +344,28 @@ Changes * Fixed a deprecation warning when using cryptography v39.0.0. * Removed the ``<4`` in the ``Requires-Python`` packaging metadata field. - 1.26.12 (2022-08-22) --------------------- +==================== * Deprecated the `urllib3[secure]` extra and the `urllib3.contrib.pyopenssl` module. Both will be removed in v2.x. See this `GitHub issue `_ for justification and info on how to migrate. - 1.26.11 (2022-07-25) --------------------- +==================== * Fixed an issue where reading more than 2 GiB in a call to ``HTTPResponse.read`` would raise an ``OverflowError`` on Python 3.9 and earlier. - 1.26.10 (2022-07-07) --------------------- +==================== * Removed support for Python 3.5 * Fixed an issue where a ``ProxyError`` recommending configuring the proxy as HTTP instead of HTTPS could appear even when an HTTPS proxy wasn't configured. - 1.26.9 (2022-03-16) -------------------- +=================== * Changed ``urllib3[brotli]`` extra to favor installing Brotli libraries that are still receiving updates like ``brotli`` and ``brotlicffi`` instead of ``brotlipy``. @@ -77,9 +374,8 @@ Changes * Fixed ``server_hostname`` being forwarded from ``PoolManager`` to ``HTTPConnectionPool`` when requesting an HTTP URL. Should only be forwarded when requesting an HTTPS URL. - 1.26.8 (2022-01-07) -------------------- +=================== * Added extra message to ``urllib3.exceptions.ProxyError`` when urllib3 detects that a proxy is configured to use HTTPS but the proxy itself appears to only use HTTP. @@ -94,7 +390,7 @@ Changes 1.26.7 (2021-09-22) -------------------- +=================== * Fixed a bug with HTTPS hostname verification involving IP addresses and lack of SNI. (Issue #2400) @@ -103,7 +399,7 @@ Changes 1.26.6 (2021-06-25) -------------------- +=================== * Deprecated the ``urllib3.contrib.ntlmpool`` module. urllib3 is not able to support it properly due to `reasons listed in this issue `_. @@ -114,7 +410,7 @@ Changes 1.26.5 (2021-05-26) -------------------- +=================== * Fixed deprecation warnings emitted in Python 3.10. * Updated vendored ``six`` library to 1.16.0. @@ -123,14 +419,14 @@ Changes 1.26.4 (2021-03-15) -------------------- +=================== * Changed behavior of the default ``SSLContext`` when connecting to HTTPS proxy during HTTPS requests. The default ``SSLContext`` now sets ``check_hostname=True``. 1.26.3 (2021-01-26) -------------------- +=================== * Fixed bytes and string comparison issue with headers (Pull #2141) @@ -140,21 +436,21 @@ Changes 1.26.2 (2020-11-12) -------------------- +=================== * Fixed an issue where ``wrap_socket`` and ``CERT_REQUIRED`` wouldn't be imported properly on Python 2.7.8 and earlier (Pull #2052) 1.26.1 (2020-11-11) -------------------- +=================== * Fixed an issue where two ``User-Agent`` headers would be sent if a ``User-Agent`` header key is passed as ``bytes`` (Pull #2047) 1.26.0 (2020-11-10) -------------------- +=================== * **NOTE: urllib3 v2.0 will drop support for Python 2**. `Read more in the v2.0 Roadmap `_. @@ -173,7 +469,7 @@ Changes * Added default ``User-Agent`` header to every request (Pull #1750) -* Added ``urllib3.util.SKIP_HEADER`` for skipping ``User-Agent``, ``Accept-Encoding``, +* Added ``urllib3.util.SKIP_HEADER`` for skipping ``User-Agent``, ``Accept-Encoding``, and ``Host`` headers from being automatically emitted with requests (Pull #2018) * Collapse ``transfer-encoding: chunked`` request data and framing into @@ -196,7 +492,7 @@ Changes 1.25.11 (2020-10-19) --------------------- +==================== * Fix retry backoff time parsed from ``Retry-After`` header when given in the HTTP date format. The HTTP date was parsed as the local timezone @@ -209,7 +505,7 @@ Changes 1.25.10 (2020-07-22) --------------------- +==================== * Added support for ``SSLKEYLOGFILE`` environment variable for logging TLS session keys with use with programs like @@ -228,7 +524,7 @@ Changes 1.25.9 (2020-04-16) -------------------- +=================== * Added ``InvalidProxyConfigurationWarning`` which is raised when erroneously specifying an HTTPS proxy URL. urllib3 doesn't currently @@ -252,7 +548,7 @@ Changes 1.25.8 (2020-01-20) -------------------- +=================== * Drop support for EOL Python 3.4 (Pull #1774) @@ -260,7 +556,7 @@ Changes 1.25.7 (2019-11-11) -------------------- +=================== * Preserve ``chunked`` parameter on retries (Pull #1715, Pull #1734) @@ -274,14 +570,14 @@ Changes 1.25.6 (2019-09-24) -------------------- +=================== * Fix issue where tilde (``~``) characters were incorrectly percent-encoded in the path. (Pull #1692) 1.25.5 (2019-09-19) -------------------- +=================== * Add mitigation for BPO-37428 affecting Python <3.7.4 and OpenSSL 1.1.1+ which caused certificate verification to be enabled when using ``cert_reqs=CERT_NONE``. @@ -289,7 +585,7 @@ Changes 1.25.4 (2019-09-19) -------------------- +=================== * Propagate Retry-After header settings to subsequent retries. (Pull #1607) @@ -308,7 +604,7 @@ Changes 1.25.3 (2019-05-23) -------------------- +=================== * Change ``HTTPSConnection`` to load system CA certificates when ``ca_certs``, ``ca_cert_dir``, and ``ssl_context`` are @@ -318,7 +614,7 @@ Changes 1.25.2 (2019-04-28) -------------------- +=================== * Change ``is_ipaddress`` to not detect IPvFuture addresses. (Pull #1583) @@ -327,7 +623,7 @@ Changes 1.25.1 (2019-04-24) -------------------- +=================== * Add support for Google's ``Brotli`` package. (Pull #1572, Pull #1579) @@ -335,7 +631,7 @@ Changes 1.25 (2019-04-22) ------------------ +================= * Require and validate certificates by default when using HTTPS (Pull #1507) @@ -362,12 +658,12 @@ Changes * Implemented a more efficient ``HTTPResponse.__iter__()`` method. (Issue #1483) 1.24.3 (2019-05-01) -------------------- +=================== * Apply fix for CVE-2019-9740. (Pull #1591) 1.24.2 (2019-04-17) -------------------- +=================== * Don't load system certificates by default when any other ``ca_certs``, ``ca_certs_dir`` or ``ssl_context`` parameters are specified. @@ -378,7 +674,7 @@ Changes 1.24.1 (2018-11-02) -------------------- +=================== * Remove quadratic behavior within ``GzipDecoder.decompress()`` (Issue #1467) @@ -386,7 +682,7 @@ Changes 1.24 (2018-10-16) ------------------ +================= * Allow key_server_hostname to be specified when initializing a PoolManager to allow custom SNI to be overridden. (Pull #1449) @@ -412,7 +708,7 @@ Changes 1.23 (2018-06-04) ------------------ +================= * Allow providing a list of headers to strip from requests when redirecting to a different host. Defaults to the ``Authorization`` header. Different @@ -440,7 +736,7 @@ Changes 1.22 (2017-07-20) ------------------ +================= * Fixed missing brackets in ``HTTP CONNECT`` when connecting to IPv6 address via IPv6 proxy. (Issue #1222) @@ -457,7 +753,7 @@ Changes 1.21.1 (2017-05-02) -------------------- +=================== * Fixed SecureTransport issue that would cause long delays in response body delivery. (Pull #1154) @@ -471,7 +767,7 @@ Changes 1.21 (2017-04-25) ------------------ +================= * Improved performance of certain selector system calls on Python 3.5 and later. (Pull #1095) @@ -504,7 +800,7 @@ Changes 1.20 (2017-01-19) ------------------ +================= * Added support for waiting for I/O using selectors other than select, improving urllib3's behaviour with large numbers of concurrent connections. @@ -541,13 +837,13 @@ Changes 1.19.1 (2016-11-16) -------------------- +=================== * Fixed AppEngine import that didn't function on Python 3.5. (Pull #1025) 1.19 (2016-11-03) ------------------ +================= * urllib3 now respects Retry-After headers on 413, 429, and 503 responses when using the default retry logic. (Pull #955) @@ -567,7 +863,7 @@ Changes 1.18.1 (2016-10-27) -------------------- +=================== * CVE-2016-9015. Users who are using urllib3 version 1.17 or 1.18 along with PyOpenSSL injection and OpenSSL 1.1.0 *must* upgrade to this version. This @@ -582,7 +878,7 @@ Changes 1.18 (2016-09-26) ------------------ +================= * Fixed incorrect message for IncompleteRead exception. (Pull #973) @@ -596,7 +892,7 @@ Changes 1.17 (2016-09-06) ------------------ +================= * Accept ``SSLContext`` objects for use in SSL/TLS negotiation. (Issue #835) @@ -642,7 +938,7 @@ Changes 1.16 (2016-06-11) ------------------ +================= * Disable IPv6 DNS when IPv6 connections are not possible. (Issue #840) @@ -667,13 +963,13 @@ Changes 1.15.1 (2016-04-11) -------------------- +=================== * Fix packaging to include backports module. (Issue #841) 1.15 (2016-04-06) ------------------ +================= * Added Retry(raise_on_status=False). (Issue #720) @@ -697,7 +993,7 @@ Changes 1.14 (2015-12-29) ------------------ +================= * contrib: SOCKS proxy support! (Issue #762) @@ -706,13 +1002,13 @@ Changes 1.13.1 (2015-12-18) -------------------- +=================== * Fixed regression in IPv6 + SSL for match_hostname. (Issue #761) 1.13 (2015-12-14) ------------------ +================= * Fixed ``pip install urllib3[secure]`` on modern pip. (Issue #706) @@ -729,7 +1025,7 @@ Changes 1.12 (2015-09-03) ------------------ +================= * Rely on ``six`` for importing ``httplib`` to work around conflicts with other Python 3 shims. (Issue #688) @@ -742,7 +1038,7 @@ Changes 1.11 (2015-07-21) ------------------ +================= * When ``ca_certs`` is given, ``cert_reqs`` defaults to ``'CERT_REQUIRED'``. (Issue #650) @@ -787,7 +1083,7 @@ Changes (Issue #674) 1.10.4 (2015-05-03) -------------------- +=================== * Migrate tests to Tornado 4. (Issue #594) @@ -803,7 +1099,7 @@ Changes 1.10.3 (2015-04-21) -------------------- +=================== * Emit ``InsecurePlatformWarning`` when SSLContext object is missing. (Issue #558) @@ -824,7 +1120,7 @@ Changes 1.10.2 (2015-02-25) -------------------- +=================== * Fix file descriptor leakage on retries. (Issue #548) @@ -836,7 +1132,7 @@ Changes 1.10.1 (2015-02-10) -------------------- +=================== * Pools can be used as context managers. (Issue #545) @@ -850,7 +1146,7 @@ Changes 1.10 (2014-12-14) ------------------ +================= * Disabled SSLv3. (Issue #473) @@ -882,7 +1178,7 @@ Changes 1.9.1 (2014-09-13) ------------------- +================== * Apply socket arguments before binding. (Issue #427) @@ -903,7 +1199,7 @@ Changes 1.9 (2014-07-04) ----------------- +================ * Shuffled around development-related files. If you're maintaining a distro package of urllib3, you may need to tweak things. (Issue #415) @@ -940,7 +1236,7 @@ Changes 1.8.3 (2014-06-23) ------------------- +================== * Fix TLS verification when using a proxy in Python 3.4.1. (Issue #385) @@ -962,13 +1258,13 @@ Changes 1.8.2 (2014-04-17) ------------------- +================== * Fix ``urllib3.util`` not being included in the package. 1.8.1 (2014-04-17) ------------------- +================== * Fix AppEngine bug of HTTPS requests going out as HTTP. (Issue #356) @@ -979,7 +1275,7 @@ Changes 1.8 (2014-03-04) ----------------- +================ * Improved url parsing in ``urllib3.util.parse_url`` (properly parse '@' in username, and blank ports like 'hostname:'). @@ -1031,7 +1327,7 @@ Changes 1.7.1 (2013-09-25) ------------------- +================== * Added granular timeout support with new ``urllib3.util.Timeout`` class. (Issue #231) @@ -1040,7 +1336,7 @@ Changes 1.7 (2013-08-14) ----------------- +================ * More exceptions are now pickle-able, with tests. (Issue #174) @@ -1079,7 +1375,7 @@ Changes 1.6 (2013-04-25) ----------------- +================ * Contrib: Optional SNI support for Py2 using PyOpenSSL. (Issue #156) @@ -1139,7 +1435,7 @@ Changes 1.5 (2012-08-02) ----------------- +================ * Added ``urllib3.add_stderr_logger()`` for quickly enabling STDERR debug logging in urllib3. @@ -1164,7 +1460,7 @@ Changes 1.4 (2012-06-16) ----------------- +================ * Minor AppEngine-related fixes. @@ -1176,7 +1472,7 @@ Changes 1.3 (2012-03-25) ----------------- +================ * Removed pre-1.0 deprecated API. @@ -1195,13 +1491,13 @@ Changes 1.2.2 (2012-02-06) ------------------- +================== * Fixed packaging bug of not shipping ``test-requirements.txt``. (Issue #47) 1.2.1 (2012-02-05) ------------------- +================== * Fixed another bug related to when ``ssl`` module is not available. (Issue #41) @@ -1210,7 +1506,7 @@ Changes 1.2 (2012-01-29) ----------------- +================ * Added Python 3 support (tested on 3.2.2) @@ -1236,7 +1532,7 @@ Changes 1.1 (2012-01-07) ----------------- +================ * Refactored ``dummyserver`` to its own root namespace module (used for testing). @@ -1253,7 +1549,7 @@ Changes 1.0.2 (2011-11-04) ------------------- +================== * Fixed typo in ``VerifiedHTTPSConnection`` which would only present as a bug if you're using the object manually. (Thanks pyos) @@ -1266,14 +1562,14 @@ Changes 1.0.1 (2011-10-10) ------------------- +================== * Fixed a bug where the same connection would get returned into the pool twice, causing extraneous "HttpConnectionPool is full" log warnings. 1.0 (2011-10-08) ----------------- +================ * Added ``PoolManager`` with LRU expiration of connections (tested and documented). @@ -1296,13 +1592,13 @@ Changes 0.4.1 (2011-07-17) ------------------- +================== * Minor bug fixes, code cleanup. 0.4 (2011-03-01) ----------------- +================ * Better unicode support. * Added ``VerifiedHTTPSConnection``. @@ -1311,13 +1607,13 @@ Changes 0.3.1 (2010-07-13) ------------------- +================== * Added ``assert_host_name`` optional parameter. Now compatible with proxies. 0.3 (2009-12-10) ----------------- +================ * Added HTTPS support. * Minor bug fixes. @@ -1326,13 +1622,13 @@ Changes 0.2 (2008-11-17) ----------------- +================ * Added unit tests. * Bug fixes. 0.1 (2008-11-16) ----------------- +================ * First release. diff --git a/LICENSE.txt b/LICENSE.txt index 429a176..e6183d0 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2008-2020 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +Copyright (c) 2008-2020 Andrey Petrov and contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 42a4117..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include README.rst CHANGES.rst LICENSE.txt dev-requirements.txt Makefile -recursive-include dummyserver * -recursive-include test * -recursive-include docs * -recursive-exclude docs/_build * diff --git a/PKG-INFO b/PKG-INFO index 4336677..d2064b9 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,51 +1,60 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.4 Name: urllib3 -Version: 1.26.18 +Version: 2.3.0 Summary: HTTP library with thread-safe connection pooling, file post, and more. -Home-page: https://urllib3.readthedocs.io/ -Author: Andrey Petrov -Author-email: andrey.petrov@shazow.net -License: MIT -Project-URL: Documentation, https://urllib3.readthedocs.io/ +Project-URL: Changelog, https://github.com/urllib3/urllib3/blob/main/CHANGES.rst +Project-URL: Documentation, https://urllib3.readthedocs.io Project-URL: Code, https://github.com/urllib3/urllib3 Project-URL: Issue tracker, https://github.com/urllib3/urllib3/issues -Keywords: urllib httplib threadsafe filepost http https ssl pooling +Author-email: Andrey Petrov +Maintainer-email: Seth Michael Larson , Quentin Pradet , Illia Volochii +License-File: LICENSE.txt +Keywords: filepost,http,httplib,https,pooling,ssl,threadsafe,urllib Classifier: Environment :: Web Environment Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3 :: Only Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Internet :: WWW/HTTP Classifier: Topic :: Software Development :: Libraries -Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* -Description-Content-Type: text/x-rst -License-File: LICENSE.txt +Requires-Python: >=3.9 Provides-Extra: brotli -Requires-Dist: brotli==1.0.9; (os_name != "nt" and python_version < "3" and platform_python_implementation == "CPython") and extra == "brotli" -Requires-Dist: brotli>=1.0.9; (python_version >= "3" and platform_python_implementation == "CPython") and extra == "brotli" -Requires-Dist: brotlicffi>=0.8.0; ((os_name != "nt" or python_version >= "3") and platform_python_implementation != "CPython") and extra == "brotli" -Requires-Dist: brotlipy>=0.6.0; (os_name == "nt" and python_version < "3") and extra == "brotli" -Provides-Extra: secure -Requires-Dist: pyOpenSSL>=0.14; extra == "secure" -Requires-Dist: cryptography>=1.3.4; extra == "secure" -Requires-Dist: idna>=2.0.0; extra == "secure" -Requires-Dist: certifi; extra == "secure" -Requires-Dist: ipaddress; python_version == "2.7" and extra == "secure" -Requires-Dist: urllib3-secure-extra; extra == "secure" +Requires-Dist: brotli>=1.0.9; (platform_python_implementation == 'CPython') and extra == 'brotli' +Requires-Dist: brotlicffi>=0.8.0; (platform_python_implementation != 'CPython') and extra == 'brotli' +Provides-Extra: h2 +Requires-Dist: h2<5,>=4; extra == 'h2' Provides-Extra: socks -Requires-Dist: PySocks!=1.5.7,<2.0,>=1.5.6; extra == "socks" +Requires-Dist: pysocks!=1.5.7,<2.0,>=1.5.6; extra == 'socks' +Provides-Extra: zstd +Requires-Dist: zstandard>=0.18.0; extra == 'zstd' +Description-Content-Type: text/markdown + +

+ +![urllib3](https://github.com/urllib3/urllib3/raw/main/docs/_static/banner_github.svg) +

+ +

+ PyPI Version + Python Versions + Join our Discord + Coverage Status + Build Status on GitHub + Documentation Status
+ OpenSSF Scorecard + SLSA 3 + CII Best Practices +

urllib3 is a powerful, *user-friendly* HTTP client for Python. Much of the Python ecosystem already uses urllib3 and you should too. @@ -57,1436 +66,89 @@ standard libraries: - Client-side SSL/TLS verification. - File uploads with multipart encoding. - Helpers for retrying requests and dealing with HTTP redirects. -- Support for gzip, deflate, and brotli encoding. +- Support for gzip, deflate, brotli, and zstd encoding. - Proxy support for HTTP and SOCKS. - 100% test coverage. urllib3 is powerful and easy to use: -.. code-block:: python +```python3 +>>> import urllib3 +>>> resp = urllib3.request("GET", "http://httpbin.org/robots.txt") +>>> resp.status +200 +>>> resp.data +b"User-agent: *\nDisallow: /deny\n" +``` + +## Installing - >>> import urllib3 - >>> http = urllib3.PoolManager() - >>> r = http.request('GET', 'http://httpbin.org/robots.txt') - >>> r.status - 200 - >>> r.data - 'User-agent: *\nDisallow: /deny\n' +urllib3 can be installed with [pip](https://pip.pypa.io): +```bash +$ python -m pip install urllib3 +``` -Installing ----------- +Alternatively, you can grab the latest source code from [GitHub](https://github.com/urllib3/urllib3): -urllib3 can be installed with `pip `_:: +```bash +$ git clone https://github.com/urllib3/urllib3.git +$ cd urllib3 +$ pip install . +``` - $ python -m pip install urllib3 -Alternatively, you can grab the latest source code from `GitHub `_:: +## Documentation - $ git clone https://github.com/urllib3/urllib3.git - $ cd urllib3 - $ git checkout 1.26.x - $ pip install . +urllib3 has usage and reference documentation at [urllib3.readthedocs.io](https://urllib3.readthedocs.io). -Documentation -------------- +## Community -urllib3 has usage and reference documentation at `urllib3.readthedocs.io `_. +urllib3 has a [community Discord channel](https://discord.gg/urllib3) for asking questions and +collaborating with other contributors. Drop by and say hello 👋 -Contributing ------------- +## Contributing urllib3 happily accepts contributions. Please see our -`contributing documentation `_ +[contributing documentation](https://urllib3.readthedocs.io/en/latest/contributing.html) for some tips on getting started. -Security Disclosures --------------------- +## Security Disclosures To report a security vulnerability, please use the -`Tidelift security contact `_. +[Tidelift security contact](https://tidelift.com/security). Tidelift will coordinate the fix and disclosure with maintainers. -Maintainers ------------ +## Maintainers -- `@sethmlarson `__ (Seth M. Larson) -- `@pquentin `__ (Quentin Pradet) -- `@theacodes `__ (Thea Flowers) -- `@haikuginger `__ (Jess Shapiro) -- `@lukasa `__ (Cory Benfield) -- `@sigmavirus24 `__ (Ian Stapleton Cordasco) -- `@shazow `__ (Andrey Petrov) +- [@sethmlarson](https://github.com/sethmlarson) (Seth M. Larson) +- [@pquentin](https://github.com/pquentin) (Quentin Pradet) +- [@illia-v](https://github.com/illia-v) (Illia Volochii) +- [@theacodes](https://github.com/theacodes) (Thea Flowers) +- [@haikuginger](https://github.com/haikuginger) (Jess Shapiro) +- [@lukasa](https://github.com/lukasa) (Cory Benfield) +- [@sigmavirus24](https://github.com/sigmavirus24) (Ian Stapleton Cordasco) +- [@shazow](https://github.com/shazow) (Andrey Petrov) 👋 -Sponsorship ------------ - -If your company benefits from this library, please consider `sponsoring its -development `_. - - -For Enterprise --------------- - -.. |tideliftlogo| image:: https://nedbatchelder.com/pix/Tidelift_Logos_RGB_Tidelift_Shorthand_On-White_small.png - :width: 75 - :alt: Tidelift - -.. list-table:: - :widths: 10 100 - - * - |tideliftlogo| - - Professional support for urllib3 is available as part of the `Tidelift - Subscription`_. Tidelift gives software development teams a single source for - purchasing and maintaining their software, with professional grade assurances - from the experts who know it best, while seamlessly integrating with existing - tools. - -.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-urllib3?utm_source=pypi-urllib3&utm_medium=referral&utm_campaign=readme - - -Changes -======= - -1.26.18 (2023-10-17) --------------------- - -* Made body stripped from HTTP requests changing the request method to GET after HTTP 303 "See Other" redirect responses. - - -1.26.17 (2023-10-02) --------------------- - -* Added the ``Cookie`` header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via ``Retry.remove_headers_on_redirect``. - - -1.26.16 (2023-05-23) --------------------- - -* Fixed thread-safety issue where accessing a ``PoolManager`` with many distinct origins - would cause connection pools to be closed while requests are in progress (`#2954 `_) - - -1.26.15 (2023-03-10) --------------------- - -* Fix socket timeout value when ``HTTPConnection`` is reused (`#2645 `__) -* Remove "!" character from the unreserved characters in IPv6 Zone ID parsing - (`#2899 `__) -* Fix IDNA handling of '\x80' byte (`#2901 `__) - -1.26.14 (2023-01-11) --------------------- - -* Fixed parsing of port 0 (zero) returning None, instead of 0. (`#2850 `__) -* Removed deprecated getheaders() calls in contrib module. - -1.26.13 (2022-11-23) --------------------- - -* Deprecated the ``HTTPResponse.getheaders()`` and ``HTTPResponse.getheader()`` methods. -* Fixed an issue where parsing a URL with leading zeroes in the port would be rejected - even when the port number after removing the zeroes was valid. -* Fixed a deprecation warning when using cryptography v39.0.0. -* Removed the ``<4`` in the ``Requires-Python`` packaging metadata field. - - -1.26.12 (2022-08-22) --------------------- - -* Deprecated the `urllib3[secure]` extra and the `urllib3.contrib.pyopenssl` module. - Both will be removed in v2.x. See this `GitHub issue `_ - for justification and info on how to migrate. - - -1.26.11 (2022-07-25) --------------------- - -* Fixed an issue where reading more than 2 GiB in a call to ``HTTPResponse.read`` would - raise an ``OverflowError`` on Python 3.9 and earlier. - - -1.26.10 (2022-07-07) --------------------- - -* Removed support for Python 3.5 -* Fixed an issue where a ``ProxyError`` recommending configuring the proxy as HTTP - instead of HTTPS could appear even when an HTTPS proxy wasn't configured. - - -1.26.9 (2022-03-16) -------------------- - -* Changed ``urllib3[brotli]`` extra to favor installing Brotli libraries that are still - receiving updates like ``brotli`` and ``brotlicffi`` instead of ``brotlipy``. - This change does not impact behavior of urllib3, only which dependencies are installed. -* Fixed a socket leaking when ``HTTPSConnection.connect()`` raises an exception. -* Fixed ``server_hostname`` being forwarded from ``PoolManager`` to ``HTTPConnectionPool`` - when requesting an HTTP URL. Should only be forwarded when requesting an HTTPS URL. - - -1.26.8 (2022-01-07) -------------------- - -* Added extra message to ``urllib3.exceptions.ProxyError`` when urllib3 detects that - a proxy is configured to use HTTPS but the proxy itself appears to only use HTTP. -* Added a mention of the size of the connection pool when discarding a connection due to the pool being full. -* Added explicit support for Python 3.11. -* Deprecated the ``Retry.MAX_BACKOFF`` class property in favor of ``Retry.DEFAULT_MAX_BACKOFF`` - to better match the rest of the default parameter names. ``Retry.MAX_BACKOFF`` is removed in v2.0. -* Changed location of the vendored ``ssl.match_hostname`` function from ``urllib3.packages.ssl_match_hostname`` - to ``urllib3.util.ssl_match_hostname`` to ensure Python 3.10+ compatibility after being repackaged - by downstream distributors. -* Fixed absolute imports, all imports are now relative. - - -1.26.7 (2021-09-22) -------------------- - -* Fixed a bug with HTTPS hostname verification involving IP addresses and lack - of SNI. (Issue #2400) -* Fixed a bug where IPv6 braces weren't stripped during certificate hostname - matching. (Issue #2240) - - -1.26.6 (2021-06-25) -------------------- - -* Deprecated the ``urllib3.contrib.ntlmpool`` module. urllib3 is not able to support - it properly due to `reasons listed in this issue `_. - If you are a user of this module please leave a comment. -* Changed ``HTTPConnection.request_chunked()`` to not erroneously emit multiple - ``Transfer-Encoding`` headers in the case that one is already specified. -* Fixed typo in deprecation message to recommend ``Retry.DEFAULT_ALLOWED_METHODS``. - - -1.26.5 (2021-05-26) -------------------- - -* Fixed deprecation warnings emitted in Python 3.10. -* Updated vendored ``six`` library to 1.16.0. -* Improved performance of URL parser when splitting - the authority component. - - -1.26.4 (2021-03-15) -------------------- - -* Changed behavior of the default ``SSLContext`` when connecting to HTTPS proxy - during HTTPS requests. The default ``SSLContext`` now sets ``check_hostname=True``. - - -1.26.3 (2021-01-26) -------------------- - -* Fixed bytes and string comparison issue with headers (Pull #2141) - -* Changed ``ProxySchemeUnknown`` error message to be - more actionable if the user supplies a proxy URL without - a scheme. (Pull #2107) - - -1.26.2 (2020-11-12) -------------------- - -* Fixed an issue where ``wrap_socket`` and ``CERT_REQUIRED`` wouldn't - be imported properly on Python 2.7.8 and earlier (Pull #2052) - - -1.26.1 (2020-11-11) -------------------- - -* Fixed an issue where two ``User-Agent`` headers would be sent if a - ``User-Agent`` header key is passed as ``bytes`` (Pull #2047) - - -1.26.0 (2020-11-10) -------------------- - -* **NOTE: urllib3 v2.0 will drop support for Python 2**. - `Read more in the v2.0 Roadmap `_. - -* Added support for HTTPS proxies contacting HTTPS servers (Pull #1923, Pull #1806) - -* Deprecated negotiating TLSv1 and TLSv1.1 by default. Users that - still wish to use TLS earlier than 1.2 without a deprecation warning - should opt-in explicitly by setting ``ssl_version=ssl.PROTOCOL_TLSv1_1`` (Pull #2002) - **Starting in urllib3 v2.0: Connections that receive a ``DeprecationWarning`` will fail** - -* Deprecated ``Retry`` options ``Retry.DEFAULT_METHOD_WHITELIST``, ``Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST`` - and ``Retry(method_whitelist=...)`` in favor of ``Retry.DEFAULT_ALLOWED_METHODS``, - ``Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT``, and ``Retry(allowed_methods=...)`` - (Pull #2000) **Starting in urllib3 v2.0: Deprecated options will be removed** - -* Added default ``User-Agent`` header to every request (Pull #1750) - -* Added ``urllib3.util.SKIP_HEADER`` for skipping ``User-Agent``, ``Accept-Encoding``, - and ``Host`` headers from being automatically emitted with requests (Pull #2018) - -* Collapse ``transfer-encoding: chunked`` request data and framing into - the same ``socket.send()`` call (Pull #1906) - -* Send ``http/1.1`` ALPN identifier with every TLS handshake by default (Pull #1894) - -* Properly terminate SecureTransport connections when CA verification fails (Pull #1977) - -* Don't emit an ``SNIMissingWarning`` when passing ``server_hostname=None`` - to SecureTransport (Pull #1903) - -* Disabled requesting TLSv1.2 session tickets as they weren't being used by urllib3 (Pull #1970) - -* Suppress ``BrokenPipeError`` when writing request body after the server - has closed the socket (Pull #1524) - -* Wrap ``ssl.SSLError`` that can be raised from reading a socket (e.g. "bad MAC") - into an ``urllib3.exceptions.SSLError`` (Pull #1939) - - -1.25.11 (2020-10-19) --------------------- - -* Fix retry backoff time parsed from ``Retry-After`` header when given - in the HTTP date format. The HTTP date was parsed as the local timezone - rather than accounting for the timezone in the HTTP date (typically - UTC) (Pull #1932, Pull #1935, Pull #1938, Pull #1949) - -* Fix issue where an error would be raised when the ``SSLKEYLOGFILE`` - environment variable was set to the empty string. Now ``SSLContext.keylog_file`` - is not set in this situation (Pull #2016) - - -1.25.10 (2020-07-22) --------------------- - -* Added support for ``SSLKEYLOGFILE`` environment variable for - logging TLS session keys with use with programs like - Wireshark for decrypting captured web traffic (Pull #1867) - -* Fixed loading of SecureTransport libraries on macOS Big Sur - due to the new dynamic linker cache (Pull #1905) - -* Collapse chunked request bodies data and framing into one - call to ``send()`` to reduce the number of TCP packets by 2-4x (Pull #1906) - -* Don't insert ``None`` into ``ConnectionPool`` if the pool - was empty when requesting a connection (Pull #1866) - -* Avoid ``hasattr`` call in ``BrotliDecoder.decompress()`` (Pull #1858) - - -1.25.9 (2020-04-16) -------------------- - -* Added ``InvalidProxyConfigurationWarning`` which is raised when - erroneously specifying an HTTPS proxy URL. urllib3 doesn't currently - support connecting to HTTPS proxies but will soon be able to - and we would like users to migrate properly without much breakage. - - See `this GitHub issue `_ - for more information on how to fix your proxy config. (Pull #1851) - -* Drain connection after ``PoolManager`` redirect (Pull #1817) - -* Ensure ``load_verify_locations`` raises ``SSLError`` for all backends (Pull #1812) - -* Rename ``VerifiedHTTPSConnection`` to ``HTTPSConnection`` (Pull #1805) - -* Allow the CA certificate data to be passed as a string (Pull #1804) - -* Raise ``ValueError`` if method contains control characters (Pull #1800) - -* Add ``__repr__`` to ``Timeout`` (Pull #1795) - - -1.25.8 (2020-01-20) -------------------- - -* Drop support for EOL Python 3.4 (Pull #1774) - -* Optimize _encode_invalid_chars (Pull #1787) - - -1.25.7 (2019-11-11) -------------------- - -* Preserve ``chunked`` parameter on retries (Pull #1715, Pull #1734) - -* Allow unset ``SERVER_SOFTWARE`` in App Engine (Pull #1704, Issue #1470) - -* Fix issue where URL fragment was sent within the request target. (Pull #1732) - -* Fix issue where an empty query section in a URL would fail to parse. (Pull #1732) - -* Remove TLS 1.3 support in SecureTransport due to Apple removing support (Pull #1703) - - -1.25.6 (2019-09-24) -------------------- - -* Fix issue where tilde (``~``) characters were incorrectly - percent-encoded in the path. (Pull #1692) - - -1.25.5 (2019-09-19) -------------------- - -* Add mitigation for BPO-37428 affecting Python <3.7.4 and OpenSSL 1.1.1+ which - caused certificate verification to be enabled when using ``cert_reqs=CERT_NONE``. - (Issue #1682) - - -1.25.4 (2019-09-19) -------------------- - -* Propagate Retry-After header settings to subsequent retries. (Pull #1607) - -* Fix edge case where Retry-After header was still respected even when - explicitly opted out of. (Pull #1607) - -* Remove dependency on ``rfc3986`` for URL parsing. - -* Fix issue where URLs containing invalid characters within ``Url.auth`` would - raise an exception instead of percent-encoding those characters. - -* Add support for ``HTTPResponse.auto_close = False`` which makes HTTP responses - work well with BufferedReaders and other ``io`` module features. (Pull #1652) - -* Percent-encode invalid characters in URL for ``HTTPConnectionPool.request()`` (Pull #1673) - - -1.25.3 (2019-05-23) -------------------- - -* Change ``HTTPSConnection`` to load system CA certificates - when ``ca_certs``, ``ca_cert_dir``, and ``ssl_context`` are - unspecified. (Pull #1608, Issue #1603) - -* Upgrade bundled rfc3986 to v1.3.2. (Pull #1609, Issue #1605) - - -1.25.2 (2019-04-28) -------------------- - -* Change ``is_ipaddress`` to not detect IPvFuture addresses. (Pull #1583) - -* Change ``parse_url`` to percent-encode invalid characters within the - path, query, and target components. (Pull #1586) - - -1.25.1 (2019-04-24) -------------------- - -* Add support for Google's ``Brotli`` package. (Pull #1572, Pull #1579) - -* Upgrade bundled rfc3986 to v1.3.1 (Pull #1578) - - -1.25 (2019-04-22) ------------------ - -* Require and validate certificates by default when using HTTPS (Pull #1507) - -* Upgraded ``urllib3.utils.parse_url()`` to be RFC 3986 compliant. (Pull #1487) - -* Added support for ``key_password`` for ``HTTPSConnectionPool`` to use - encrypted ``key_file`` without creating your own ``SSLContext`` object. (Pull #1489) - -* Add TLSv1.3 support to CPython, pyOpenSSL, and SecureTransport ``SSLContext`` - implementations. (Pull #1496) - -* Switched the default multipart header encoder from RFC 2231 to HTML 5 working draft. (Issue #303, Pull #1492) - -* Fixed issue where OpenSSL would block if an encrypted client private key was - given and no password was given. Instead an ``SSLError`` is raised. (Pull #1489) - -* Added support for Brotli content encoding. It is enabled automatically if - ``brotlipy`` package is installed which can be requested with - ``urllib3[brotli]`` extra. (Pull #1532) - -* Drop ciphers using DSS key exchange from default TLS cipher suites. - Improve default ciphers when using SecureTransport. (Pull #1496) - -* Implemented a more efficient ``HTTPResponse.__iter__()`` method. (Issue #1483) - -1.24.3 (2019-05-01) -------------------- - -* Apply fix for CVE-2019-9740. (Pull #1591) - -1.24.2 (2019-04-17) -------------------- - -* Don't load system certificates by default when any other ``ca_certs``, ``ca_certs_dir`` or - ``ssl_context`` parameters are specified. - -* Remove Authorization header regardless of case when redirecting to cross-site. (Issue #1510) - -* Add support for IPv6 addresses in subjectAltName section of certificates. (Issue #1269) - - -1.24.1 (2018-11-02) -------------------- - -* Remove quadratic behavior within ``GzipDecoder.decompress()`` (Issue #1467) - -* Restored functionality of ``ciphers`` parameter for ``create_urllib3_context()``. (Issue #1462) - - -1.24 (2018-10-16) ------------------ - -* Allow key_server_hostname to be specified when initializing a PoolManager to allow custom SNI to be overridden. (Pull #1449) - -* Test against Python 3.7 on AppVeyor. (Pull #1453) - -* Early-out ipv6 checks when running on App Engine. (Pull #1450) - -* Change ambiguous description of backoff_factor (Pull #1436) - -* Add ability to handle multiple Content-Encodings (Issue #1441 and Pull #1442) - -* Skip DNS names that can't be idna-decoded when using pyOpenSSL (Issue #1405). - -* Add a server_hostname parameter to HTTPSConnection which allows for - overriding the SNI hostname sent in the handshake. (Pull #1397) - -* Drop support for EOL Python 2.6 (Pull #1429 and Pull #1430) - -* Fixed bug where responses with header Content-Type: message/* erroneously - raised HeaderParsingError, resulting in a warning being logged. (Pull #1439) - -* Move urllib3 to src/urllib3 (Pull #1409) - - -1.23 (2018-06-04) ------------------ - -* Allow providing a list of headers to strip from requests when redirecting - to a different host. Defaults to the ``Authorization`` header. Different - headers can be set via ``Retry.remove_headers_on_redirect``. (Issue #1316) - -* Fix ``util.selectors._fileobj_to_fd`` to accept ``long`` (Issue #1247). - -* Dropped Python 3.3 support. (Pull #1242) - -* Put the connection back in the pool when calling stream() or read_chunked() on - a chunked HEAD response. (Issue #1234) - -* Fixed pyOpenSSL-specific ssl client authentication issue when clients - attempted to auth via certificate + chain (Issue #1060) - -* Add the port to the connectionpool connect print (Pull #1251) - -* Don't use the ``uuid`` module to create multipart data boundaries. (Pull #1380) - -* ``read_chunked()`` on a closed response returns no chunks. (Issue #1088) - -* Add Python 2.6 support to ``contrib.securetransport`` (Pull #1359) - -* Added support for auth info in url for SOCKS proxy (Pull #1363) - - -1.22 (2017-07-20) ------------------ - -* Fixed missing brackets in ``HTTP CONNECT`` when connecting to IPv6 address via - IPv6 proxy. (Issue #1222) - -* Made the connection pool retry on ``SSLError``. The original ``SSLError`` - is available on ``MaxRetryError.reason``. (Issue #1112) - -* Drain and release connection before recursing on retry/redirect. Fixes - deadlocks with a blocking connectionpool. (Issue #1167) - -* Fixed compatibility for cookiejar. (Issue #1229) - -* pyopenssl: Use vendored version of ``six``. (Issue #1231) - - -1.21.1 (2017-05-02) -------------------- - -* Fixed SecureTransport issue that would cause long delays in response body - delivery. (Pull #1154) - -* Fixed regression in 1.21 that threw exceptions when users passed the - ``socket_options`` flag to the ``PoolManager``. (Issue #1165) - -* Fixed regression in 1.21 that threw exceptions when users passed the - ``assert_hostname`` or ``assert_fingerprint`` flag to the ``PoolManager``. - (Pull #1157) - - -1.21 (2017-04-25) ------------------ - -* Improved performance of certain selector system calls on Python 3.5 and - later. (Pull #1095) - -* Resolved issue where the PyOpenSSL backend would not wrap SysCallError - exceptions appropriately when sending data. (Pull #1125) - -* Selectors now detects a monkey-patched select module after import for modules - that patch the select module like eventlet, greenlet. (Pull #1128) - -* Reduced memory consumption when streaming zlib-compressed responses - (as opposed to raw deflate streams). (Pull #1129) - -* Connection pools now use the entire request context when constructing the - pool key. (Pull #1016) - -* ``PoolManager.connection_from_*`` methods now accept a new keyword argument, - ``pool_kwargs``, which are merged with the existing ``connection_pool_kw``. - (Pull #1016) - -* Add retry counter for ``status_forcelist``. (Issue #1147) - -* Added ``contrib`` module for using SecureTransport on macOS: - ``urllib3.contrib.securetransport``. (Pull #1122) - -* urllib3 now only normalizes the case of ``http://`` and ``https://`` schemes: - for schemes it does not recognise, it assumes they are case-sensitive and - leaves them unchanged. - (Issue #1080) - - -1.20 (2017-01-19) ------------------ - -* Added support for waiting for I/O using selectors other than select, - improving urllib3's behaviour with large numbers of concurrent connections. - (Pull #1001) - -* Updated the date for the system clock check. (Issue #1005) - -* ConnectionPools now correctly consider hostnames to be case-insensitive. - (Issue #1032) - -* Outdated versions of PyOpenSSL now cause the PyOpenSSL contrib module - to fail when it is injected, rather than at first use. (Pull #1063) - -* Outdated versions of cryptography now cause the PyOpenSSL contrib module - to fail when it is injected, rather than at first use. (Issue #1044) - -* Automatically attempt to rewind a file-like body object when a request is - retried or redirected. (Pull #1039) - -* Fix some bugs that occur when modules incautiously patch the queue module. - (Pull #1061) - -* Prevent retries from occurring on read timeouts for which the request method - was not in the method whitelist. (Issue #1059) - -* Changed the PyOpenSSL contrib module to lazily load idna to avoid - unnecessarily bloating the memory of programs that don't need it. (Pull - #1076) - -* Add support for IPv6 literals with zone identifiers. (Pull #1013) - -* Added support for socks5h:// and socks4a:// schemes when working with SOCKS - proxies, and controlled remote DNS appropriately. (Issue #1035) - - -1.19.1 (2016-11-16) -------------------- - -* Fixed AppEngine import that didn't function on Python 3.5. (Pull #1025) - - -1.19 (2016-11-03) ------------------ - -* urllib3 now respects Retry-After headers on 413, 429, and 503 responses when - using the default retry logic. (Pull #955) - -* Remove markers from setup.py to assist ancient setuptools versions. (Issue - #986) - -* Disallow superscripts and other integerish things in URL ports. (Issue #989) - -* Allow urllib3's HTTPResponse.stream() method to continue to work with - non-httplib underlying FPs. (Pull #990) - -* Empty filenames in multipart headers are now emitted as such, rather than - being suppressed. (Issue #1015) - -* Prefer user-supplied Host headers on chunked uploads. (Issue #1009) - - -1.18.1 (2016-10-27) -------------------- - -* CVE-2016-9015. Users who are using urllib3 version 1.17 or 1.18 along with - PyOpenSSL injection and OpenSSL 1.1.0 *must* upgrade to this version. This - release fixes a vulnerability whereby urllib3 in the above configuration - would silently fail to validate TLS certificates due to erroneously setting - invalid flags in OpenSSL's ``SSL_CTX_set_verify`` function. These erroneous - flags do not cause a problem in OpenSSL versions before 1.1.0, which - interprets the presence of any flag as requesting certificate validation. - - There is no PR for this patch, as it was prepared for simultaneous disclosure - and release. The master branch received the same fix in Pull #1010. - - -1.18 (2016-09-26) ------------------ - -* Fixed incorrect message for IncompleteRead exception. (Pull #973) - -* Accept ``iPAddress`` subject alternative name fields in TLS certificates. - (Issue #258) - -* Fixed consistency of ``HTTPResponse.closed`` between Python 2 and 3. - (Issue #977) - -* Fixed handling of wildcard certificates when using PyOpenSSL. (Issue #979) - - -1.17 (2016-09-06) ------------------ - -* Accept ``SSLContext`` objects for use in SSL/TLS negotiation. (Issue #835) - -* ConnectionPool debug log now includes scheme, host, and port. (Issue #897) - -* Substantially refactored documentation. (Issue #887) - -* Used URLFetch default timeout on AppEngine, rather than hardcoding our own. - (Issue #858) - -* Normalize the scheme and host in the URL parser (Issue #833) - -* ``HTTPResponse`` contains the last ``Retry`` object, which now also - contains retries history. (Issue #848) - -* Timeout can no longer be set as boolean, and must be greater than zero. - (Pull #924) - -* Removed pyasn1 and ndg-httpsclient from dependencies used for PyOpenSSL. We - now use cryptography and idna, both of which are already dependencies of - PyOpenSSL. (Pull #930) - -* Fixed infinite loop in ``stream`` when amt=None. (Issue #928) - -* Try to use the operating system's certificates when we are using an - ``SSLContext``. (Pull #941) - -* Updated cipher suite list to allow ChaCha20+Poly1305. AES-GCM is preferred to - ChaCha20, but ChaCha20 is then preferred to everything else. (Pull #947) - -* Updated cipher suite list to remove 3DES-based cipher suites. (Pull #958) - -* Removed the cipher suite fallback to allow HIGH ciphers. (Pull #958) - -* Implemented ``length_remaining`` to determine remaining content - to be read. (Pull #949) - -* Implemented ``enforce_content_length`` to enable exceptions when - incomplete data chunks are received. (Pull #949) - -* Dropped connection start, dropped connection reset, redirect, forced retry, - and new HTTPS connection log levels to DEBUG, from INFO. (Pull #967) - - -1.16 (2016-06-11) ------------------ - -* Disable IPv6 DNS when IPv6 connections are not possible. (Issue #840) - -* Provide ``key_fn_by_scheme`` pool keying mechanism that can be - overridden. (Issue #830) - -* Normalize scheme and host to lowercase for pool keys, and include - ``source_address``. (Issue #830) - -* Cleaner exception chain in Python 3 for ``_make_request``. - (Issue #861) - -* Fixed installing ``urllib3[socks]`` extra. (Issue #864) - -* Fixed signature of ``ConnectionPool.close`` so it can actually safely be - called by subclasses. (Issue #873) - -* Retain ``release_conn`` state across retries. (Issues #651, #866) - -* Add customizable ``HTTPConnectionPool.ResponseCls``, which defaults to - ``HTTPResponse`` but can be replaced with a subclass. (Issue #879) - - -1.15.1 (2016-04-11) -------------------- - -* Fix packaging to include backports module. (Issue #841) - - -1.15 (2016-04-06) ------------------ - -* Added Retry(raise_on_status=False). (Issue #720) - -* Always use setuptools, no more distutils fallback. (Issue #785) - -* Dropped support for Python 3.2. (Issue #786) - -* Chunked transfer encoding when requesting with ``chunked=True``. - (Issue #790) - -* Fixed regression with IPv6 port parsing. (Issue #801) - -* Append SNIMissingWarning messages to allow users to specify it in - the PYTHONWARNINGS environment variable. (Issue #816) - -* Handle unicode headers in Py2. (Issue #818) - -* Log certificate when there is a hostname mismatch. (Issue #820) - -* Preserve order of request/response headers. (Issue #821) - - -1.14 (2015-12-29) ------------------ - -* contrib: SOCKS proxy support! (Issue #762) - -* Fixed AppEngine handling of transfer-encoding header and bug - in Timeout defaults checking. (Issue #763) - - -1.13.1 (2015-12-18) -------------------- - -* Fixed regression in IPv6 + SSL for match_hostname. (Issue #761) - - -1.13 (2015-12-14) ------------------ - -* Fixed ``pip install urllib3[secure]`` on modern pip. (Issue #706) - -* pyopenssl: Fixed SSL3_WRITE_PENDING error. (Issue #717) - -* pyopenssl: Support for TLSv1.1 and TLSv1.2. (Issue #696) - -* Close connections more defensively on exception. (Issue #734) - -* Adjusted ``read_chunked`` to handle gzipped, chunk-encoded bodies without - repeatedly flushing the decoder, to function better on Jython. (Issue #743) - -* Accept ``ca_cert_dir`` for SSL-related PoolManager configuration. (Issue #758) - - -1.12 (2015-09-03) ------------------ - -* Rely on ``six`` for importing ``httplib`` to work around - conflicts with other Python 3 shims. (Issue #688) - -* Add support for directories of certificate authorities, as supported by - OpenSSL. (Issue #701) - -* New exception: ``NewConnectionError``, raised when we fail to establish - a new connection, usually ``ECONNREFUSED`` socket error. - - -1.11 (2015-07-21) ------------------ - -* When ``ca_certs`` is given, ``cert_reqs`` defaults to - ``'CERT_REQUIRED'``. (Issue #650) - -* ``pip install urllib3[secure]`` will install Certifi and - PyOpenSSL as dependencies. (Issue #678) - -* Made ``HTTPHeaderDict`` usable as a ``headers`` input value - (Issues #632, #679) - -* Added `urllib3.contrib.appengine `_ - which has an ``AppEngineManager`` for using ``URLFetch`` in a - Google AppEngine environment. (Issue #664) - -* Dev: Added test suite for AppEngine. (Issue #631) - -* Fix performance regression when using PyOpenSSL. (Issue #626) - -* Passing incorrect scheme (e.g. ``foo://``) will raise - ``ValueError`` instead of ``AssertionError`` (backwards - compatible for now, but please migrate). (Issue #640) - -* Fix pools not getting replenished when an error occurs during a - request using ``release_conn=False``. (Issue #644) - -* Fix pool-default headers not applying for url-encoded requests - like GET. (Issue #657) - -* log.warning in Python 3 when headers are skipped due to parsing - errors. (Issue #642) - -* Close and discard connections if an error occurs during read. - (Issue #660) - -* Fix host parsing for IPv6 proxies. (Issue #668) - -* Separate warning type SubjectAltNameWarning, now issued once - per host. (Issue #671) - -* Fix ``httplib.IncompleteRead`` not getting converted to - ``ProtocolError`` when using ``HTTPResponse.stream()`` - (Issue #674) - -1.10.4 (2015-05-03) -------------------- - -* Migrate tests to Tornado 4. (Issue #594) - -* Append default warning configuration rather than overwrite. - (Issue #603) - -* Fix streaming decoding regression. (Issue #595) - -* Fix chunked requests losing state across keep-alive connections. - (Issue #599) - -* Fix hanging when chunked HEAD response has no body. (Issue #605) - - -1.10.3 (2015-04-21) -------------------- - -* Emit ``InsecurePlatformWarning`` when SSLContext object is missing. - (Issue #558) - -* Fix regression of duplicate header keys being discarded. - (Issue #563) - -* ``Response.stream()`` returns a generator for chunked responses. - (Issue #560) - -* Set upper-bound timeout when waiting for a socket in PyOpenSSL. - (Issue #585) - -* Work on platforms without `ssl` module for plain HTTP requests. - (Issue #587) - -* Stop relying on the stdlib's default cipher list. (Issue #588) - - -1.10.2 (2015-02-25) -------------------- - -* Fix file descriptor leakage on retries. (Issue #548) - -* Removed RC4 from default cipher list. (Issue #551) - -* Header performance improvements. (Issue #544) - -* Fix PoolManager not obeying redirect retry settings. (Issue #553) - - -1.10.1 (2015-02-10) -------------------- - -* Pools can be used as context managers. (Issue #545) - -* Don't re-use connections which experienced an SSLError. (Issue #529) - -* Don't fail when gzip decoding an empty stream. (Issue #535) - -* Add sha256 support for fingerprint verification. (Issue #540) - -* Fixed handling of header values containing commas. (Issue #533) - - -1.10 (2014-12-14) ------------------ - -* Disabled SSLv3. (Issue #473) - -* Add ``Url.url`` property to return the composed url string. (Issue #394) - -* Fixed PyOpenSSL + gevent ``WantWriteError``. (Issue #412) - -* ``MaxRetryError.reason`` will always be an exception, not string. - (Issue #481) - -* Fixed SSL-related timeouts not being detected as timeouts. (Issue #492) - -* Py3: Use ``ssl.create_default_context()`` when available. (Issue #473) - -* Emit ``InsecureRequestWarning`` for *every* insecure HTTPS request. - (Issue #496) - -* Emit ``SecurityWarning`` when certificate has no ``subjectAltName``. - (Issue #499) - -* Close and discard sockets which experienced SSL-related errors. - (Issue #501) - -* Handle ``body`` param in ``.request(...)``. (Issue #513) - -* Respect timeout with HTTPS proxy. (Issue #505) - -* PyOpenSSL: Handle ZeroReturnError exception. (Issue #520) - - -1.9.1 (2014-09-13) ------------------- - -* Apply socket arguments before binding. (Issue #427) - -* More careful checks if fp-like object is closed. (Issue #435) - -* Fixed packaging issues of some development-related files not - getting included. (Issue #440) - -* Allow performing *only* fingerprint verification. (Issue #444) - -* Emit ``SecurityWarning`` if system clock is waaay off. (Issue #445) - -* Fixed PyOpenSSL compatibility with PyPy. (Issue #450) - -* Fixed ``BrokenPipeError`` and ``ConnectionError`` handling in Py3. - (Issue #443) - - - -1.9 (2014-07-04) ----------------- - -* Shuffled around development-related files. If you're maintaining a distro - package of urllib3, you may need to tweak things. (Issue #415) - -* Unverified HTTPS requests will trigger a warning on the first request. See - our new `security documentation - `_ for details. - (Issue #426) - -* New retry logic and ``urllib3.util.retry.Retry`` configuration object. - (Issue #326) - -* All raised exceptions should now wrapped in a - ``urllib3.exceptions.HTTPException``-extending exception. (Issue #326) - -* All errors during a retry-enabled request should be wrapped in - ``urllib3.exceptions.MaxRetryError``, including timeout-related exceptions - which were previously exempt. Underlying error is accessible from the - ``.reason`` property. (Issue #326) - -* ``urllib3.exceptions.ConnectionError`` renamed to - ``urllib3.exceptions.ProtocolError``. (Issue #326) - -* Errors during response read (such as IncompleteRead) are now wrapped in - ``urllib3.exceptions.ProtocolError``. (Issue #418) - -* Requesting an empty host will raise ``urllib3.exceptions.LocationValueError``. - (Issue #417) - -* Catch read timeouts over SSL connections as - ``urllib3.exceptions.ReadTimeoutError``. (Issue #419) - -* Apply socket arguments before connecting. (Issue #427) - - -1.8.3 (2014-06-23) ------------------- - -* Fix TLS verification when using a proxy in Python 3.4.1. (Issue #385) - -* Add ``disable_cache`` option to ``urllib3.util.make_headers``. (Issue #393) - -* Wrap ``socket.timeout`` exception with - ``urllib3.exceptions.ReadTimeoutError``. (Issue #399) - -* Fixed proxy-related bug where connections were being reused incorrectly. - (Issues #366, #369) - -* Added ``socket_options`` keyword parameter which allows to define - ``setsockopt`` configuration of new sockets. (Issue #397) - -* Removed ``HTTPConnection.tcp_nodelay`` in favor of - ``HTTPConnection.default_socket_options``. (Issue #397) - -* Fixed ``TypeError`` bug in Python 2.6.4. (Issue #411) - - -1.8.2 (2014-04-17) ------------------- - -* Fix ``urllib3.util`` not being included in the package. - - -1.8.1 (2014-04-17) ------------------- - -* Fix AppEngine bug of HTTPS requests going out as HTTP. (Issue #356) - -* Don't install ``dummyserver`` into ``site-packages`` as it's only needed - for the test suite. (Issue #362) - -* Added support for specifying ``source_address``. (Issue #352) - - -1.8 (2014-03-04) ----------------- - -* Improved url parsing in ``urllib3.util.parse_url`` (properly parse '@' in - username, and blank ports like 'hostname:'). - -* New ``urllib3.connection`` module which contains all the HTTPConnection - objects. - -* Several ``urllib3.util.Timeout``-related fixes. Also changed constructor - signature to a more sensible order. [Backwards incompatible] - (Issues #252, #262, #263) - -* Use ``backports.ssl_match_hostname`` if it's installed. (Issue #274) - -* Added ``.tell()`` method to ``urllib3.response.HTTPResponse`` which - returns the number of bytes read so far. (Issue #277) - -* Support for platforms without threading. (Issue #289) - -* Expand default-port comparison in ``HTTPConnectionPool.is_same_host`` - to allow a pool with no specified port to be considered equal to to an - HTTP/HTTPS url with port 80/443 explicitly provided. (Issue #305) - -* Improved default SSL/TLS settings to avoid vulnerabilities. - (Issue #309) - -* Fixed ``urllib3.poolmanager.ProxyManager`` not retrying on connect errors. - (Issue #310) - -* Disable Nagle's Algorithm on the socket for non-proxies. A subset of requests - will send the entire HTTP request ~200 milliseconds faster; however, some of - the resulting TCP packets will be smaller. (Issue #254) - -* Increased maximum number of SubjectAltNames in ``urllib3.contrib.pyopenssl`` - from the default 64 to 1024 in a single certificate. (Issue #318) - -* Headers are now passed and stored as a custom - ``urllib3.collections_.HTTPHeaderDict`` object rather than a plain ``dict``. - (Issue #329, #333) - -* Headers no longer lose their case on Python 3. (Issue #236) - -* ``urllib3.contrib.pyopenssl`` now uses the operating system's default CA - certificates on inject. (Issue #332) - -* Requests with ``retries=False`` will immediately raise any exceptions without - wrapping them in ``MaxRetryError``. (Issue #348) - -* Fixed open socket leak with SSL-related failures. (Issue #344, #348) - - -1.7.1 (2013-09-25) ------------------- - -* Added granular timeout support with new ``urllib3.util.Timeout`` class. - (Issue #231) - -* Fixed Python 3.4 support. (Issue #238) - - -1.7 (2013-08-14) ----------------- - -* More exceptions are now pickle-able, with tests. (Issue #174) - -* Fixed redirecting with relative URLs in Location header. (Issue #178) - -* Support for relative urls in ``Location: ...`` header. (Issue #179) - -* ``urllib3.response.HTTPResponse`` now inherits from ``io.IOBase`` for bonus - file-like functionality. (Issue #187) - -* Passing ``assert_hostname=False`` when creating a HTTPSConnectionPool will - skip hostname verification for SSL connections. (Issue #194) - -* New method ``urllib3.response.HTTPResponse.stream(...)`` which acts as a - generator wrapped around ``.read(...)``. (Issue #198) - -* IPv6 url parsing enforces brackets around the hostname. (Issue #199) - -* Fixed thread race condition in - ``urllib3.poolmanager.PoolManager.connection_from_host(...)`` (Issue #204) - -* ``ProxyManager`` requests now include non-default port in ``Host: ...`` - header. (Issue #217) - -* Added HTTPS proxy support in ``ProxyManager``. (Issue #170 #139) - -* New ``RequestField`` object can be passed to the ``fields=...`` param which - can specify headers. (Issue #220) - -* Raise ``urllib3.exceptions.ProxyError`` when connecting to proxy fails. - (Issue #221) - -* Use international headers when posting file names. (Issue #119) - -* Improved IPv6 support. (Issue #203) - - -1.6 (2013-04-25) ----------------- - -* Contrib: Optional SNI support for Py2 using PyOpenSSL. (Issue #156) - -* ``ProxyManager`` automatically adds ``Host: ...`` header if not given. - -* Improved SSL-related code. ``cert_req`` now optionally takes a string like - "REQUIRED" or "NONE". Same with ``ssl_version`` takes strings like "SSLv23" - The string values reflect the suffix of the respective constant variable. - (Issue #130) - -* Vendored ``socksipy`` now based on Anorov's fork which handles unexpectedly - closed proxy connections and larger read buffers. (Issue #135) - -* Ensure the connection is closed if no data is received, fixes connection leak - on some platforms. (Issue #133) - -* Added SNI support for SSL/TLS connections on Py32+. (Issue #89) - -* Tests fixed to be compatible with Py26 again. (Issue #125) - -* Added ability to choose SSL version by passing an ``ssl.PROTOCOL_*`` constant - to the ``ssl_version`` parameter of ``HTTPSConnectionPool``. (Issue #109) - -* Allow an explicit content type to be specified when encoding file fields. - (Issue #126) - -* Exceptions are now pickleable, with tests. (Issue #101) - -* Fixed default headers not getting passed in some cases. (Issue #99) - -* Treat "content-encoding" header value as case-insensitive, per RFC 2616 - Section 3.5. (Issue #110) - -* "Connection Refused" SocketErrors will get retried rather than raised. - (Issue #92) - -* Updated vendored ``six``, no longer overrides the global ``six`` module - namespace. (Issue #113) - -* ``urllib3.exceptions.MaxRetryError`` contains a ``reason`` property holding - the exception that prompted the final retry. If ``reason is None`` then it - was due to a redirect. (Issue #92, #114) - -* Fixed ``PoolManager.urlopen()`` from not redirecting more than once. - (Issue #149) - -* Don't assume ``Content-Type: text/plain`` for multi-part encoding parameters - that are not files. (Issue #111) - -* Pass `strict` param down to ``httplib.HTTPConnection``. (Issue #122) - -* Added mechanism to verify SSL certificates by fingerprint (md5, sha1) or - against an arbitrary hostname (when connecting by IP or for misconfigured - servers). (Issue #140) - -* Streaming decompression support. (Issue #159) - - -1.5 (2012-08-02) ----------------- - -* Added ``urllib3.add_stderr_logger()`` for quickly enabling STDERR debug - logging in urllib3. - -* Native full URL parsing (including auth, path, query, fragment) available in - ``urllib3.util.parse_url(url)``. - -* Built-in redirect will switch method to 'GET' if status code is 303. - (Issue #11) - -* ``urllib3.PoolManager`` strips the scheme and host before sending the request - uri. (Issue #8) - -* New ``urllib3.exceptions.DecodeError`` exception for when automatic decoding, - based on the Content-Type header, fails. - -* Fixed bug with pool depletion and leaking connections (Issue #76). Added - explicit connection closing on pool eviction. Added - ``urllib3.PoolManager.clear()``. - -* 99% -> 100% unit test coverage. - - -1.4 (2012-06-16) ----------------- - -* Minor AppEngine-related fixes. - -* Switched from ``mimetools.choose_boundary`` to ``uuid.uuid4()``. - -* Improved url parsing. (Issue #73) - -* IPv6 url support. (Issue #72) - - -1.3 (2012-03-25) ----------------- - -* Removed pre-1.0 deprecated API. - -* Refactored helpers into a ``urllib3.util`` submodule. - -* Fixed multipart encoding to support list-of-tuples for keys with multiple - values. (Issue #48) - -* Fixed multiple Set-Cookie headers in response not getting merged properly in - Python 3. (Issue #53) - -* AppEngine support with Py27. (Issue #61) - -* Minor ``encode_multipart_formdata`` fixes related to Python 3 strings vs - bytes. - - -1.2.2 (2012-02-06) ------------------- - -* Fixed packaging bug of not shipping ``test-requirements.txt``. (Issue #47) - - -1.2.1 (2012-02-05) ------------------- - -* Fixed another bug related to when ``ssl`` module is not available. (Issue #41) - -* Location parsing errors now raise ``urllib3.exceptions.LocationParseError`` - which inherits from ``ValueError``. - - -1.2 (2012-01-29) ----------------- - -* Added Python 3 support (tested on 3.2.2) - -* Dropped Python 2.5 support (tested on 2.6.7, 2.7.2) - -* Use ``select.poll`` instead of ``select.select`` for platforms that support - it. - -* Use ``Queue.LifoQueue`` instead of ``Queue.Queue`` for more aggressive - connection reusing. Configurable by overriding ``ConnectionPool.QueueCls``. - -* Fixed ``ImportError`` during install when ``ssl`` module is not available. - (Issue #41) - -* Fixed ``PoolManager`` redirects between schemes (such as HTTP -> HTTPS) not - completing properly. (Issue #28, uncovered by Issue #10 in v1.1) - -* Ported ``dummyserver`` to use ``tornado`` instead of ``webob`` + - ``eventlet``. Removed extraneous unsupported dummyserver testing backends. - Added socket-level tests. - -* More tests. Achievement Unlocked: 99% Coverage. - - -1.1 (2012-01-07) ----------------- - -* Refactored ``dummyserver`` to its own root namespace module (used for - testing). - -* Added hostname verification for ``VerifiedHTTPSConnection`` by vendoring in - Py32's ``ssl_match_hostname``. (Issue #25) - -* Fixed cross-host HTTP redirects when using ``PoolManager``. (Issue #10) - -* Fixed ``decode_content`` being ignored when set through ``urlopen``. (Issue - #27) - -* Fixed timeout-related bugs. (Issues #17, #23) - - -1.0.2 (2011-11-04) ------------------- - -* Fixed typo in ``VerifiedHTTPSConnection`` which would only present as a bug if - you're using the object manually. (Thanks pyos) - -* Made RecentlyUsedContainer (and consequently PoolManager) more thread-safe by - wrapping the access log in a mutex. (Thanks @christer) - -* Made RecentlyUsedContainer more dict-like (corrected ``__delitem__`` and - ``__getitem__`` behaviour), with tests. Shouldn't affect core urllib3 code. - - -1.0.1 (2011-10-10) ------------------- - -* Fixed a bug where the same connection would get returned into the pool twice, - causing extraneous "HttpConnectionPool is full" log warnings. - - -1.0 (2011-10-08) ----------------- - -* Added ``PoolManager`` with LRU expiration of connections (tested and - documented). -* Added ``ProxyManager`` (needs tests, docs, and confirmation that it works - with HTTPS proxies). -* Added optional partial-read support for responses when - ``preload_content=False``. You can now make requests and just read the headers - without loading the content. -* Made response decoding optional (default on, same as before). -* Added optional explicit boundary string for ``encode_multipart_formdata``. -* Convenience request methods are now inherited from ``RequestMethods``. Old - helpers like ``get_url`` and ``post_url`` should be abandoned in favour of - the new ``request(method, url, ...)``. -* Refactored code to be even more decoupled, reusable, and extendable. -* License header added to ``.py`` files. -* Embiggened the documentation: Lots of Sphinx-friendly docstrings in the code - and docs in ``docs/`` and on https://urllib3.readthedocs.io/. -* Embettered all the things! -* Started writing this file. - - -0.4.1 (2011-07-17) ------------------- - -* Minor bug fixes, code cleanup. - - -0.4 (2011-03-01) ----------------- - -* Better unicode support. -* Added ``VerifiedHTTPSConnection``. -* Added ``NTLMConnectionPool`` in contrib. -* Minor improvements. - - -0.3.1 (2010-07-13) ------------------- - -* Added ``assert_host_name`` optional parameter. Now compatible with proxies. - - -0.3 (2009-12-10) ----------------- - -* Added HTTPS support. -* Minor bug fixes. -* Refactored, broken backwards compatibility with 0.2. -* API to be treated as stable from this version forward. - +## Sponsorship -0.2 (2008-11-17) ----------------- +If your company benefits from this library, please consider [sponsoring its +development](https://urllib3.readthedocs.io/en/latest/sponsors.html). -* Added unit tests. -* Bug fixes. +## For Enterprise -0.1 (2008-11-16) ----------------- +Professional support for urllib3 is available as part of the [Tidelift +Subscription][1]. Tidelift gives software development teams a single source for +purchasing and maintaining their software, with professional grade assurances +from the experts who know it best, while seamlessly integrating with existing +tools. -* First release. +[1]: https://tidelift.com/subscription/pkg/pypi-urllib3?utm_source=pypi-urllib3&utm_medium=referral&utm_campaign=readme diff --git a/README.md b/README.md new file mode 100644 index 0000000..c9e704f --- /dev/null +++ b/README.md @@ -0,0 +1,114 @@ +

+ +![urllib3](https://github.com/urllib3/urllib3/raw/main/docs/_static/banner_github.svg) + +

+ +

+ PyPI Version + Python Versions + Join our Discord + Coverage Status + Build Status on GitHub + Documentation Status
+ OpenSSF Scorecard + SLSA 3 + CII Best Practices +

+ +urllib3 is a powerful, *user-friendly* HTTP client for Python. Much of the +Python ecosystem already uses urllib3 and you should too. +urllib3 brings many critical features that are missing from the Python +standard libraries: + +- Thread safety. +- Connection pooling. +- Client-side SSL/TLS verification. +- File uploads with multipart encoding. +- Helpers for retrying requests and dealing with HTTP redirects. +- Support for gzip, deflate, brotli, and zstd encoding. +- Proxy support for HTTP and SOCKS. +- 100% test coverage. + +urllib3 is powerful and easy to use: + +```python3 +>>> import urllib3 +>>> resp = urllib3.request("GET", "http://httpbin.org/robots.txt") +>>> resp.status +200 +>>> resp.data +b"User-agent: *\nDisallow: /deny\n" +``` + +## Installing + +urllib3 can be installed with [pip](https://pip.pypa.io): + +```bash +$ python -m pip install urllib3 +``` + +Alternatively, you can grab the latest source code from [GitHub](https://github.com/urllib3/urllib3): + +```bash +$ git clone https://github.com/urllib3/urllib3.git +$ cd urllib3 +$ pip install . +``` + + +## Documentation + +urllib3 has usage and reference documentation at [urllib3.readthedocs.io](https://urllib3.readthedocs.io). + + +## Community + +urllib3 has a [community Discord channel](https://discord.gg/urllib3) for asking questions and +collaborating with other contributors. Drop by and say hello 👋 + + +## Contributing + +urllib3 happily accepts contributions. Please see our +[contributing documentation](https://urllib3.readthedocs.io/en/latest/contributing.html) +for some tips on getting started. + + +## Security Disclosures + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). +Tidelift will coordinate the fix and disclosure with maintainers. + + +## Maintainers + +- [@sethmlarson](https://github.com/sethmlarson) (Seth M. Larson) +- [@pquentin](https://github.com/pquentin) (Quentin Pradet) +- [@illia-v](https://github.com/illia-v) (Illia Volochii) +- [@theacodes](https://github.com/theacodes) (Thea Flowers) +- [@haikuginger](https://github.com/haikuginger) (Jess Shapiro) +- [@lukasa](https://github.com/lukasa) (Cory Benfield) +- [@sigmavirus24](https://github.com/sigmavirus24) (Ian Stapleton Cordasco) +- [@shazow](https://github.com/shazow) (Andrey Petrov) + +👋 + + +## Sponsorship + +If your company benefits from this library, please consider [sponsoring its +development](https://urllib3.readthedocs.io/en/latest/sponsors.html). + + +## For Enterprise + +Professional support for urllib3 is available as part of the [Tidelift +Subscription][1]. Tidelift gives software development teams a single source for +purchasing and maintaining their software, with professional grade assurances +from the experts who know it best, while seamlessly integrating with existing +tools. + +[1]: https://tidelift.com/subscription/pkg/pypi-urllib3?utm_source=pypi-urllib3&utm_medium=referral&utm_campaign=readme diff --git a/README.rst b/README.rst deleted file mode 100644 index bd287ea..0000000 --- a/README.rst +++ /dev/null @@ -1,120 +0,0 @@ -.. raw:: html - -

- - urllib3 - -

-

- PyPI Version - Python Versions - Join our Discord - Coverage Status - Build Status on GitHub - Build Status on Travis - Documentation Status -

- -urllib3 is a powerful, *user-friendly* HTTP client for Python. Much of the -Python ecosystem already uses urllib3 and you should too. -urllib3 brings many critical features that are missing from the Python -standard libraries: - -- Thread safety. -- Connection pooling. -- Client-side SSL/TLS verification. -- File uploads with multipart encoding. -- Helpers for retrying requests and dealing with HTTP redirects. -- Support for gzip, deflate, and brotli encoding. -- Proxy support for HTTP and SOCKS. -- 100% test coverage. - -urllib3 is powerful and easy to use: - -.. code-block:: python - - >>> import urllib3 - >>> http = urllib3.PoolManager() - >>> r = http.request('GET', 'http://httpbin.org/robots.txt') - >>> r.status - 200 - >>> r.data - 'User-agent: *\nDisallow: /deny\n' - - -Installing ----------- - -urllib3 can be installed with `pip `_:: - - $ python -m pip install urllib3 - -Alternatively, you can grab the latest source code from `GitHub `_:: - - $ git clone https://github.com/urllib3/urllib3.git - $ cd urllib3 - $ git checkout 1.26.x - $ pip install . - - -Documentation -------------- - -urllib3 has usage and reference documentation at `urllib3.readthedocs.io `_. - - -Contributing ------------- - -urllib3 happily accepts contributions. Please see our -`contributing documentation `_ -for some tips on getting started. - - -Security Disclosures --------------------- - -To report a security vulnerability, please use the -`Tidelift security contact `_. -Tidelift will coordinate the fix and disclosure with maintainers. - - -Maintainers ------------ - -- `@sethmlarson `__ (Seth M. Larson) -- `@pquentin `__ (Quentin Pradet) -- `@theacodes `__ (Thea Flowers) -- `@haikuginger `__ (Jess Shapiro) -- `@lukasa `__ (Cory Benfield) -- `@sigmavirus24 `__ (Ian Stapleton Cordasco) -- `@shazow `__ (Andrey Petrov) - -👋 - - -Sponsorship ------------ - -If your company benefits from this library, please consider `sponsoring its -development `_. - - -For Enterprise --------------- - -.. |tideliftlogo| image:: https://nedbatchelder.com/pix/Tidelift_Logos_RGB_Tidelift_Shorthand_On-White_small.png - :width: 75 - :alt: Tidelift - -.. list-table:: - :widths: 10 100 - - * - |tideliftlogo| - - Professional support for urllib3 is available as part of the `Tidelift - Subscription`_. Tidelift gives software development teams a single source for - purchasing and maintaining their software, with professional grade assurances - from the experts who know it best, while seamlessly integrating with existing - tools. - -.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-urllib3?utm_source=pypi-urllib3&utm_medium=referral&utm_campaign=readme diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 0000000..2c8afeb --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1 @@ +/files diff --git a/debian/changelog b/debian/changelog index c47c9eb..34c1e4a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,92 @@ +python-urllib3 (2.3.0-3) unstable; urgency=medium + + * Team upload. + * CVE-2025-50181: Fix a security issue where restricting the maximum + number of followed redirects at the `urllib3.PoolManager` level via the + `retries` parameter did not work (closes: #1108076). + * CVE-2025-50182: Make the Node.js runtime respect redirect parameters + such as `retries` and `redirects` (closes: #1108077). + + -- Colin Watson Sun, 13 Jul 2025 14:09:35 +0200 + +python-urllib3 (2.3.0-2) unstable; urgency=medium + + * Team upload. + * Ensure compatibility with httpx>=0.28 (closes: #1099277). + + -- Colin Watson Wed, 12 Mar 2025 11:20:41 +0000 + +python-urllib3 (2.3.0-1) unstable; urgency=medium + + * Team upload. + * New upstream release. + * SingleTLSLayerTestCase: Catch BrokenPipeError. + + -- Colin Watson Fri, 17 Jan 2025 13:54:06 +0000 + +python-urllib3 (2.2.3-4) unstable; urgency=medium + + * Team upload. + * Mark package "Multi-Arch: foreign" (Closes: #1078034) + + -- Alexandre Detiste Wed, 11 Dec 2024 13:16:51 +0100 + +python-urllib3 (2.2.3-3) unstable; urgency=medium + + * Team upload. + * Upload to unstable. + + -- Colin Watson Fri, 29 Nov 2024 16:38:06 +0000 + +python-urllib3 (2.2.3-2) experimental; urgency=medium + + * Team upload. + * Mark test_http2_probe_blocked_per_thread with requires_network. + + -- Colin Watson Tue, 12 Nov 2024 23:59:17 +0000 + +python-urllib3 (2.2.3-1) experimental; urgency=medium + + * Team upload. + * New upstream release: + - CVE-2024-37891: Added the Proxy-Authorization header to the list of + headers to strip from requests when redirecting to a different host + (closes: #1074149). + - Added support for Python 3.13 (closes: #1082278). + * Temporarily vendor hypercorn, since urllib3 needs a patched version for + its tests (commit d1719f8c1570cbd8e6a3719ffdb14a4d72880abb; see + https://github.com/urllib3/urllib3/issues/3334; closes: #1086794). + + -- Colin Watson Sun, 10 Nov 2024 23:57:18 +0000 + +python-urllib3 (2.0.7-2) unstable; urgency=medium + + * Team upload. + * Remove build & autopkgtest dependency on python3-mock + * Disable a test that compare today's date against some "expiry date". + * Patch: add a patch from upstream to handle new Exception strings. + + -- Alexandre Detiste Wed, 12 Jun 2024 20:04:48 +0200 + +python-urllib3 (2.0.7-1) experimental; urgency=medium + + [ Stefano Rivera ] + * New upstream release. + * Drop six patch and dependency on python3-six, superseded upstream. + (Closes: #1025218, LP: #1897633) + * Drop bundled backports.makefile from copyright, removed upstream. + * Build with pybuild pyproject plugin. + * Mark Build-Depends with nocheck. + * Re-enable the full test suite. + * Patch: Handle ConnectionRefusedError in test.requires_network() + * Patch: Mark tests that require network + * Patch: Don't make requests to evil.com in tests. + * Export CI=1 in tests, to increase timeouts. + * Allow stderr in the autopkgtest. + * Patch: Skip expensive integration tests, that often cause timeouts. + + -- Daniele Tricoli Sun, 12 Nov 2023 23:57:09 +0100 + python-urllib3 (1.26.18-2) unstable; urgency=medium * Team upload. diff --git a/debian/control b/debian/control index a54b217..73ecd72 100644 --- a/debian/control +++ b/debian/control @@ -5,16 +5,25 @@ Section: python Priority: optional Build-Depends: debhelper-compat (= 13), - dh-python, + dh-sequence-python3, + pybuild-plugin-pyproject, python3-all, - python3-brotli, - python3-coverage, - python3-idna, - python3-mock, - python3-pytest, - python3-setuptools, - python3-six, - python3-tornado + python3-brotli , + python3-coverage , + python3-cryptography , + python3-hatch-vcs, + python3-hatchling, + python3-h2, + python3-httpx , + python3-idna , + python3-openssl , + python3-pytest , + python3-pytest-timeout , + python3-quart , + python3-quart-trio , + python3-socks , + python3-trio , + python3-trustme Standards-Version: 4.6.2 Rules-Requires-Root: no Homepage: https://urllib3.readthedocs.org @@ -23,10 +32,10 @@ Vcs-Browser: https://salsa.debian.org/python-team/packages/python-urllib3 Package: python3-urllib3 Architecture: all +Multi-Arch: foreign Depends: ${misc:Depends}, ${python3:Depends}, - python3-six Recommends: ca-certificates Suggests: @@ -47,5 +56,3 @@ Description: HTTP library with thread-safe connection pooling for Python3 - Thread-safe and sanity-safe. - Small and easy to understand codebase perfect for extending and building upon. - . - This package contains the Python 3 version of the library. diff --git a/debian/copyright b/debian/copyright index 8ae3b2a..5aa5453 100644 --- a/debian/copyright +++ b/debian/copyright @@ -7,22 +7,14 @@ Files: * Copyright: 2008-2020, Andrey Petrov and contributors License: Expat -Files: src/urllib3/contrib/_securetransport/* -Copyright: 2015-2016, Will Bond -License: Expat - -Files: src/urllib3/packages/backports/makefile.py -Copyright: 2007, Python Software Foundation -License: PSF-2 - -Files: src/urllib3/packages/six.py -Copyright: 2010-2015, Benjamin Peterson -License: Expat - Files: debian/* Copyright: 2012-2023, Daniele Tricoli License: Expat +Files: debian/vendor/hypercorn/* +Copyright: 2018-2022 Philip G Jones +License: Expat + License: Expat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -42,49 +34,3 @@ License: Expat CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -License: PSF-2 - 1. This LICENSE AGREEMENT is between the Python Software Foundation - ("PSF"), and the Individual or Organization ("Licensee") accessing - and otherwise using this software ("Python") in source or binary form - and its associated documentation. - . - 2. Subject to the terms and conditions of this License Agreement, PSF - hereby grants Licensee a nonexclusive, royalty-free, world-wide - license to reproduce, analyze, test, perform and/or display publicly, - prepare derivative works, distribute, and otherwise use Python alone - or in any derivative version, provided, however, that PSF's License - Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, - 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights - Reserved" are retained in Python alone or in any derivative version - prepared by Licensee. - . - 3. In the event Licensee prepares a derivative work that is based on - or incorporates Python or any part thereof, and wants to make the - derivative work available to others as provided herein, then Licensee - hereby agrees to include in any such work a brief summary of the - changes made to Python. - . - 4. PSF is making Python available to Licensee on an "AS IS" - basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR - IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND - DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR - FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT - INFRINGE ANY THIRD PARTY RIGHTS. - . - 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON - FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A - RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY - DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - . - 6. This License Agreement will automatically terminate upon a - material breach of its terms and conditions. - . - 7. Nothing in this License Agreement shall be deemed to create any - relationship of agency, partnership, or joint venture between PSF and - Licensee. This License Agreement does not grant permission to use PSF - trademarks or trade name in a trademark sense to endorse or promote - products or services of Licensee, or any third party. - . - 8. By copying, installing or otherwise using Python, Licensee agrees - to be bound by the terms and conditions of this License Agreement. diff --git a/debian/patches/0002-implement-missing-importlib-methods.patch b/debian/patches/0002-implement-missing-importlib-methods.patch deleted file mode 100644 index 27e805f..0000000 --- a/debian/patches/0002-implement-missing-importlib-methods.patch +++ /dev/null @@ -1,47 +0,0 @@ -From: Thomas Grainger -Date: Wed, 9 Nov 2022 15:09:26 +0000 -Subject: implement missing importlib methods - -Adds support for Python 3.12. - -Bug-Debian: https://bugs.debian.org/1056511 -Origin: https://github.com/urllib3/urllib3/pull/2790/commits/2984ea1c4681e899002372ee7b961efb72f4583c ---- - test/test_no_ssl.py | 17 +++++++++++++++++ - 1 file changed, 17 insertions(+) - -diff --git a/test/test_no_ssl.py b/test/test_no_ssl.py -index 7cf6260..b790942 100644 ---- a/test/test_no_ssl.py -+++ b/test/test_no_ssl.py -@@ -6,6 +6,7 @@ Test what happens if Python was built without SSL - """ - - import sys -+import importlib.util - - import pytest - -@@ -29,6 +30,22 @@ class ImportBlocker(object): - def load_module(self, fullname): - raise ImportError("import of {0} is blocked".format(fullname)) - -+ def exec_module(self, module): -+ raise ImportError(f"import of {0} is blocked".format(module)) -+ -+ def find_spec( -+ self, -+ fullname: str, -+ path, -+ target = None, -+ ): -+ -+ loader = self.find_module(fullname, path) -+ if loader is None: -+ return None -+ -+ return importlib.util.spec_from_loader(fullname, loader) -+ - - class ModuleStash(object): - """ diff --git a/debian/patches/01_do-not-use-embedded-python-six.patch b/debian/patches/01_do-not-use-embedded-python-six.patch deleted file mode 100644 index 42f6297..0000000 --- a/debian/patches/01_do-not-use-embedded-python-six.patch +++ /dev/null @@ -1,602 +0,0 @@ -From: Daniele Tricoli -Date: Thu, 8 Oct 2015 13:19:46 -0700 -Subject: Do not use embedded copy of python-six. - -Forwarded: not-needed - -Patch-Name: 01_do-not-use-embedded-python-six.patch ---- - dummyserver/handlers.py | 6 +++--- - src/urllib3/_collections.py | 4 ++-- - src/urllib3/connection.py | 6 +++--- - src/urllib3/connectionpool.py | 5 +++-- - src/urllib3/contrib/_securetransport/bindings.py | 2 +- - src/urllib3/contrib/appengine.py | 2 +- - src/urllib3/contrib/ntlmpool.py | 2 +- - src/urllib3/contrib/pyopenssl.py | 2 +- - src/urllib3/contrib/securetransport.py | 3 ++- - src/urllib3/exceptions.py | 2 +- - src/urllib3/fields.py | 2 +- - src/urllib3/filepost.py | 4 ++-- - src/urllib3/poolmanager.py | 4 ++-- - src/urllib3/request.py | 4 ++-- - src/urllib3/response.py | 2 +- - src/urllib3/util/connection.py | 5 +++-- - src/urllib3/util/queue.py | 4 ++-- - src/urllib3/util/request.py | 2 +- - src/urllib3/util/response.py | 2 +- - src/urllib3/util/retry.py | 2 +- - src/urllib3/util/ssl_.py | 2 +- - src/urllib3/util/ssltransport.py | 3 ++- - src/urllib3/util/url.py | 2 +- - test/__init__.py | 2 +- - test/test_collections.py | 2 +- - test/test_compatibility.py | 2 +- - test/test_connectionpool.py | 7 ++++--- - test/test_fields.py | 2 +- - test/test_filepost.py | 2 +- - test/test_queue_monkeypatch.py | 2 +- - test/test_request.py | 2 +- - test/test_response.py | 2 +- - test/test_retry.py | 4 ++-- - test/test_retry_deprecated.py | 4 ++-- - test/test_util.py | 2 +- - test/with_dummyserver/test_connectionpool.py | 4 ++-- - test/with_dummyserver/test_https.py | 2 +- - test/with_dummyserver/test_socketlevel.py | 2 +- - 38 files changed, 59 insertions(+), 54 deletions(-) - -diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py -index acd181d..7d3e413 100644 ---- a/dummyserver/handlers.py -+++ b/dummyserver/handlers.py -@@ -14,9 +14,9 @@ from io import BytesIO - from tornado import httputil - from tornado.web import RequestHandler - --from urllib3.packages.six import binary_type, ensure_str --from urllib3.packages.six.moves.http_client import responses --from urllib3.packages.six.moves.urllib.parse import urlsplit -+from six import binary_type, ensure_str -+from six.moves.http_client import responses -+from six.moves.urllib.parse import urlsplit - - log = logging.getLogger(__name__) - -diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py -index bceb845..6bc6e2c 100644 ---- a/src/urllib3/_collections.py -+++ b/src/urllib3/_collections.py -@@ -19,8 +19,8 @@ except ImportError: # Platform-specific: No threads available - from collections import OrderedDict - - from .exceptions import InvalidHeader --from .packages import six --from .packages.six import iterkeys, itervalues -+import six -+from six import iterkeys, itervalues - - __all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"] - -diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py -index 54b96b1..d9d411a 100644 ---- a/src/urllib3/connection.py -+++ b/src/urllib3/connection.py -@@ -9,9 +9,9 @@ import warnings - from socket import error as SocketError - from socket import timeout as SocketTimeout - --from .packages import six --from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection --from .packages.six.moves.http_client import HTTPException # noqa: F401 -+import six -+from six.moves.http_client import HTTPConnection as _HTTPConnection -+from six.moves.http_client import HTTPException # noqa: F401 - from .util.proxy import create_proxy_ssl_context - - try: # Compiled with SSL? -diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py -index 5a6adcb..476f33e 100644 ---- a/src/urllib3/connectionpool.py -+++ b/src/urllib3/connectionpool.py -@@ -9,6 +9,9 @@ import warnings - from socket import error as SocketError - from socket import timeout as SocketTimeout - -+import six -+from six.moves import queue -+ - from ._collections import HTTPHeaderDict - from .connection import ( - BaseSSLError, -@@ -35,8 +38,6 @@ from .exceptions import ( - SSLError, - TimeoutError, - ) --from .packages import six --from .packages.six.moves import queue - from .request import RequestMethods - from .response import HTTPResponse - from .util.connection import is_connection_dropped -diff --git a/src/urllib3/contrib/_securetransport/bindings.py b/src/urllib3/contrib/_securetransport/bindings.py -index 264d564..3619b16 100644 ---- a/src/urllib3/contrib/_securetransport/bindings.py -+++ b/src/urllib3/contrib/_securetransport/bindings.py -@@ -48,7 +48,7 @@ from ctypes import ( - ) - from ctypes.util import find_library - --from ...packages.six import raise_from -+from six import raise_from - - if platform.system() != "Darwin": - raise ImportError("Only macOS is supported") -diff --git a/src/urllib3/contrib/appengine.py b/src/urllib3/contrib/appengine.py -index a5a6d91..2147407 100644 ---- a/src/urllib3/contrib/appengine.py -+++ b/src/urllib3/contrib/appengine.py -@@ -52,7 +52,7 @@ from ..exceptions import ( - SSLError, - TimeoutError, - ) --from ..packages.six.moves.urllib.parse import urljoin -+from six.moves.urllib.parse import urljoin - from ..request import RequestMethods - from ..response import HTTPResponse - from ..util.retry import Retry -diff --git a/src/urllib3/contrib/ntlmpool.py b/src/urllib3/contrib/ntlmpool.py -index 4716657..f6f0153 100644 ---- a/src/urllib3/contrib/ntlmpool.py -+++ b/src/urllib3/contrib/ntlmpool.py -@@ -11,7 +11,7 @@ from logging import getLogger - from ntlm import ntlm - - from .. import HTTPSConnectionPool --from ..packages.six.moves.http_client import HTTPSConnection -+from six.moves.http_client import HTTPSConnection - - warnings.warn( - "The 'urllib3.contrib.ntlmpool' module is deprecated and will be removed " -diff --git a/src/urllib3/contrib/pyopenssl.py b/src/urllib3/contrib/pyopenssl.py -index 1ed214b..e3b79f9 100644 ---- a/src/urllib3/contrib/pyopenssl.py -+++ b/src/urllib3/contrib/pyopenssl.py -@@ -76,7 +76,7 @@ import sys - import warnings - - from .. import util --from ..packages import six -+import six - from ..util.ssl_ import PROTOCOL_TLS_CLIENT - - warnings.warn( -diff --git a/src/urllib3/contrib/securetransport.py b/src/urllib3/contrib/securetransport.py -index e311c0c..6c46a3b 100644 ---- a/src/urllib3/contrib/securetransport.py -+++ b/src/urllib3/contrib/securetransport.py -@@ -64,8 +64,9 @@ import struct - import threading - import weakref - -+import six -+ - from .. import util --from ..packages import six - from ..util.ssl_ import PROTOCOL_TLS_CLIENT - from ._securetransport.bindings import CoreFoundation, Security, SecurityConst - from ._securetransport.low_level import ( -diff --git a/src/urllib3/exceptions.py b/src/urllib3/exceptions.py -index cba6f3f..053758e 100644 ---- a/src/urllib3/exceptions.py -+++ b/src/urllib3/exceptions.py -@@ -1,6 +1,6 @@ - from __future__ import absolute_import - --from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead -+from six.moves.http_client import IncompleteRead as httplib_IncompleteRead - - # Base Exceptions - -diff --git a/src/urllib3/fields.py b/src/urllib3/fields.py -index 9d630f4..65aad4b 100644 ---- a/src/urllib3/fields.py -+++ b/src/urllib3/fields.py -@@ -4,7 +4,7 @@ import email.utils - import mimetypes - import re - --from .packages import six -+import six - - - def guess_content_type(filename, default="application/octet-stream"): -diff --git a/src/urllib3/filepost.py b/src/urllib3/filepost.py -index 36c9252..cc2ecdb 100644 ---- a/src/urllib3/filepost.py -+++ b/src/urllib3/filepost.py -@@ -6,8 +6,8 @@ import os - from io import BytesIO - - from .fields import RequestField --from .packages import six --from .packages.six import b -+import six -+from six import b - - writer = codecs.lookup("utf-8")[3] - -diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py -index fb51bf7..127accf 100644 ---- a/src/urllib3/poolmanager.py -+++ b/src/urllib3/poolmanager.py -@@ -13,8 +13,8 @@ from .exceptions import ( - ProxySchemeUnsupported, - URLSchemeUnknown, - ) --from .packages import six --from .packages.six.moves.urllib.parse import urljoin -+import six -+from six.moves.urllib.parse import urljoin - from .request import RequestMethods - from .util.proxy import connection_requires_http_tunnel - from .util.retry import Retry -diff --git a/src/urllib3/request.py b/src/urllib3/request.py -index 3b4cf99..ac90a94 100644 ---- a/src/urllib3/request.py -+++ b/src/urllib3/request.py -@@ -3,8 +3,8 @@ from __future__ import absolute_import - import sys - - from .filepost import encode_multipart_formdata --from .packages import six --from .packages.six.moves.urllib.parse import urlencode -+import six -+from six.moves.urllib.parse import urlencode - - __all__ = ["RequestMethods"] - -diff --git a/src/urllib3/response.py b/src/urllib3/response.py -index 0bd13d4..30a6bb2 100644 ---- a/src/urllib3/response.py -+++ b/src/urllib3/response.py -@@ -32,7 +32,7 @@ from .exceptions import ( - ResponseNotChunked, - SSLError, - ) --from .packages import six -+import six - from .util.response import is_fp_closed, is_response_to_head - - log = logging.getLogger(__name__) -diff --git a/src/urllib3/util/connection.py b/src/urllib3/util/connection.py -index 6af1138..953266f 100644 ---- a/src/urllib3/util/connection.py -+++ b/src/urllib3/util/connection.py -@@ -2,9 +2,10 @@ from __future__ import absolute_import - - import socket - -+import six -+ - from ..contrib import _appengine_environ - from ..exceptions import LocationParseError --from ..packages import six - from .wait import NoWayToWaitForSocketError, wait_for_read - - -@@ -66,7 +67,7 @@ def create_connection( - host.encode("idna") - except UnicodeError: - return six.raise_from( -- LocationParseError(u"'%s', label empty or too long" % host), None -+ LocationParseError("'%s', label empty or too long" % host), None - ) - - for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): -diff --git a/src/urllib3/util/queue.py b/src/urllib3/util/queue.py -index 4178410..7f214cb 100644 ---- a/src/urllib3/util/queue.py -+++ b/src/urllib3/util/queue.py -@@ -1,7 +1,7 @@ - import collections - --from ..packages import six --from ..packages.six.moves import queue -+import six -+from six.moves import queue - - if six.PY2: - # Queue is imported for side effects on MS Windows. See issue #229. -diff --git a/src/urllib3/util/request.py b/src/urllib3/util/request.py -index b574b08..7427875 100644 ---- a/src/urllib3/util/request.py -+++ b/src/urllib3/util/request.py -@@ -3,7 +3,7 @@ from __future__ import absolute_import - from base64 import b64encode - - from ..exceptions import UnrewindableBodyError --from ..packages.six import b, integer_types -+from six import b, integer_types - - # Pass as a value within ``headers`` to skip - # emitting some HTTP headers that are added automatically. -diff --git a/src/urllib3/util/response.py b/src/urllib3/util/response.py -index 5ea609c..bd6ecd7 100644 ---- a/src/urllib3/util/response.py -+++ b/src/urllib3/util/response.py -@@ -3,7 +3,7 @@ from __future__ import absolute_import - from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect - - from ..exceptions import HeaderParsingError --from ..packages.six.moves import http_client as httplib -+from six.moves import http_client as httplib - - - def is_fp_closed(obj): -diff --git a/src/urllib3/util/retry.py b/src/urllib3/util/retry.py -index 60ef6c4..8b16f72 100644 ---- a/src/urllib3/util/retry.py -+++ b/src/urllib3/util/retry.py -@@ -17,7 +17,7 @@ from ..exceptions import ( - ReadTimeoutError, - ResponseError, - ) --from ..packages import six -+import six - - log = logging.getLogger(__name__) - -diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py -index 8f86781..11183c4 100644 ---- a/src/urllib3/util/ssl_.py -+++ b/src/urllib3/util/ssl_.py -@@ -13,7 +13,7 @@ from ..exceptions import ( - SNIMissingWarning, - SSLError, - ) --from ..packages import six -+import six - from .url import BRACELESS_IPV6_ADDRZ_RE, IPV4_RE - - SSLContext = None -diff --git a/src/urllib3/util/ssltransport.py b/src/urllib3/util/ssltransport.py -index 4a7105d..bdbc8eb 100644 ---- a/src/urllib3/util/ssltransport.py -+++ b/src/urllib3/util/ssltransport.py -@@ -2,8 +2,9 @@ import io - import socket - import ssl - -+import six -+ - from ..exceptions import ProxySchemeUnsupported --from ..packages import six - - SSL_BLOCKSIZE = 16384 - -diff --git a/src/urllib3/util/url.py b/src/urllib3/util/url.py -index e5682d3..8e280dd 100644 ---- a/src/urllib3/util/url.py -+++ b/src/urllib3/util/url.py -@@ -4,7 +4,7 @@ import re - from collections import namedtuple - - from ..exceptions import LocationParseError --from ..packages import six -+import six - - url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"] - -diff --git a/test/__init__.py b/test/__init__.py -index 2307b2d..d5cc98b 100644 ---- a/test/__init__.py -+++ b/test/__init__.py -@@ -18,7 +18,7 @@ except ImportError: - - from urllib3 import util - from urllib3.exceptions import HTTPWarning --from urllib3.packages import six -+import six - from urllib3.util import ssl_ - - try: -diff --git a/test/test_collections.py b/test/test_collections.py -index 4b8624c..056fddd 100644 ---- a/test/test_collections.py -+++ b/test/test_collections.py -@@ -3,7 +3,7 @@ import pytest - from urllib3._collections import HTTPHeaderDict - from urllib3._collections import RecentlyUsedContainer as Container - from urllib3.exceptions import InvalidHeader --from urllib3.packages import six -+import six - - xrange = six.moves.xrange - -diff --git a/test/test_compatibility.py b/test/test_compatibility.py -index 58a9ab5..0c86d4d 100644 ---- a/test/test_compatibility.py -+++ b/test/test_compatibility.py -@@ -3,7 +3,7 @@ import warnings - import pytest - - from urllib3.connection import HTTPConnection --from urllib3.packages.six.moves import http_cookiejar, urllib -+from six.moves import http_cookiejar, urllib - from urllib3.response import HTTPResponse - - -diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py -index 872d01c..3a9fb9d 100644 ---- a/test/test_connectionpool.py -+++ b/test/test_connectionpool.py -@@ -7,6 +7,9 @@ from test import SHORT_TIMEOUT - - import pytest - from mock import Mock -+from six.moves import http_client as httplib -+from six.moves.http_client import HTTPException -+from six.moves.queue import Empty - - from dummyserver.server import DEFAULT_CA - from urllib3._collections import HTTPHeaderDict -@@ -26,9 +29,7 @@ from urllib3.exceptions import ( - SSLError, - TimeoutError, - ) --from urllib3.packages.six.moves import http_client as httplib --from urllib3.packages.six.moves.http_client import HTTPException --from urllib3.packages.six.moves.queue import Empty -+from urllib3.packages.ssl_match_hostname import CertificateError - from urllib3.response import HTTPResponse - from urllib3.util.ssl_match_hostname import CertificateError - from urllib3.util.timeout import Timeout -diff --git a/test/test_fields.py b/test/test_fields.py -index 98ce17c..cde4b96 100644 ---- a/test/test_fields.py -+++ b/test/test_fields.py -@@ -1,7 +1,7 @@ - import pytest - - from urllib3.fields import RequestField, format_header_param_rfc2231, guess_content_type --from urllib3.packages.six import u -+from six import u - - - class TestRequestField(object): -diff --git a/test/test_filepost.py b/test/test_filepost.py -index 5b0cfe1..da6ecdc 100644 ---- a/test/test_filepost.py -+++ b/test/test_filepost.py -@@ -2,7 +2,7 @@ import pytest - - from urllib3.fields import RequestField - from urllib3.filepost import encode_multipart_formdata, iter_fields --from urllib3.packages.six import b, u -+from six import b, u - - BOUNDARY = "!! test boundary !!" - -diff --git a/test/test_queue_monkeypatch.py b/test/test_queue_monkeypatch.py -index f8420a0..475d3be 100644 ---- a/test/test_queue_monkeypatch.py -+++ b/test/test_queue_monkeypatch.py -@@ -5,7 +5,7 @@ import pytest - - from urllib3 import HTTPConnectionPool - from urllib3.exceptions import EmptyPoolError --from urllib3.packages.six.moves import queue -+from six.moves import queue - - - class BadError(Exception): -diff --git a/test/test_request.py b/test/test_request.py -index 1db819d..236f831 100644 ---- a/test/test_request.py -+++ b/test/test_request.py -@@ -3,7 +3,7 @@ import types - import pytest - - import urllib3 --from urllib3.packages import six -+import six - - - @pytest.mark.skipif( -diff --git a/test/test_response.py b/test/test_response.py -index e09e385..ec4a2bf 100644 ---- a/test/test_response.py -+++ b/test/test_response.py -@@ -24,7 +24,7 @@ from urllib3.exceptions import ( - SSLError, - httplib_IncompleteRead, - ) --from urllib3.packages.six.moves import http_client as httplib -+from six.moves import http_client as httplib - from urllib3.response import HTTPResponse, brotli - from urllib3.util.response import is_fp_closed - from urllib3.util.retry import RequestHistory, Retry -diff --git a/test/test_retry.py b/test/test_retry.py -index 95a33e7..7297aa5 100644 ---- a/test/test_retry.py -+++ b/test/test_retry.py -@@ -11,8 +11,8 @@ from urllib3.exceptions import ( - ResponseError, - SSLError, - ) --from urllib3.packages import six --from urllib3.packages.six.moves import xrange -+import six -+from six.moves import xrange - from urllib3.response import HTTPResponse - from urllib3.util.retry import RequestHistory, Retry - -diff --git a/test/test_retry_deprecated.py b/test/test_retry_deprecated.py -index 5133a51..30acfce 100644 ---- a/test/test_retry_deprecated.py -+++ b/test/test_retry_deprecated.py -@@ -12,8 +12,8 @@ from urllib3.exceptions import ( - ResponseError, - SSLError, - ) --from urllib3.packages import six --from urllib3.packages.six.moves import xrange -+import six -+from six.moves import xrange - from urllib3.response import HTTPResponse - from urllib3.util.retry import RequestHistory, Retry - -diff --git a/test/test_util.py b/test/test_util.py -index 2f16dbf..2e360b7 100644 ---- a/test/test_util.py -+++ b/test/test_util.py -@@ -19,7 +19,7 @@ from urllib3.exceptions import ( - TimeoutStateError, - UnrewindableBodyError, - ) --from urllib3.packages import six -+import six - from urllib3.poolmanager import ProxyConfig - from urllib3.util import is_fp_closed - from urllib3.util.connection import _has_ipv6, allowed_gai_family, create_connection -diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py -index cde027b..40bf9a9 100644 ---- a/test/with_dummyserver/test_connectionpool.py -+++ b/test/with_dummyserver/test_connectionpool.py -@@ -30,8 +30,8 @@ from urllib3.exceptions import ( - ReadTimeoutError, - UnrewindableBodyError, - ) --from urllib3.packages.six import b, u --from urllib3.packages.six.moves.urllib.parse import urlencode -+from six import b, u -+from six.moves.urllib.parse import urlencode - from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS - from urllib3.util.retry import RequestHistory, Retry - from urllib3.util.timeout import Timeout -diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py -index f37f8e6..5ddc879 100644 ---- a/test/with_dummyserver/test_https.py -+++ b/test/with_dummyserver/test_https.py -@@ -41,7 +41,7 @@ from urllib3.exceptions import ( - SSLError, - SystemTimeWarning, - ) --from urllib3.packages import six -+import six - from urllib3.util.timeout import Timeout - - from .. import has_alpn -diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py -index 9ee3dff..49b1430 100644 ---- a/test/with_dummyserver/test_socketlevel.py -+++ b/test/with_dummyserver/test_socketlevel.py -@@ -17,7 +17,7 @@ from urllib3.exceptions import ( - ReadTimeoutError, - SSLError, - ) --from urllib3.packages.six.moves import http_client as httplib -+from six.moves import http_client as httplib - from urllib3.poolmanager import proxy_from_url - from urllib3.util import ssl_, ssl_wrap_socket - from urllib3.util.retry import Retry diff --git a/debian/patches/CVE-2025-50181.patch b/debian/patches/CVE-2025-50181.patch new file mode 100644 index 0000000..3ab34c3 --- /dev/null +++ b/debian/patches/CVE-2025-50181.patch @@ -0,0 +1,279 @@ +From: Illia Volochii +Date: Wed, 18 Jun 2025 16:25:01 +0300 +Subject: Merge commit from fork + +* Apply Quentin's suggestion + +Co-authored-by: Quentin Pradet + +* Add tests for disabled redirects in the pool manager + +* Add a possible fix for the issue with not raised `MaxRetryError` + +* Make urllib3 handle redirects instead of JS when JSPI is used + +* Fix info in the new comment + +* State that redirects with XHR are not controlled by urllib3 + +* Remove excessive params from new test requests + +* Add tests reaching max non-0 redirects + +* Test redirects with Emscripten + +* Fix `test_merge_pool_kwargs` + +* Add a changelog entry + +* Parametrize tests + +* Drop a fix for Emscripten + +* Apply Seth's suggestion to docs + +Co-authored-by: Seth Michael Larson + +* Use a minor release instead of the patch one + +--------- + +Co-authored-by: Quentin Pradet +Co-authored-by: Seth Michael Larson + +Origin: upstream, https://github.com/urllib3/urllib3/commit/f05b1329126d5be6de501f9d1e3e36738bc08857 +Bug-Debian: https://bugs.debian.org/1108076 +Last-Update: 2025-07-13 +--- + docs/reference/contrib/emscripten.rst | 2 +- + dummyserver/app.py | 1 + + src/urllib3/poolmanager.py | 18 ++++- + test/contrib/emscripten/test_emscripten.py | 16 +++++ + test/test_poolmanager.py | 5 +- + test/with_dummyserver/test_poolmanager.py | 101 +++++++++++++++++++++++++++++ + 6 files changed, 139 insertions(+), 4 deletions(-) + +diff --git a/docs/reference/contrib/emscripten.rst b/docs/reference/contrib/emscripten.rst +index 99fb20f..a8f1cda 100644 +--- a/docs/reference/contrib/emscripten.rst ++++ b/docs/reference/contrib/emscripten.rst +@@ -65,7 +65,7 @@ Features which are usable with Emscripten support are: + * Timeouts + * Retries + * Streaming (with Web Workers and Cross-Origin Isolation) +-* Redirects ++* Redirects (determined by browser/runtime, not restrictable with urllib3) + * Decompressing response bodies + + Features which don't work with Emscripten: +diff --git a/dummyserver/app.py b/dummyserver/app.py +index 97b1b23..0eeb93f 100644 +--- a/dummyserver/app.py ++++ b/dummyserver/app.py +@@ -227,6 +227,7 @@ async def encodingrequest() -> ResponseReturnValue: + + + @hypercorn_app.route("/redirect", methods=["GET", "POST", "PUT"]) ++@pyodide_testing_app.route("/redirect", methods=["GET", "POST", "PUT"]) + async def redirect() -> ResponseReturnValue: + "Perform a redirect to ``target``" + values = await request.values +diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py +index 085d1db..5763fea 100644 +--- a/src/urllib3/poolmanager.py ++++ b/src/urllib3/poolmanager.py +@@ -203,6 +203,22 @@ class PoolManager(RequestMethods): + **connection_pool_kw: typing.Any, + ) -> None: + super().__init__(headers) ++ if "retries" in connection_pool_kw: ++ retries = connection_pool_kw["retries"] ++ if not isinstance(retries, Retry): ++ # When Retry is initialized, raise_on_redirect is based ++ # on a redirect boolean value. ++ # But requests made via a pool manager always set ++ # redirect to False, and raise_on_redirect always ends ++ # up being False consequently. ++ # Here we fix the issue by setting raise_on_redirect to ++ # a value needed by the pool manager without considering ++ # the redirect boolean. ++ raise_on_redirect = retries is not False ++ retries = Retry.from_int(retries, redirect=False) ++ retries.raise_on_redirect = raise_on_redirect ++ connection_pool_kw = connection_pool_kw.copy() ++ connection_pool_kw["retries"] = retries + self.connection_pool_kw = connection_pool_kw + + self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool] +@@ -456,7 +472,7 @@ class PoolManager(RequestMethods): + kw["body"] = None + kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change() + +- retries = kw.get("retries") ++ retries = kw.get("retries", response.retries) + if not isinstance(retries, Retry): + retries = Retry.from_int(retries, redirect=redirect) + +diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py +index 9317a09..5eaa674 100644 +--- a/test/contrib/emscripten/test_emscripten.py ++++ b/test/contrib/emscripten/test_emscripten.py +@@ -944,6 +944,22 @@ def test_retries( + pyodide_test(selenium_coverage, testserver_http.http_host, find_unused_port()) + + ++def test_redirects( ++ selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ++) -> None: ++ @run_in_pyodide # type: ignore[misc] ++ def pyodide_test(selenium_coverage: typing.Any, host: str, port: int) -> None: ++ from urllib3 import request ++ ++ redirect_url = f"http://{host}:{port}/redirect" ++ response = request("GET", redirect_url) ++ assert response.status == 200 ++ ++ pyodide_test( ++ selenium_coverage, testserver_http.http_host, testserver_http.http_port ++ ) ++ ++ + def test_insecure_requests_warning( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo + ) -> None: +diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py +index ab5f203..b481a19 100644 +--- a/test/test_poolmanager.py ++++ b/test/test_poolmanager.py +@@ -379,9 +379,10 @@ class TestPoolManager: + + def test_merge_pool_kwargs(self) -> None: + """Assert _merge_pool_kwargs works in the happy case""" +- p = PoolManager(retries=100) ++ retries = retry.Retry(total=100) ++ p = PoolManager(retries=retries) + merged = p._merge_pool_kwargs({"new_key": "value"}) +- assert {"retries": 100, "new_key": "value"} == merged ++ assert {"retries": retries, "new_key": "value"} == merged + + def test_merge_pool_kwargs_none(self) -> None: + """Assert false-y values to _merge_pool_kwargs result in defaults""" +diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py +index af77241..7f163ab 100644 +--- a/test/with_dummyserver/test_poolmanager.py ++++ b/test/with_dummyserver/test_poolmanager.py +@@ -84,6 +84,89 @@ class TestPoolManager(HypercornDummyServerTestCase): + assert r.status == 200 + assert r.data == b"Dummy server!" + ++ @pytest.mark.parametrize( ++ "retries", ++ (0, Retry(total=0), Retry(redirect=0), Retry(total=0, redirect=0)), ++ ) ++ def test_redirects_disabled_for_pool_manager_with_0( ++ self, retries: typing.Literal[0] | Retry ++ ) -> None: ++ """ ++ Check handling redirects when retries is set to 0 on the pool ++ manager. ++ """ ++ with PoolManager(retries=retries) as http: ++ with pytest.raises(MaxRetryError): ++ http.request("GET", f"{self.base_url}/redirect") ++ ++ # Setting redirect=True should not change the behavior. ++ with pytest.raises(MaxRetryError): ++ http.request("GET", f"{self.base_url}/redirect", redirect=True) ++ ++ # Setting redirect=False should not make it follow the redirect, ++ # but MaxRetryError should not be raised. ++ response = http.request("GET", f"{self.base_url}/redirect", redirect=False) ++ assert response.status == 303 ++ ++ @pytest.mark.parametrize( ++ "retries", ++ ( ++ False, ++ Retry(total=False), ++ Retry(redirect=False), ++ Retry(total=False, redirect=False), ++ ), ++ ) ++ def test_redirects_disabled_for_pool_manager_with_false( ++ self, retries: typing.Literal[False] | Retry ++ ) -> None: ++ """ ++ Check that setting retries set to False on the pool manager disables ++ raising MaxRetryError and redirect=True does not change the ++ behavior. ++ """ ++ with PoolManager(retries=retries) as http: ++ response = http.request("GET", f"{self.base_url}/redirect") ++ assert response.status == 303 ++ ++ response = http.request("GET", f"{self.base_url}/redirect", redirect=True) ++ assert response.status == 303 ++ ++ response = http.request("GET", f"{self.base_url}/redirect", redirect=False) ++ assert response.status == 303 ++ ++ def test_redirects_disabled_for_individual_request(self) -> None: ++ """ ++ Check handling redirects when they are meant to be disabled ++ on the request level. ++ """ ++ with PoolManager() as http: ++ # Check when redirect is not passed. ++ with pytest.raises(MaxRetryError): ++ http.request("GET", f"{self.base_url}/redirect", retries=0) ++ response = http.request("GET", f"{self.base_url}/redirect", retries=False) ++ assert response.status == 303 ++ ++ # Check when redirect=True. ++ with pytest.raises(MaxRetryError): ++ http.request( ++ "GET", f"{self.base_url}/redirect", retries=0, redirect=True ++ ) ++ response = http.request( ++ "GET", f"{self.base_url}/redirect", retries=False, redirect=True ++ ) ++ assert response.status == 303 ++ ++ # Check when redirect=False. ++ response = http.request( ++ "GET", f"{self.base_url}/redirect", retries=0, redirect=False ++ ) ++ assert response.status == 303 ++ response = http.request( ++ "GET", f"{self.base_url}/redirect", retries=False, redirect=False ++ ) ++ assert response.status == 303 ++ + def test_cross_host_redirect(self) -> None: + with PoolManager() as http: + cross_host_location = f"{self.base_url_alt}/echo?a=b" +@@ -138,6 +221,24 @@ class TestPoolManager(HypercornDummyServerTestCase): + pool = http.connection_from_host(self.host, self.port) + assert pool.num_connections == 1 + ++ # Check when retries are configured for the pool manager. ++ with PoolManager(retries=1) as http: ++ with pytest.raises(MaxRetryError): ++ http.request( ++ "GET", ++ f"{self.base_url}/redirect", ++ fields={"target": f"/redirect?target={self.base_url}/"}, ++ ) ++ ++ # Here we allow more retries for the request. ++ response = http.request( ++ "GET", ++ f"{self.base_url}/redirect", ++ fields={"target": f"/redirect?target={self.base_url}/"}, ++ retries=2, ++ ) ++ assert response.status == 200 ++ + def test_redirect_cross_host_remove_headers(self) -> None: + with PoolManager() as http: + r = http.request( diff --git a/debian/patches/CVE-2025-50182.patch b/debian/patches/CVE-2025-50182.patch new file mode 100644 index 0000000..ecba754 --- /dev/null +++ b/debian/patches/CVE-2025-50182.patch @@ -0,0 +1,121 @@ +From: Illia Volochii +Date: Wed, 18 Jun 2025 16:30:35 +0300 +Subject: Merge commit from fork + +Origin: upstream, https://github.com/urllib3/urllib3/commit/7eb4a2aafe49a279c29b6d1f0ed0f42e9736194f +Bug-Debian: https://bugs.debian.org/1108077 +Last-Update: 2025-07-13 +--- + docs/reference/contrib/emscripten.rst | 2 +- + src/urllib3/contrib/emscripten/fetch.py | 20 +++++++++++++ + test/contrib/emscripten/test_emscripten.py | 46 ++++++++++++++++++++++++++++++ + 3 files changed, 67 insertions(+), 1 deletion(-) + +diff --git a/docs/reference/contrib/emscripten.rst b/docs/reference/contrib/emscripten.rst +index a8f1cda..4670757 100644 +--- a/docs/reference/contrib/emscripten.rst ++++ b/docs/reference/contrib/emscripten.rst +@@ -65,7 +65,7 @@ Features which are usable with Emscripten support are: + * Timeouts + * Retries + * Streaming (with Web Workers and Cross-Origin Isolation) +-* Redirects (determined by browser/runtime, not restrictable with urllib3) ++* Redirects (urllib3 controls redirects in Node.js but not in browsers where behavior is determined by runtime) + * Decompressing response bodies + + Features which don't work with Emscripten: +diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py +index a514306..6695821 100644 +--- a/src/urllib3/contrib/emscripten/fetch.py ++++ b/src/urllib3/contrib/emscripten/fetch.py +@@ -573,6 +573,11 @@ def send_jspi_request( + "method": request.method, + "signal": js_abort_controller.signal, + } ++ # Node.js returns the whole response (unlike opaqueredirect in browsers), ++ # so urllib3 can set `redirect: manual` to control redirects itself. ++ # https://stackoverflow.com/a/78524615 ++ if _is_node_js(): ++ fetch_data["redirect"] = "manual" + # Call JavaScript fetch (async api, returns a promise) + fetcher_promise_js = js.fetch(request.url, _obj_from_dict(fetch_data)) + # Now suspend WebAssembly until we resolve that promise +@@ -693,6 +698,21 @@ def has_jspi() -> bool: + return False + + ++def _is_node_js() -> bool: ++ """ ++ Check if we are in Node.js. ++ ++ :return: True if we are in Node.js. ++ :rtype: bool ++ """ ++ return ( ++ hasattr(js, "process") ++ and hasattr(js.process, "release") ++ # According to the Node.js documentation, the release name is always "node". ++ and js.process.release.name == "node" ++ ) ++ ++ + def streaming_ready() -> bool | None: + if _fetcher: + return _fetcher.streaming_ready +diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py +index 5eaa674..fbf89fc 100644 +--- a/test/contrib/emscripten/test_emscripten.py ++++ b/test/contrib/emscripten/test_emscripten.py +@@ -960,6 +960,52 @@ def test_redirects( + ) + + ++@pytest.mark.with_jspi ++def test_disabled_redirects( ++ selenium_coverage: typing.Any, testserver_http: PyodideServerInfo ++) -> None: ++ """ ++ Test that urllib3 can control redirects in Node.js. ++ """ ++ ++ @run_in_pyodide # type: ignore[misc] ++ def pyodide_test(selenium_coverage: typing.Any, host: str, port: int) -> None: ++ import pytest ++ ++ from urllib3 import PoolManager, request ++ from urllib3.contrib.emscripten.fetch import _is_node_js ++ from urllib3.exceptions import MaxRetryError ++ ++ if not _is_node_js(): ++ pytest.skip("urllib3 does not control redirects in browsers.") ++ ++ redirect_url = f"http://{host}:{port}/redirect" ++ ++ with PoolManager(retries=0) as http: ++ with pytest.raises(MaxRetryError): ++ http.request("GET", redirect_url) ++ ++ response = http.request("GET", redirect_url, redirect=False) ++ assert response.status == 303 ++ ++ with PoolManager(retries=False) as http: ++ response = http.request("GET", redirect_url) ++ assert response.status == 303 ++ ++ with pytest.raises(MaxRetryError): ++ request("GET", redirect_url, retries=0) ++ ++ response = request("GET", redirect_url, redirect=False) ++ assert response.status == 303 ++ ++ response = request("GET", redirect_url, retries=0, redirect=False) ++ assert response.status == 303 ++ ++ pyodide_test( ++ selenium_coverage, testserver_http.http_host, testserver_http.http_port ++ ) ++ ++ + def test_insecure_requests_warning( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo + ) -> None: diff --git a/debian/patches/httpx-0.28.patch b/debian/patches/httpx-0.28.patch new file mode 100644 index 0000000..7489f5d --- /dev/null +++ b/debian/patches/httpx-0.28.patch @@ -0,0 +1,55 @@ +From: Carl Smedstad +Date: Mon, 30 Dec 2024 16:04:28 +0100 +Subject: Ensure compatibility with httpx>=0.28 + +Version 0.28 of httpx removed support for supplying a path (of string +type) to verify, only a bool or an SSL context is now supported. +See: https://github.com/encode/httpx/releases/tag/0.28.0 + +Running the test suite with httpx 0.28 will break the dummy server and a +such number of tests in test/with_dummyserver/. + +To resolve this, create an SSL context in the ProxyApp init function and +supply that to AsyncClient, instead of a raw string. This change is +backwards compatible, i.e. the test suite will still succeed against +the currently pinned version of httpx, 0.25.2. + +Origin: https://github.com/urllib3/urllib3/pull/3545 +Bug-Debian: https://bugs.debian.org/1099277 +Last-Update: 2025-03-12 +--- + dummyserver/asgi_proxy.py | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/dummyserver/asgi_proxy.py b/dummyserver/asgi_proxy.py +index 107c5e0..00c0a1b 100755 +--- a/dummyserver/asgi_proxy.py ++++ b/dummyserver/asgi_proxy.py +@@ -1,5 +1,6 @@ + from __future__ import annotations + ++import ssl + import typing + + import httpx +@@ -29,7 +30,10 @@ async def _read_body(receive: ASGIReceiveCallable) -> bytes: + + class ProxyApp: + def __init__(self, upstream_ca_certs: str | None = None): +- self.upstream_ca_certs = upstream_ca_certs ++ self.ssl_context = None ++ if upstream_ca_certs: ++ self.ssl_context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) ++ self.ssl_context.load_verify_locations(cafile=upstream_ca_certs) + + async def __call__( + self, scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable +@@ -48,7 +52,7 @@ class ProxyApp: + receive: ASGIReceiveCallable, + send: ASGISendCallable, + ) -> None: +- async with httpx.AsyncClient(verify=self.upstream_ca_certs or True) as client: ++ async with httpx.AsyncClient(verify=self.ssl_context or True) as client: + client_response = await client.request( + method=scope["method"], + url=scope["path"], diff --git a/debian/patches/openssl-3.4.0.patch b/debian/patches/openssl-3.4.0.patch new file mode 100644 index 0000000..281b9a8 --- /dev/null +++ b/debian/patches/openssl-3.4.0.patch @@ -0,0 +1,31 @@ +From: Colin Watson +Date: Thu, 9 Jan 2025 21:29:30 +0000 +Subject: SingleTLSLayerTestCase: Catch BrokenPipeError + +OpenSSL 3.4.0 returns `ERR_LIB_SYS` in some more situations than it used +to. In the case exercised by +`SingleTLSLayerTestCase.test_close_after_handshake`, +https://github.com/python/cpython/pull/127361 (also backported to the +3.12 and 3.13 branches) turns this into `BrokenPipeError`. It seems +reasonable to treat this in the same way as `ConnectionAbortedError` and +`ConnectionResetError`. + +Forwarded: https://github.com/urllib3/urllib3/pull/3547 +Last-Update: 2025-01-17 +--- + test/test_ssltransport.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py +index 4f0880d..0097179 100644 +--- a/test/test_ssltransport.py ++++ b/test/test_ssltransport.py +@@ -122,7 +122,7 @@ class SingleTLSLayerTestCase(SocketDummyServerTestCase): + return + validate_request(request) + ssock.send(sample_response()) +- except (ConnectionAbortedError, ConnectionResetError): ++ except (ConnectionAbortedError, ConnectionResetError, BrokenPipeError): + return + + chosen_handler = handler if handler else socket_handler diff --git a/debian/patches/series b/debian/patches/series index 5859106..6c2bb46 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,2 +1,5 @@ -01_do-not-use-embedded-python-six.patch -0002-implement-missing-importlib-methods.patch +test_http2_probe_blocked_per_thread-requires_network.patch +openssl-3.4.0.patch +httpx-0.28.patch +CVE-2025-50181.patch +CVE-2025-50182.patch diff --git a/debian/patches/test_http2_probe_blocked_per_thread-requires_network.patch b/debian/patches/test_http2_probe_blocked_per_thread-requires_network.patch new file mode 100644 index 0000000..1e9aa90 --- /dev/null +++ b/debian/patches/test_http2_probe_blocked_per_thread-requires_network.patch @@ -0,0 +1,23 @@ +From: Colin Watson +Date: Tue, 12 Nov 2024 23:47:03 +0000 +Subject: Mark test_http2_probe_blocked_per_thread with requires_network + +It fails if it can't connect to `TARPIT_HOST`. + +Last-Update: 2024-11-12 +--- + test/with_dummyserver/test_https.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py +index 162d089..6702f3a 100644 +--- a/test/with_dummyserver/test_https.py ++++ b/test/with_dummyserver/test_https.py +@@ -1049,6 +1049,7 @@ class BaseTestHTTPS(HTTPSHypercornDummyServerTestCase): + finally: + urllib3.http2.extract_from_urllib3() + ++ @requires_network() + def test_http2_probe_blocked_per_thread(self) -> None: + state, current_thread, last_action = None, None, time.perf_counter() + diff --git a/debian/pybuild.testfiles b/debian/pybuild.testfiles new file mode 100644 index 0000000..14288ec --- /dev/null +++ b/debian/pybuild.testfiles @@ -0,0 +1,3 @@ +dummyserver +pyproject.toml +test diff --git a/debian/rules b/debian/rules index 05f73e8..9e418c7 100755 --- a/debian/rules +++ b/debian/rules @@ -2,32 +2,26 @@ export PYBUILD_NAME=urllib3 export PYTHONWARNINGS=d -export PYBUILD_BEFORE_TEST=rm -f test/conftest.py +# Use patched hypercorn; see https://github.com/urllib3/urllib3/issues/3334 +export PYTHONPATH=$(CURDIR)/debian/vendor export PYBUILD_TEST_PYTEST=1 -# test_respect_retry_after_header_sleep needs pytest-freezegun -export PYBUILD_TEST_ARGS=\ - --ignore=test/appengine \ - --ignore=test/contrib/test_pyopenssl.py \ - --ignore=test/contrib/test_securetransport.py \ - --ignore=test/contrib/test_socks.py \ - --ignore=test/test_connectionpool.py \ - --ignore=test/test_ssl.py \ - --ignore=test/test_ssltransport.py \ - --ignore=test/with_dummyserver \ - -k 'not test_recent_date and not test_respect_retry_after_header_sleep' -# Skip timing tests -export TRAVIS=1 +export PYBUILD_TEST_ARGS=-k "not requires_network and not test_recent_date" export LC_ALL=C.UTF-8 +export CI=1 # Use longer timeouts, see https://github.com/urllib3/urllib3/issues/3164 +# pybuild exports http_proxy=http://127.0.0.1:9/ by default, but that breaks +# some tests here and the package doesn't seem to try to connect to the +# internet without it. +export http_proxy= %: - dh $@ --with python3 --buildsystem=pybuild + dh $@ --buildsystem=pybuild override_dh_clean: dh_clean find . -type d -name .pytest_cache -exec rm -rf {} + -override_dh_auto_configure: - rm -f src/urllib3/packages/six.py +override_dh_auto_test: + PYTHONPATH=$(CURDIR)/debian/vendor dh_auto_test override_dh_installchangelogs: dh_installchangelogs CHANGES.rst diff --git a/debian/tests/control b/debian/tests/control index fdb5f27..680034a 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -1,11 +1,19 @@ Tests: python3-urllib3 +Restrictions: allow-stderr Depends: python3-all, python3-brotli, python3-coverage, + python3-cryptography, + python3-h2, + python3-httpx, python3-idna, - python3-mock, + python3-openssl, python3-pytest, - python3-six, - python3-tornado, + python3-pytest-timeout, + python3-quart, + python3-quart-trio, + python3-socks, + python3-trio, + python3-trustme, python3-urllib3, diff --git a/debian/tests/python3-urllib3 b/debian/tests/python3-urllib3 index 2796111..fedc00e 100755 --- a/debian/tests/python3-urllib3 +++ b/debian/tests/python3-urllib3 @@ -2,23 +2,17 @@ set -efu export PYTHONWARNINGS=d +# Use patched hypercorn; see https://github.com/urllib3/urllib3/issues/3334 +export PYTHONPATH=$(pwd)/debian/vendor +# Use longer timeouts, see https://github.com/urllib3/urllib3/issues/3164 +export CI=1 python3_all="$(py3versions -s 2>/dev/null)" -cp -r test "$AUTOPKGTEST_TMP/" +cp -r dummyserver test "$AUTOPKGTEST_TMP/" cd "$AUTOPKGTEST_TMP" -rm -f test/conftest.py for py in $python3_all; do echo "=== $py ===" - $py -m pytest --verbose --ignore=test/appengine \ - --ignore=test/contrib/test_pyopenssl.py \ - --ignore=test/contrib/test_securetransport.py \ - --ignore=test/contrib/test_socks.py \ - --ignore=test/test_connectionpool.py \ - --ignore=test/test_ssl.py \ - --ignore=test/test_ssltransport.py \ - --ignore=test/with_dummyserver \ - -k 'not test_recent_date and not test_respect_retry_after_header_sleep' 2>&1 - + $py -m pytest --verbose -k "not requires_network and not test_recent_date" done diff --git a/debian/vendor/hypercorn/__init__.py b/debian/vendor/hypercorn/__init__.py new file mode 100644 index 0000000..5931e8c --- /dev/null +++ b/debian/vendor/hypercorn/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from .config import Config + +__all__ = ("Config",) diff --git a/debian/vendor/hypercorn/__main__.py b/debian/vendor/hypercorn/__main__.py new file mode 100644 index 0000000..b3dc0e8 --- /dev/null +++ b/debian/vendor/hypercorn/__main__.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import argparse +import ssl +import sys +import warnings +from typing import List, Optional + +from .config import Config +from .run import run + +sentinel = object() + + +def _load_config(config_path: Optional[str]) -> Config: + if config_path is None: + return Config() + elif config_path.startswith("python:"): + return Config.from_object(config_path[len("python:") :]) + elif config_path.startswith("file:"): + return Config.from_pyfile(config_path[len("file:") :]) + else: + return Config.from_toml(config_path) + + +def main(sys_args: Optional[List[str]] = None) -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "application", help="The application to dispatch to as path.to.module:instance.path" + ) + parser.add_argument("--access-log", help="Deprecated, see access-logfile", default=sentinel) + parser.add_argument( + "--access-logfile", + help="The target location for the access log, use `-` for stdout", + default=sentinel, + ) + parser.add_argument( + "--access-logformat", + help="The log format for the access log, see help docs", + default=sentinel, + ) + parser.add_argument( + "--backlog", help="The maximum number of pending connections", type=int, default=sentinel + ) + parser.add_argument( + "-b", + "--bind", + dest="binds", + help=""" The TCP host/address to bind to. Should be either host:port, host, + unix:path or fd://num, e.g. 127.0.0.1:5000, 127.0.0.1, + unix:/tmp/socket or fd://33 respectively. """, + default=[], + action="append", + ) + parser.add_argument("--ca-certs", help="Path to the SSL CA certificate file", default=sentinel) + parser.add_argument("--certfile", help="Path to the SSL certificate file", default=sentinel) + parser.add_argument("--cert-reqs", help="See verify mode argument", type=int, default=sentinel) + parser.add_argument("--ciphers", help="Ciphers to use for the SSL setup", default=sentinel) + parser.add_argument( + "-c", + "--config", + help="Location of a TOML config file, or when prefixed with `file:` a Python file, or when prefixed with `python:` a Python module.", # noqa: E501 + default=None, + ) + parser.add_argument( + "--debug", + help="Enable debug mode, i.e. extra logging and checks", + action="store_true", + default=sentinel, + ) + parser.add_argument("--error-log", help="Deprecated, see error-logfile", default=sentinel) + parser.add_argument( + "--error-logfile", + "--log-file", + dest="error_logfile", + help="The target location for the error log, use `-` for stderr", + default=sentinel, + ) + parser.add_argument( + "--graceful-timeout", + help="""Time to wait after SIGTERM or Ctrl-C for any remaining requests (tasks) + to complete.""", + default=sentinel, + type=int, + ) + parser.add_argument( + "--read-timeout", + help="""Seconds to wait before timing out reads on TCP sockets""", + default=sentinel, + type=int, + ) + parser.add_argument( + "-g", "--group", help="Group to own any unix sockets.", default=sentinel, type=int + ) + parser.add_argument( + "-k", + "--worker-class", + dest="worker_class", + help="The type of worker to use. " + "Options include asyncio, uvloop (pip install hypercorn[uvloop]), " + "and trio (pip install hypercorn[trio]).", + default=sentinel, + ) + parser.add_argument( + "--keep-alive", + help="Seconds to keep inactive connections alive for", + default=sentinel, + type=int, + ) + parser.add_argument("--keyfile", help="Path to the SSL key file", default=sentinel) + parser.add_argument( + "--keyfile-password", help="Password to decrypt the SSL key file", default=sentinel + ) + parser.add_argument( + "--insecure-bind", + dest="insecure_binds", + help="""The TCP host/address to bind to. SSL options will not apply to these binds. + See *bind* for formatting options. Care must be taken! See HTTP -> HTTPS redirection docs. + """, + default=[], + action="append", + ) + parser.add_argument( + "--log-config", + help=""""A Python logging configuration file. This can be prefixed with + 'json:' or 'toml:' to load the configuration from a file in + that format. Default is the logging ini format.""", + default=sentinel, + ) + parser.add_argument( + "--log-level", help="The (error) log level, defaults to info", default=sentinel + ) + parser.add_argument( + "-p", "--pid", help="Location to write the PID (Program ID) to.", default=sentinel + ) + parser.add_argument( + "--quic-bind", + dest="quic_binds", + help="""The UDP/QUIC host/address to bind to. See *bind* for formatting + options. + """, + default=[], + action="append", + ) + parser.add_argument( + "--reload", + help="Enable automatic reloads on code changes", + action="store_true", + default=sentinel, + ) + parser.add_argument( + "--root-path", help="The setting for the ASGI root_path variable", default=sentinel + ) + parser.add_argument( + "--server-name", + dest="server_names", + help="""The hostnames that can be served, requests to different hosts + will be responded to with 404s. + """, + default=[], + action="append", + ) + parser.add_argument( + "--statsd-host", help="The host:port of the statsd server", default=sentinel + ) + parser.add_argument("--statsd-prefix", help="Prefix for all statsd messages", default="") + parser.add_argument( + "-m", + "--umask", + help="The permissions bit mask to use on any unix sockets.", + default=sentinel, + type=int, + ) + parser.add_argument( + "-u", "--user", help="User to own any unix sockets.", default=sentinel, type=int + ) + + def _convert_verify_mode(value: str) -> ssl.VerifyMode: + try: + return ssl.VerifyMode[value] + except KeyError: + raise argparse.ArgumentTypeError(f"'{value}' is not a valid verify mode") + + parser.add_argument( + "--verify-mode", + help="SSL verify mode for peer's certificate, see ssl.VerifyMode enum for possible values.", + type=_convert_verify_mode, + default=sentinel, + ) + parser.add_argument( + "--websocket-ping-interval", + help="""If set this is the time in seconds between pings sent to the client. + This can be used to keep the websocket connection alive.""", + default=sentinel, + type=int, + ) + parser.add_argument( + "-w", + "--workers", + dest="workers", + help="The number of workers to spawn and use", + default=sentinel, + type=int, + ) + args = parser.parse_args(sys_args or sys.argv[1:]) + config = _load_config(args.config) + config.application_path = args.application + + if args.log_level is not sentinel: + config.loglevel = args.log_level + if args.access_logformat is not sentinel: + config.access_log_format = args.access_logformat + if args.access_log is not sentinel: + warnings.warn( + "The --access-log argument is deprecated, use `--access-logfile` instead", + DeprecationWarning, + ) + config.accesslog = args.access_log + if args.access_logfile is not sentinel: + config.accesslog = args.access_logfile + if args.backlog is not sentinel: + config.backlog = args.backlog + if args.ca_certs is not sentinel: + config.ca_certs = args.ca_certs + if args.certfile is not sentinel: + config.certfile = args.certfile + if args.cert_reqs is not sentinel: + config.cert_reqs = args.cert_reqs + if args.ciphers is not sentinel: + config.ciphers = args.ciphers + if args.debug is not sentinel: + config.debug = args.debug + if args.error_log is not sentinel: + warnings.warn( + "The --error-log argument is deprecated, use `--error-logfile` instead", + DeprecationWarning, + ) + config.errorlog = args.error_log + if args.error_logfile is not sentinel: + config.errorlog = args.error_logfile + if args.graceful_timeout is not sentinel: + config.graceful_timeout = args.graceful_timeout + if args.read_timeout is not sentinel: + config.read_timeout = args.read_timeout + if args.group is not sentinel: + config.group = args.group + if args.keep_alive is not sentinel: + config.keep_alive_timeout = args.keep_alive + if args.keyfile is not sentinel: + config.keyfile = args.keyfile + if args.keyfile_password is not sentinel: + config.keyfile_password = args.keyfile_password + if args.log_config is not sentinel: + config.logconfig = args.log_config + if args.pid is not sentinel: + config.pid_path = args.pid + if args.root_path is not sentinel: + config.root_path = args.root_path + if args.reload is not sentinel: + config.use_reloader = args.reload + if args.statsd_host is not sentinel: + config.statsd_host = args.statsd_host + if args.statsd_prefix is not sentinel: + config.statsd_prefix = args.statsd_prefix + if args.umask is not sentinel: + config.umask = args.umask + if args.user is not sentinel: + config.user = args.user + if args.worker_class is not sentinel: + config.worker_class = args.worker_class + if args.verify_mode is not sentinel: + config.verify_mode = args.verify_mode + if args.websocket_ping_interval is not sentinel: + config.websocket_ping_interval = args.websocket_ping_interval + if args.workers is not sentinel: + config.workers = args.workers + + if len(args.binds) > 0: + config.bind = args.binds + if len(args.insecure_binds) > 0: + config.insecure_bind = args.insecure_binds + if len(args.quic_binds) > 0: + config.quic_bind = args.quic_binds + if len(args.server_names) > 0: + config.server_names = args.server_names + + run(config) + + +if __name__ == "__main__": + main() diff --git a/debian/vendor/hypercorn/app_wrappers.py b/debian/vendor/hypercorn/app_wrappers.py new file mode 100644 index 0000000..769e014 --- /dev/null +++ b/debian/vendor/hypercorn/app_wrappers.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from functools import partial +from io import BytesIO +from typing import Callable, List, Optional, Tuple + +from .typing import ( + ASGIFramework, + ASGIReceiveCallable, + ASGISendCallable, + HTTPScope, + Scope, + WSGIFramework, +) + + +class InvalidPathError(Exception): + pass + + +class ASGIWrapper: + def __init__(self, app: ASGIFramework) -> None: + self.app = app + + async def __call__( + self, + scope: Scope, + receive: ASGIReceiveCallable, + send: ASGISendCallable, + sync_spawn: Callable, + call_soon: Callable, + ) -> None: + await self.app(scope, receive, send) + + +class WSGIWrapper: + def __init__(self, app: WSGIFramework, max_body_size: int) -> None: + self.app = app + self.max_body_size = max_body_size + + async def __call__( + self, + scope: Scope, + receive: ASGIReceiveCallable, + send: ASGISendCallable, + sync_spawn: Callable, + call_soon: Callable, + ) -> None: + if scope["type"] == "http": + await self.handle_http(scope, receive, send, sync_spawn, call_soon) + elif scope["type"] == "websocket": + await send({"type": "websocket.close"}) # type: ignore + elif scope["type"] == "lifespan": + return + else: + raise Exception(f"Unknown scope type, {scope['type']}") + + async def handle_http( + self, + scope: HTTPScope, + receive: ASGIReceiveCallable, + send: ASGISendCallable, + sync_spawn: Callable, + call_soon: Callable, + ) -> None: + body = bytearray() + while True: + message = await receive() + body.extend(message.get("body", b"")) # type: ignore + if len(body) > self.max_body_size: + await send({"type": "http.response.start", "status": 400, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + return + if not message.get("more_body"): + break + + try: + environ = _build_environ(scope, body) + except InvalidPathError: + await send({"type": "http.response.start", "status": 404, "headers": []}) + else: + await sync_spawn(self.run_app, environ, partial(call_soon, send)) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + + def run_app(self, environ: dict, send: Callable) -> None: + headers: List[Tuple[bytes, bytes]] + status_code: Optional[int] = None + + def start_response( + status: str, + response_headers: List[Tuple[str, str]], + exc_info: Optional[Exception] = None, + ) -> None: + nonlocal headers, status_code + + raw, _ = status.split(" ", 1) + status_code = int(raw) + headers = [ + (name.lower().encode("ascii"), value.encode("ascii")) + for name, value in response_headers + ] + send({"type": "http.response.start", "status": status_code, "headers": headers}) + + for output in self.app(environ, start_response): + send({"type": "http.response.body", "body": output, "more_body": True}) + + +def _build_environ(scope: HTTPScope, body: bytes) -> dict: + server = scope.get("server") or ("localhost", 80) + path = scope["path"] + script_name = scope.get("root_path", "") + if path.startswith(script_name): + path = path[len(script_name) :] + path = path if path != "" else "/" + else: + raise InvalidPathError() + + environ = { + "REQUEST_METHOD": scope["method"], + "SCRIPT_NAME": script_name.encode("utf8").decode("latin1"), + "PATH_INFO": path.encode("utf8").decode("latin1"), + "QUERY_STRING": scope["query_string"].decode("ascii"), + "SERVER_NAME": server[0], + "SERVER_PORT": server[1], + "SERVER_PROTOCOL": "HTTP/%s" % scope["http_version"], + "wsgi.version": (1, 0), + "wsgi.url_scheme": scope.get("scheme", "http"), + "wsgi.input": BytesIO(body), + "wsgi.errors": BytesIO(), + "wsgi.multithread": True, + "wsgi.multiprocess": True, + "wsgi.run_once": False, + } + + if scope.get("client") is not None: + environ["REMOTE_ADDR"] = scope["client"][0] + + for raw_name, raw_value in scope.get("headers", []): + name = raw_name.decode("latin1") + if name == "content-length": + corrected_name = "CONTENT_LENGTH" + elif name == "content-type": + corrected_name = "CONTENT_TYPE" + else: + corrected_name = "HTTP_%s" % name.upper().replace("-", "_") + # HTTPbis say only ASCII chars are allowed in headers, but we latin1 just in case + value = raw_value.decode("latin1") + if corrected_name in environ: + value = environ[corrected_name] + "," + value # type: ignore + environ[corrected_name] = value + return environ diff --git a/debian/vendor/hypercorn/asyncio/__init__.py b/debian/vendor/hypercorn/asyncio/__init__.py new file mode 100644 index 0000000..3755da0 --- /dev/null +++ b/debian/vendor/hypercorn/asyncio/__init__.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import warnings +from typing import Awaitable, Callable, Literal, Optional + +from .run import worker_serve +from ..config import Config +from ..typing import Framework +from ..utils import wrap_app + + +async def serve( + app: Framework, + config: Config, + *, + shutdown_trigger: Optional[Callable[..., Awaitable]] = None, + mode: Optional[Literal["asgi", "wsgi"]] = None, +) -> None: + """Serve an ASGI or WSGI framework app given the config. + + This allows for a programmatic way to serve an ASGI or WSGI + framework, it can be used via, + + .. code-block:: python + + asyncio.run(serve(app, config)) + + It is assumed that the event-loop is configured before calling + this function, therefore configuration values that relate to loop + setup or process setup are ignored. + + Arguments: + app: The ASGI or WSGI application to serve. + config: A Hypercorn configuration object. + shutdown_trigger: This should return to trigger a graceful + shutdown. + mode: Specify if the app is WSGI or ASGI. + """ + if config.debug: + warnings.warn("The config `debug` has no affect when using serve", Warning) + if config.workers != 1: + warnings.warn("The config `workers` has no affect when using serve", Warning) + + await worker_serve( + wrap_app(app, config.wsgi_max_body_size, mode), config, shutdown_trigger=shutdown_trigger + ) diff --git a/debian/vendor/hypercorn/asyncio/lifespan.py b/debian/vendor/hypercorn/asyncio/lifespan.py new file mode 100644 index 0000000..244950c --- /dev/null +++ b/debian/vendor/hypercorn/asyncio/lifespan.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import asyncio +from functools import partial +from typing import Any, Callable + +from ..config import Config +from ..typing import AppWrapper, ASGIReceiveEvent, ASGISendEvent, LifespanScope +from ..utils import LifespanFailureError, LifespanTimeoutError + + +class UnexpectedMessageError(Exception): + pass + + +class Lifespan: + def __init__(self, app: AppWrapper, config: Config, loop: asyncio.AbstractEventLoop) -> None: + self.app = app + self.config = config + self.startup = asyncio.Event() + self.shutdown = asyncio.Event() + self.app_queue: asyncio.Queue = asyncio.Queue(config.max_app_queue_size) + self.supported = True + self.loop = loop + + # This mimics the Trio nursery.start task_status and is + # required to ensure the support has been checked before + # waiting on timeouts. + self._started = asyncio.Event() + + async def handle_lifespan(self) -> None: + self._started.set() + scope: LifespanScope = { + "type": "lifespan", + "asgi": {"spec_version": "2.0", "version": "3.0"}, + } + + def _call_soon(func: Callable, *args: Any) -> Any: + future = asyncio.run_coroutine_threadsafe(func(*args), self.loop) + return future.result() + + try: + await self.app( + scope, + self.asgi_receive, + self.asgi_send, + partial(self.loop.run_in_executor, None), + _call_soon, + ) + except LifespanFailureError: + # Lifespan failures should crash the server + raise + except Exception: + self.supported = False + if not self.startup.is_set(): + await self.config.log.warning( + "ASGI Framework Lifespan error, continuing without Lifespan support" + ) + elif not self.shutdown.is_set(): + await self.config.log.exception( + "ASGI Framework Lifespan error, shutdown without Lifespan support" + ) + else: + await self.config.log.exception("ASGI Framework Lifespan errored after shutdown.") + finally: + self.startup.set() + self.shutdown.set() + + async def wait_for_startup(self) -> None: + await self._started.wait() + if not self.supported: + return + + await self.app_queue.put({"type": "lifespan.startup"}) + try: + await asyncio.wait_for(self.startup.wait(), timeout=self.config.startup_timeout) + except asyncio.TimeoutError as error: + raise LifespanTimeoutError("startup") from error + + async def wait_for_shutdown(self) -> None: + await self._started.wait() + if not self.supported: + return + + await self.app_queue.put({"type": "lifespan.shutdown"}) + try: + await asyncio.wait_for(self.shutdown.wait(), timeout=self.config.shutdown_timeout) + except asyncio.TimeoutError as error: + raise LifespanTimeoutError("shutdown") from error + + async def asgi_receive(self) -> ASGIReceiveEvent: + return await self.app_queue.get() + + async def asgi_send(self, message: ASGISendEvent) -> None: + if message["type"] == "lifespan.startup.complete": + self.startup.set() + elif message["type"] == "lifespan.shutdown.complete": + self.shutdown.set() + elif message["type"] == "lifespan.startup.failed": + self.startup.set() + raise LifespanFailureError("startup", message.get("message", "")) + elif message["type"] == "lifespan.shutdown.failed": + self.shutdown.set() + raise LifespanFailureError("shutdown", message.get("message", "")) + else: + raise UnexpectedMessageError(message["type"]) diff --git a/debian/vendor/hypercorn/asyncio/run.py b/debian/vendor/hypercorn/asyncio/run.py new file mode 100644 index 0000000..4774538 --- /dev/null +++ b/debian/vendor/hypercorn/asyncio/run.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import asyncio +import platform +import signal +import ssl +from functools import partial +from multiprocessing.synchronize import Event as EventType +from os import getpid +from socket import socket +from typing import Any, Awaitable, Callable, Optional, Set + +from .lifespan import Lifespan +from .statsd import StatsdLogger +from .tcp_server import TCPServer +from .udp_server import UDPServer +from .worker_context import WorkerContext +from ..config import Config, Sockets +from ..typing import AppWrapper +from ..utils import ( + check_multiprocess_shutdown_event, + load_application, + raise_shutdown, + repr_socket_addr, + ShutdownError, +) + +try: + from asyncio import Runner +except ImportError: + from taskgroup import Runner # type: ignore + + +def _share_socket(sock: socket) -> socket: + # Windows requires the socket be explicitly shared across + # multiple workers (processes). + from socket import fromshare # type: ignore + + sock_data = sock.share(getpid()) # type: ignore + return fromshare(sock_data) + + +async def worker_serve( + app: AppWrapper, + config: Config, + *, + sockets: Optional[Sockets] = None, + shutdown_trigger: Optional[Callable[..., Awaitable]] = None, +) -> None: + config.set_statsd_logger_class(StatsdLogger) + + loop = asyncio.get_event_loop() + + if shutdown_trigger is None: + signal_event = asyncio.Event() + + def _signal_handler(*_: Any) -> None: # noqa: N803 + signal_event.set() + + for signal_name in {"SIGINT", "SIGTERM", "SIGBREAK"}: + if hasattr(signal, signal_name): + try: + loop.add_signal_handler(getattr(signal, signal_name), _signal_handler) + except NotImplementedError: + # Add signal handler may not be implemented on Windows + signal.signal(getattr(signal, signal_name), _signal_handler) + + shutdown_trigger = signal_event.wait # type: ignore + + lifespan = Lifespan(app, config, loop) + + lifespan_task = loop.create_task(lifespan.handle_lifespan()) + await lifespan.wait_for_startup() + if lifespan_task.done(): + exception = lifespan_task.exception() + if exception is not None: + raise exception + + if sockets is None: + sockets = config.create_sockets() + + ssl_handshake_timeout = None + if config.ssl_enabled: + ssl_context = config.create_ssl_context() + ssl_handshake_timeout = config.ssl_handshake_timeout + + context = WorkerContext() + server_tasks: Set[asyncio.Task] = set() + + async def _server_callback(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + nonlocal server_tasks + + task = asyncio.current_task(loop) + server_tasks.add(task) + task.add_done_callback(server_tasks.discard) + await TCPServer(app, loop, config, context, reader, writer) + + servers = [] + for sock in sockets.secure_sockets: + if config.workers > 1 and platform.system() == "Windows": + sock = _share_socket(sock) + + servers.append( + await asyncio.start_server( + _server_callback, + backlog=config.backlog, + ssl=ssl_context, + sock=sock, + ssl_handshake_timeout=ssl_handshake_timeout, + ) + ) + bind = repr_socket_addr(sock.family, sock.getsockname()) + await config.log.info(f"Running on https://{bind} (CTRL + C to quit)") + + for sock in sockets.insecure_sockets: + if config.workers > 1 and platform.system() == "Windows": + sock = _share_socket(sock) + + servers.append( + await asyncio.start_server(_server_callback, backlog=config.backlog, sock=sock) + ) + bind = repr_socket_addr(sock.family, sock.getsockname()) + await config.log.info(f"Running on http://{bind} (CTRL + C to quit)") + + for sock in sockets.quic_sockets: + if config.workers > 1 and platform.system() == "Windows": + sock = _share_socket(sock) + + _, protocol = await loop.create_datagram_endpoint( + lambda: UDPServer(app, loop, config, context), sock=sock + ) + task = loop.create_task(protocol.run()) + server_tasks.add(task) + task.add_done_callback(server_tasks.discard) + bind = repr_socket_addr(sock.family, sock.getsockname()) + await config.log.info(f"Running on https://{bind} (QUIC) (CTRL + C to quit)") + + try: + await raise_shutdown(shutdown_trigger) + except (ShutdownError, KeyboardInterrupt): + pass + finally: + await context.terminated.set() + + for server in servers: + server.close() + await server.wait_closed() + + try: + gathered_server_tasks = asyncio.gather(*server_tasks) + await asyncio.wait_for(gathered_server_tasks, config.graceful_timeout) + except asyncio.TimeoutError: + pass + finally: + # Retrieve the Gathered Tasks Cancelled Exception, to + # prevent a warning that this hasn't been done. + gathered_server_tasks.exception() + + await lifespan.wait_for_shutdown() + lifespan_task.cancel() + await lifespan_task + + +def asyncio_worker( + config: Config, sockets: Optional[Sockets] = None, shutdown_event: Optional[EventType] = None +) -> None: + app = load_application(config.application_path, config.wsgi_max_body_size) + + shutdown_trigger = None + if shutdown_event is not None: + shutdown_trigger = partial(check_multiprocess_shutdown_event, shutdown_event, asyncio.sleep) + + if config.workers > 1 and platform.system() == "Windows": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) # type: ignore + + _run( + partial(worker_serve, app, config, sockets=sockets), + debug=config.debug, + shutdown_trigger=shutdown_trigger, + ) + + +def uvloop_worker( + config: Config, sockets: Optional[Sockets] = None, shutdown_event: Optional[EventType] = None +) -> None: + try: + import uvloop + except ImportError as error: + raise Exception("uvloop is not installed") from error + else: + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + app = load_application(config.application_path, config.wsgi_max_body_size) + + shutdown_trigger = None + if shutdown_event is not None: + shutdown_trigger = partial(check_multiprocess_shutdown_event, shutdown_event, asyncio.sleep) + + _run( + partial(worker_serve, app, config, sockets=sockets), + debug=config.debug, + shutdown_trigger=shutdown_trigger, + ) + + +def _run( + main: Callable, + *, + debug: bool = False, + shutdown_trigger: Optional[Callable[..., Awaitable[None]]] = None, +) -> None: + with Runner(debug=debug) as runner: + runner.get_loop().set_exception_handler(_exception_handler) + runner.run(main(shutdown_trigger=shutdown_trigger)) + + +def _exception_handler(loop: asyncio.AbstractEventLoop, context: dict) -> None: + exception = context.get("exception") + if isinstance(exception, ssl.SSLError): + pass # Handshake failure + else: + loop.default_exception_handler(context) diff --git a/debian/vendor/hypercorn/asyncio/statsd.py b/debian/vendor/hypercorn/asyncio/statsd.py new file mode 100644 index 0000000..cd2cafa --- /dev/null +++ b/debian/vendor/hypercorn/asyncio/statsd.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import asyncio +from typing import Optional + +from ..config import Config +from ..statsd import StatsdLogger as Base + + +class _DummyProto(asyncio.DatagramProtocol): + pass + + +class StatsdLogger(Base): + def __init__(self, config: Config) -> None: + super().__init__(config) + self.address = config.statsd_host.rsplit(":", 1) + self.transport: Optional[asyncio.BaseTransport] = None + + async def _socket_send(self, message: bytes) -> None: + if self.transport is None: + self.transport, _ = await asyncio.get_event_loop().create_datagram_endpoint( + _DummyProto, remote_addr=(self.address[0], int(self.address[1])) + ) + + self.transport.sendto(message) # type: ignore diff --git a/debian/vendor/hypercorn/asyncio/task_group.py b/debian/vendor/hypercorn/asyncio/task_group.py new file mode 100644 index 0000000..2e58903 --- /dev/null +++ b/debian/vendor/hypercorn/asyncio/task_group.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import asyncio +from functools import partial +from types import TracebackType +from typing import Any, Awaitable, Callable, Optional + +from ..config import Config +from ..typing import AppWrapper, ASGIReceiveCallable, ASGIReceiveEvent, ASGISendEvent, Scope + +try: + from asyncio import TaskGroup as AsyncioTaskGroup +except ImportError: + from taskgroup import TaskGroup as AsyncioTaskGroup # type: ignore + + +async def _handle( + app: AppWrapper, + config: Config, + scope: Scope, + receive: ASGIReceiveCallable, + send: Callable[[Optional[ASGISendEvent]], Awaitable[None]], + sync_spawn: Callable, + call_soon: Callable, +) -> None: + try: + await app(scope, receive, send, sync_spawn, call_soon) + except asyncio.CancelledError: + raise + except Exception: + await config.log.exception("Error in ASGI Framework") + finally: + await send(None) + + +class TaskGroup: + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._task_group = AsyncioTaskGroup() + + async def spawn_app( + self, + app: AppWrapper, + config: Config, + scope: Scope, + send: Callable[[Optional[ASGISendEvent]], Awaitable[None]], + ) -> Callable[[ASGIReceiveEvent], Awaitable[None]]: + app_queue: asyncio.Queue[ASGIReceiveEvent] = asyncio.Queue(config.max_app_queue_size) + + def _call_soon(func: Callable, *args: Any) -> Any: + future = asyncio.run_coroutine_threadsafe(func(*args), self._loop) + return future.result() + + self.spawn( + _handle, + app, + config, + scope, + app_queue.get, + send, + partial(self._loop.run_in_executor, None), + _call_soon, + ) + return app_queue.put + + def spawn(self, func: Callable, *args: Any) -> None: + self._task_group.create_task(func(*args)) + + async def __aenter__(self) -> "TaskGroup": + await self._task_group.__aenter__() + return self + + async def __aexit__(self, exc_type: type, exc_value: BaseException, tb: TracebackType) -> None: + await self._task_group.__aexit__(exc_type, exc_value, tb) diff --git a/debian/vendor/hypercorn/asyncio/tcp_server.py b/debian/vendor/hypercorn/asyncio/tcp_server.py new file mode 100644 index 0000000..34bb4fb --- /dev/null +++ b/debian/vendor/hypercorn/asyncio/tcp_server.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import asyncio +from ssl import SSLError +from typing import Any, Generator, Optional + +from .task_group import TaskGroup +from .worker_context import WorkerContext +from ..config import Config +from ..events import Closed, Event, RawData, Updated +from ..protocol import ProtocolWrapper +from ..typing import AppWrapper +from ..utils import parse_socket_addr + +MAX_RECV = 2**16 + + +class TCPServer: + def __init__( + self, + app: AppWrapper, + loop: asyncio.AbstractEventLoop, + config: Config, + context: WorkerContext, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + self.app = app + self.config = config + self.context = context + self.loop = loop + self.protocol: ProtocolWrapper + self.reader = reader + self.writer = writer + self.send_lock = asyncio.Lock() + self.idle_lock = asyncio.Lock() + + self._idle_handle: Optional[asyncio.Task] = None + + def __await__(self) -> Generator[Any, None, None]: + return self.run().__await__() + + async def run(self) -> None: + socket = self.writer.get_extra_info("socket") + try: + client = parse_socket_addr(socket.family, socket.getpeername()) + server = parse_socket_addr(socket.family, socket.getsockname()) + ssl_object = self.writer.get_extra_info("ssl_object") + if ssl_object is not None: + tls = {} + alpn_protocol = ssl_object.selected_alpn_protocol() + else: + tls = None + alpn_protocol = "http/1.1" + + async with TaskGroup(self.loop) as task_group: + self.protocol = ProtocolWrapper( + self.app, + self.config, + self.context, + task_group, + tls, + client, + server, + self.protocol_send, + alpn_protocol, + (self.reader, self.writer), + ) + await self.protocol.initiate() + await self._start_idle() + await self._read_data() + except OSError: + pass + finally: + await self._close() + + async def protocol_send(self, event: Event) -> None: + if isinstance(event, RawData): + async with self.send_lock: + try: + self.writer.write(event.data) + await self.writer.drain() + except (ConnectionError, RuntimeError): + await self.protocol.handle(Closed()) + elif isinstance(event, Closed): + await self._close() + elif isinstance(event, Updated): + if event.idle: + await self._start_idle() + else: + await self._stop_idle() + + async def _read_data(self) -> None: + while not self.reader.at_eof(): + try: + data = await asyncio.wait_for(self.reader.read(MAX_RECV), self.config.read_timeout) + except ( + ConnectionError, + OSError, + asyncio.TimeoutError, + TimeoutError, + SSLError, + ): + break + else: + await self.protocol.handle(RawData(data)) + + await self.protocol.handle(Closed()) + + async def _close(self) -> None: + try: + self.writer.write_eof() + except (NotImplementedError, OSError, RuntimeError): + pass # Likely SSL connection + + try: + self.writer.close() + await self.writer.wait_closed() + except (BrokenPipeError, ConnectionAbortedError, ConnectionResetError, RuntimeError): + pass # Already closed + + await self._stop_idle() + + async def _initiate_server_close(self) -> None: + await self.protocol.handle(Closed()) + self.writer.close() + + async def _start_idle(self) -> None: + async with self.idle_lock: + if self._idle_handle is None: + self._idle_handle = self.loop.create_task(self._run_idle()) + + async def _stop_idle(self) -> None: + async with self.idle_lock: + if self._idle_handle is not None: + self._idle_handle.cancel() + try: + await self._idle_handle + except asyncio.CancelledError: + pass + self._idle_handle = None + + async def _run_idle(self) -> None: + try: + await asyncio.wait_for(self.context.terminated.wait(), self.config.keep_alive_timeout) + except asyncio.TimeoutError: + pass + await asyncio.shield(self._initiate_server_close()) diff --git a/debian/vendor/hypercorn/asyncio/udp_server.py b/debian/vendor/hypercorn/asyncio/udp_server.py new file mode 100644 index 0000000..629ab9f --- /dev/null +++ b/debian/vendor/hypercorn/asyncio/udp_server.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import asyncio +from typing import Optional, Tuple, TYPE_CHECKING + +from .task_group import TaskGroup +from .worker_context import WorkerContext +from ..config import Config +from ..events import Event, RawData +from ..typing import AppWrapper +from ..utils import parse_socket_addr + +if TYPE_CHECKING: + # h3/Quic is an optional part of Hypercorn + from ..protocol.quic import QuicProtocol # noqa: F401 + + +class UDPServer(asyncio.DatagramProtocol): + def __init__( + self, + app: AppWrapper, + loop: asyncio.AbstractEventLoop, + config: Config, + context: WorkerContext, + ) -> None: + self.app = app + self.config = config + self.context = context + self.loop = loop + self.protocol: "QuicProtocol" + self.protocol_queue: asyncio.Queue = asyncio.Queue(10) + self.transport: Optional[asyncio.DatagramTransport] = None + + def connection_made(self, transport: asyncio.DatagramTransport) -> None: # type: ignore + self.transport = transport + + def datagram_received(self, data: bytes, address: Tuple[bytes, str]) -> None: # type: ignore + try: + self.protocol_queue.put_nowait(RawData(data=data, address=address)) # type: ignore + except asyncio.QueueFull: + pass # Just throw the data away, is UDP + + async def run(self) -> None: + # h3/Quic is an optional part of Hypercorn + from ..protocol.quic import QuicProtocol # noqa: F811 + + socket = self.transport.get_extra_info("socket") + server = parse_socket_addr(socket.family, socket.getsockname()) + async with TaskGroup(self.loop) as task_group: + self.protocol = QuicProtocol( + self.app, self.config, self.context, task_group, server, self.protocol_send + ) + + while not self.context.terminated.is_set() or not self.protocol.idle: + event = await self.protocol_queue.get() + await self.protocol.handle(event) + + async def protocol_send(self, event: Event) -> None: + if isinstance(event, RawData): + self.transport.sendto(event.data, event.address) diff --git a/debian/vendor/hypercorn/asyncio/worker_context.py b/debian/vendor/hypercorn/asyncio/worker_context.py new file mode 100644 index 0000000..fe9ad1c --- /dev/null +++ b/debian/vendor/hypercorn/asyncio/worker_context.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import asyncio +from typing import Type, Union + +from ..typing import Event + + +class EventWrapper: + def __init__(self) -> None: + self._event = asyncio.Event() + + async def clear(self) -> None: + self._event.clear() + + async def wait(self) -> None: + await self._event.wait() + + async def set(self) -> None: + self._event.set() + + def is_set(self) -> bool: + return self._event.is_set() + + +class WorkerContext: + event_class: Type[Event] = EventWrapper + + def __init__(self) -> None: + self.terminated = self.event_class() + + @staticmethod + async def sleep(wait: Union[float, int]) -> None: + return await asyncio.sleep(wait) + + @staticmethod + def time() -> float: + return asyncio.get_event_loop().time() diff --git a/debian/vendor/hypercorn/config.py b/debian/vendor/hypercorn/config.py new file mode 100644 index 0000000..26f50f0 --- /dev/null +++ b/debian/vendor/hypercorn/config.py @@ -0,0 +1,401 @@ +from __future__ import annotations + +import importlib +import importlib.util +import logging +import os +import socket +import stat +import sys +import types +import warnings +from dataclasses import dataclass +from ssl import ( + create_default_context, + OP_NO_COMPRESSION, + Purpose, + SSLContext, + TLSVersion, + VerifyFlags, + VerifyMode, +) +from time import time +from typing import Any, AnyStr, Dict, List, Mapping, Optional, Tuple, Type, Union +from wsgiref.handlers import format_date_time + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +from .logging import Logger + +BYTES = 1 +OCTETS = 1 +SECONDS = 1.0 + +FilePath = Union[AnyStr, os.PathLike] +SocketKind = Union[int, socket.SocketKind] + + +@dataclass +class Sockets: + secure_sockets: List[socket.socket] + insecure_sockets: List[socket.socket] + quic_sockets: List[socket.socket] + + +class SocketTypeError(Exception): + def __init__(self, expected: SocketKind, actual: SocketKind) -> None: + super().__init__( + f'Unexpected socket type, wanted "{socket.SocketKind(expected)}" got ' + f'"{socket.SocketKind(actual)}"' + ) + + +class Config: + _bind = ["127.0.0.1:8000"] + _insecure_bind: List[str] = [] + _quic_bind: List[str] = [] + _quic_addresses: List[Tuple] = [] + _log: Optional[Logger] = None + _root_path: str = "" + + access_log_format = '%(h)s %(l)s %(l)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + accesslog: Union[logging.Logger, str, None] = None + alpn_protocols = ["h2", "http/1.1"] + alt_svc_headers: List[str] = [] + application_path: str + backlog = 100 + ca_certs: Optional[str] = None + certfile: Optional[str] = None + ciphers: str = "ECDHE+AESGCM" + debug = False + dogstatsd_tags = "" + errorlog: Union[logging.Logger, str, None] = "-" + graceful_timeout: float = 3 * SECONDS + read_timeout: Optional[int] = None + group: Optional[int] = None + h11_max_incomplete_size = 16 * 1024 * BYTES + h11_pass_raw_headers = False + h2_max_concurrent_streams = 100 + h2_max_header_list_size = 2**16 + h2_max_inbound_frame_size = 2**14 * OCTETS + include_date_header = True + include_server_header = True + keep_alive_timeout = 5 * SECONDS + keyfile: Optional[str] = None + keyfile_password: Optional[str] = None + logconfig: Optional[str] = None + logconfig_dict: Optional[dict] = None + logger_class = Logger + loglevel: str = "INFO" + max_app_queue_size: int = 10 + pid_path: Optional[str] = None + server_names: List[str] = [] + shutdown_timeout = 60 * SECONDS + ssl_handshake_timeout = 60 * SECONDS + startup_timeout = 60 * SECONDS + statsd_host: Optional[str] = None + statsd_prefix = "" + umask: Optional[int] = None + use_reloader = False + user: Optional[int] = None + verify_flags: Optional[VerifyFlags] = None + verify_mode: Optional[VerifyMode] = None + websocket_max_message_size = 16 * 1024 * 1024 * BYTES + websocket_ping_interval: Optional[float] = None + worker_class = "asyncio" + workers = 1 + wsgi_max_body_size = 16 * 1024 * 1024 * BYTES + + def set_cert_reqs(self, value: int) -> None: + warnings.warn("Please use verify_mode instead", Warning) + self.verify_mode = VerifyMode(value) + + cert_reqs = property(None, set_cert_reqs) + + @property + def log(self) -> Logger: + if self._log is None: + self._log = self.logger_class(self) + return self._log + + @property + def bind(self) -> List[str]: + return self._bind + + @bind.setter + def bind(self, value: Union[List[str], str]) -> None: + if isinstance(value, str): + self._bind = [value] + else: + self._bind = value + + @property + def insecure_bind(self) -> List[str]: + return self._insecure_bind + + @insecure_bind.setter + def insecure_bind(self, value: Union[List[str], str]) -> None: + if isinstance(value, str): + self._insecure_bind = [value] + else: + self._insecure_bind = value + + @property + def quic_bind(self) -> List[str]: + return self._quic_bind + + @quic_bind.setter + def quic_bind(self, value: Union[List[str], str]) -> None: + if isinstance(value, str): + self._quic_bind = [value] + else: + self._quic_bind = value + + @property + def root_path(self) -> str: + return self._root_path + + @root_path.setter + def root_path(self, value: str) -> None: + self._root_path = value.rstrip("/") + + def create_ssl_context(self) -> Optional[SSLContext]: + if not self.ssl_enabled: + return None + + context = create_default_context(Purpose.CLIENT_AUTH) + context.set_ciphers(self.ciphers) + context.minimum_version = TLSVersion.TLSv1_2 # RFC 7540 Section 9.2: MUST be TLS >=1.2 + context.options = OP_NO_COMPRESSION # RFC 7540 Section 9.2.1: MUST disable compression + context.set_alpn_protocols(self.alpn_protocols) + + if self.certfile is not None and self.keyfile is not None: + context.load_cert_chain( + certfile=self.certfile, + keyfile=self.keyfile, + password=self.keyfile_password, + ) + + if self.ca_certs is not None: + context.load_verify_locations(self.ca_certs) + if self.verify_mode is not None: + context.verify_mode = self.verify_mode + if self.verify_flags is not None: + context.verify_flags = self.verify_flags + + return context + + @property + def ssl_enabled(self) -> bool: + return self.certfile is not None and self.keyfile is not None + + def create_sockets(self) -> Sockets: + if self.ssl_enabled: + secure_sockets = self._create_sockets(self.bind) + insecure_sockets = self._create_sockets(self.insecure_bind) + quic_sockets = self._create_sockets(self.quic_bind, socket.SOCK_DGRAM) + self._set_quic_addresses(quic_sockets) + else: + secure_sockets = [] + insecure_sockets = self._create_sockets(self.bind) + quic_sockets = [] + return Sockets(secure_sockets, insecure_sockets, quic_sockets) + + def _set_quic_addresses(self, sockets: List[socket.socket]) -> None: + self._quic_addresses = [] + for sock in sockets: + name = sock.getsockname() + if type(name) is not str and len(name) >= 2: + self._quic_addresses.append(name) + else: + warnings.warn( + f'Cannot create a alt-svc header for the QUIC socket with address "{name}"', + Warning, + ) + + def _create_sockets( + self, binds: List[str], type_: int = socket.SOCK_STREAM + ) -> List[socket.socket]: + sockets: List[socket.socket] = [] + for bind in binds: + binding: Any = None + if bind.startswith("unix:"): + sock = socket.socket(socket.AF_UNIX, type_) + binding = bind[5:] + try: + if stat.S_ISSOCK(os.stat(binding).st_mode): + os.remove(binding) + except FileNotFoundError: + pass + elif bind.startswith("fd://"): + sock = socket.socket(fileno=int(bind[5:])) + actual_type = sock.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE) + if actual_type != type_: + raise SocketTypeError(type_, actual_type) + else: + bind = bind.replace("[", "").replace("]", "") + try: + value = bind.rsplit(":", 1) + host, port = value[0], int(value[1]) + except (ValueError, IndexError): + host, port = bind, 8000 + sock = socket.socket(socket.AF_INET6 if ":" in host else socket.AF_INET, type_) + if self.workers > 1: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + except AttributeError: + pass + binding = (host, port) + + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + if bind.startswith("unix:"): + if self.umask is not None: + current_umask = os.umask(self.umask) + sock.bind(binding) + if self.user is not None and self.group is not None: + os.chown(binding, self.user, self.group) + if self.umask is not None: + os.umask(current_umask) + elif bind.startswith("fd://"): + pass + else: + sock.bind(binding) + + sock.setblocking(False) + try: + sock.set_inheritable(True) + except AttributeError: + pass + sockets.append(sock) + return sockets + + def response_headers(self, protocol: str) -> List[Tuple[bytes, bytes]]: + headers = [] + if self.include_date_header: + headers.append((b"date", format_date_time(time()).encode("ascii"))) + if self.include_server_header: + headers.append((b"server", f"hypercorn-{protocol}".encode("ascii"))) + + for alt_svc_header in self.alt_svc_headers: + headers.append((b"alt-svc", alt_svc_header.encode())) + if len(self.alt_svc_headers) == 0 and self._quic_addresses: + from aioquic.h3.connection import H3_ALPN + + for version in H3_ALPN: + for addr in self._quic_addresses: + port = addr[1] + headers.append((b"alt-svc", b'%s=":%d"; ma=3600' % (version.encode(), port))) + + return headers + + def set_statsd_logger_class(self, statsd_logger: Type[Logger]) -> None: + if self.logger_class == Logger and self.statsd_host is not None: + self.logger_class = statsd_logger + + @classmethod + def from_mapping( + cls: Type["Config"], mapping: Optional[Mapping[str, Any]] = None, **kwargs: Any + ) -> "Config": + """Create a configuration from a mapping. + + This allows either a mapping to be directly passed or as + keyword arguments, for example, + + .. code-block:: python + + config = {'keep_alive_timeout': 10} + Config.from_mapping(config) + Config.from_mapping(keep_alive_timeout=10) + + Arguments: + mapping: Optionally a mapping object. + kwargs: Optionally a collection of keyword arguments to + form a mapping. + """ + mappings: Dict[str, Any] = {} + if mapping is not None: + mappings.update(mapping) + mappings.update(kwargs) + config = cls() + for key, value in mappings.items(): + try: + setattr(config, key, value) + except AttributeError: + pass + + return config + + @classmethod + def from_pyfile(cls: Type["Config"], filename: FilePath) -> "Config": + """Create a configuration from a Python file. + + .. code-block:: python + + Config.from_pyfile('hypercorn_config.py') + + Arguments: + filename: The filename which gives the path to the file. + """ + file_path = os.fspath(filename) + spec = importlib.util.spec_from_file_location("module.name", file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return cls.from_object(module) + + @classmethod + def from_toml(cls: Type["Config"], filename: FilePath) -> "Config": + """Load the configuration values from a TOML formatted file. + + This allows configuration to be loaded as so + + .. code-block:: python + + Config.from_toml('config.toml') + + Arguments: + filename: The filename which gives the path to the file. + """ + file_path = os.fspath(filename) + with open(file_path, "rb") as file_: + data = tomllib.load(file_) + return cls.from_mapping(data) + + @classmethod + def from_object(cls: Type["Config"], instance: Union[object, str]) -> "Config": + """Create a configuration from a Python object. + + This can be used to reference modules or objects within + modules for example, + + .. code-block:: python + + Config.from_object('module') + Config.from_object('module.instance') + from module import instance + Config.from_object(instance) + + are valid. + + Arguments: + instance: Either a str referencing a python object or the + object itself. + + """ + if isinstance(instance, str): + try: + instance = importlib.import_module(instance) + except ImportError: + path, config = instance.rsplit(".", 1) + module = importlib.import_module(path) + instance = getattr(module, config) + + mapping = { + key: getattr(instance, key) + for key in dir(instance) + if not isinstance(getattr(instance, key), types.ModuleType) and not key.startswith("__") + } + return cls.from_mapping(mapping) diff --git a/debian/vendor/hypercorn/events.py b/debian/vendor/hypercorn/events.py new file mode 100644 index 0000000..e829616 --- /dev/null +++ b/debian/vendor/hypercorn/events.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from abc import ABC +from dataclasses import dataclass +from typing import Optional, Tuple + + +class Event(ABC): + pass + + +@dataclass(frozen=True) +class RawData(Event): + data: bytes + address: Optional[Tuple[str, int]] = None + + +@dataclass(frozen=True) +class Closed(Event): + pass + + +@dataclass(frozen=True) +class Updated(Event): + idle: bool diff --git a/debian/vendor/hypercorn/logging.py b/debian/vendor/hypercorn/logging.py new file mode 100644 index 0000000..d9b8901 --- /dev/null +++ b/debian/vendor/hypercorn/logging.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import json +import logging +import os +import sys +import time +from http import HTTPStatus +from logging.config import dictConfig, fileConfig +from typing import Any, IO, Mapping, Optional, TYPE_CHECKING, Union + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + + +if TYPE_CHECKING: + from .config import Config + from .typing import ResponseSummary, WWWScope + + +def _create_logger( + name: str, + target: Union[logging.Logger, str, None], + level: Optional[str], + sys_default: IO, + *, + propagate: bool = True, +) -> Optional[logging.Logger]: + if isinstance(target, logging.Logger): + return target + + if target: + logger = logging.getLogger(name) + logger.handlers = [ + logging.StreamHandler(sys_default) if target == "-" else logging.FileHandler(target) # type: ignore # noqa: E501 + ] + logger.propagate = propagate + formatter = logging.Formatter( + "%(asctime)s [%(process)d] [%(levelname)s] %(message)s", + "[%Y-%m-%d %H:%M:%S %z]", + ) + logger.handlers[0].setFormatter(formatter) + if level is not None: + logger.setLevel(logging.getLevelName(level.upper())) + return logger + else: + return None + + +class Logger: + def __init__(self, config: "Config") -> None: + self.access_log_format = config.access_log_format + + self.access_logger = _create_logger( + "hypercorn.access", + config.accesslog, + config.loglevel, + sys.stdout, + propagate=False, + ) + self.error_logger = _create_logger( + "hypercorn.error", config.errorlog, config.loglevel, sys.stderr + ) + + if config.logconfig is not None: + if config.logconfig.startswith("json:"): + with open(config.logconfig[5:]) as file_: + dictConfig(json.load(file_)) + elif config.logconfig.startswith("toml:"): + with open(config.logconfig[5:], "rb") as file_: + dictConfig(tomllib.load(file_)) + else: + log_config = { + "__file__": config.logconfig, + "here": os.path.dirname(config.logconfig), + } + fileConfig(config.logconfig, defaults=log_config, disable_existing_loggers=False) + else: + if config.logconfig_dict is not None: + dictConfig(config.logconfig_dict) + + async def access( + self, request: "WWWScope", response: "ResponseSummary", request_time: float + ) -> None: + if self.access_logger is not None: + self.access_logger.info( + self.access_log_format, self.atoms(request, response, request_time) + ) + + async def critical(self, message: str, *args: Any, **kwargs: Any) -> None: + if self.error_logger is not None: + self.error_logger.critical(message, *args, **kwargs) + + async def error(self, message: str, *args: Any, **kwargs: Any) -> None: + if self.error_logger is not None: + self.error_logger.error(message, *args, **kwargs) + + async def warning(self, message: str, *args: Any, **kwargs: Any) -> None: + if self.error_logger is not None: + self.error_logger.warning(message, *args, **kwargs) + + async def info(self, message: str, *args: Any, **kwargs: Any) -> None: + if self.error_logger is not None: + self.error_logger.info(message, *args, **kwargs) + + async def debug(self, message: str, *args: Any, **kwargs: Any) -> None: + if self.error_logger is not None: + self.error_logger.debug(message, *args, **kwargs) + + async def exception(self, message: str, *args: Any, **kwargs: Any) -> None: + if self.error_logger is not None: + self.error_logger.exception(message, *args, **kwargs) + + async def log(self, level: int, message: str, *args: Any, **kwargs: Any) -> None: + if self.error_logger is not None: + self.error_logger.log(level, message, *args, **kwargs) + + def atoms( + self, request: "WWWScope", response: Optional["ResponseSummary"], request_time: float + ) -> Mapping[str, str]: + """Create and return an access log atoms dictionary. + + This can be overidden and customised if desired. It should + return a mapping between an access log format key and a value. + """ + return AccessLogAtoms(request, response, request_time) + + def __getattr__(self, name: str) -> Any: + return getattr(self.error_logger, name) + + +class AccessLogAtoms(dict): + def __init__( + self, request: "WWWScope", response: Optional["ResponseSummary"], request_time: float + ) -> None: + for name, value in request["headers"]: + self[f"{{{name.decode('latin1').lower()}}}i"] = value.decode("latin1") + for name, value in os.environ.items(): + self[f"{{{name.lower()}}}e"] = value + protocol = request.get("http_version", "ws") + client = request.get("client") + if client is None: + remote_addr = None + elif len(client) == 2: + remote_addr = f"{client[0]}:{client[1]}" + elif len(client) == 1: + remote_addr = client[0] + else: # make sure not to throw UnboundLocalError + remote_addr = f"" + if request["type"] == "http": + method = request["method"] + else: + method = "GET" + query_string = request["query_string"].decode() + path_with_qs = request["path"] + ("?" + query_string if query_string else "") + + status_code = "-" + status_phrase = "-" + if response is not None: + for name, value in response.get("headers", []): # type: ignore + self[f"{{{name.decode('latin1').lower()}}}o"] = value.decode("latin1") # type: ignore # noqa: E501 + status_code = str(response["status"]) + try: + status_phrase = HTTPStatus(response["status"]).phrase + except ValueError: + status_phrase = f"" + self.update( + { + "h": remote_addr, + "l": "-", + "t": time.strftime("[%d/%b/%Y:%H:%M:%S %z]"), + "r": f"{method} {request['path']} {protocol}", + "R": f"{method} {path_with_qs} {protocol}", + "s": status_code, + "st": status_phrase, + "S": request["scheme"], + "m": method, + "U": request["path"], + "Uq": path_with_qs, + "q": query_string, + "H": protocol, + "b": self["{Content-Length}o"], + "B": self["{Content-Length}o"], + "f": self["{Referer}i"], + "a": self["{User-Agent}i"], + "T": int(request_time), + "D": int(request_time * 1_000_000), + "L": f"{request_time:.6f}", + "p": f"<{os.getpid()}>", + } + ) + + def __getitem__(self, key: str) -> str: + try: + if key.startswith("{"): + return super().__getitem__(key.lower()) + else: + return super().__getitem__(key) + except KeyError: + return "-" diff --git a/debian/vendor/hypercorn/middleware/__init__.py b/debian/vendor/hypercorn/middleware/__init__.py new file mode 100644 index 0000000..83ea29c --- /dev/null +++ b/debian/vendor/hypercorn/middleware/__init__.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from .dispatcher import DispatcherMiddleware +from .http_to_https import HTTPToHTTPSRedirectMiddleware +from .wsgi import AsyncioWSGIMiddleware, TrioWSGIMiddleware + +__all__ = ( + "AsyncioWSGIMiddleware", + "DispatcherMiddleware", + "HTTPToHTTPSRedirectMiddleware", + "TrioWSGIMiddleware", +) diff --git a/debian/vendor/hypercorn/middleware/dispatcher.py b/debian/vendor/hypercorn/middleware/dispatcher.py new file mode 100644 index 0000000..009541b --- /dev/null +++ b/debian/vendor/hypercorn/middleware/dispatcher.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import asyncio +from functools import partial +from typing import Callable, Dict + +from ..asyncio.task_group import TaskGroup +from ..typing import ASGIFramework, Scope + +MAX_QUEUE_SIZE = 10 + + +class _DispatcherMiddleware: + def __init__(self, mounts: Dict[str, ASGIFramework]) -> None: + self.mounts = mounts + + async def __call__(self, scope: Scope, receive: Callable, send: Callable) -> None: + if scope["type"] == "lifespan": + await self._handle_lifespan(scope, receive, send) + else: + for path, app in self.mounts.items(): + if scope["path"].startswith(path): + scope["path"] = scope["path"][len(path) :] or "/" + return await app(scope, receive, send) + await send( + { + "type": "http.response.start", + "status": 404, + "headers": [(b"content-length", b"0")], + } + ) + await send({"type": "http.response.body"}) + + async def _handle_lifespan(self, scope: Scope, receive: Callable, send: Callable) -> None: + pass + + +class AsyncioDispatcherMiddleware(_DispatcherMiddleware): + async def _handle_lifespan(self, scope: Scope, receive: Callable, send: Callable) -> None: + self.app_queues: Dict[str, asyncio.Queue] = { + path: asyncio.Queue(MAX_QUEUE_SIZE) for path in self.mounts + } + self.startup_complete = {path: False for path in self.mounts} + self.shutdown_complete = {path: False for path in self.mounts} + + async with TaskGroup(asyncio.get_event_loop()) as task_group: + for path, app in self.mounts.items(): + task_group.spawn( + app, + scope, + self.app_queues[path].get, + partial(self.send, path, send), + ) + + while True: + message = await receive() + for queue in self.app_queues.values(): + await queue.put(message) + if message["type"] == "lifespan.shutdown": + break + + async def send(self, path: str, send: Callable, message: dict) -> None: + if message["type"] == "lifespan.startup.complete": + self.startup_complete[path] = True + if all(self.startup_complete.values()): + await send({"type": "lifespan.startup.complete"}) + elif message["type"] == "lifespan.shutdown.complete": + self.shutdown_complete[path] = True + if all(self.shutdown_complete.values()): + await send({"type": "lifespan.shutdown.complete"}) + + +class TrioDispatcherMiddleware(_DispatcherMiddleware): + async def _handle_lifespan(self, scope: Scope, receive: Callable, send: Callable) -> None: + import trio + + self.app_queues = {path: trio.open_memory_channel(MAX_QUEUE_SIZE) for path in self.mounts} + self.startup_complete = {path: False for path in self.mounts} + self.shutdown_complete = {path: False for path in self.mounts} + + async with trio.open_nursery() as nursery: + for path, app in self.mounts.items(): + nursery.start_soon( + app, + scope, + self.app_queues[path][1].receive, + partial(self.send, path, send), + ) + + while True: + message = await receive() + for channels in self.app_queues.values(): + await channels[0].send(message) + if message["type"] == "lifespan.shutdown": + break + + async def send(self, path: str, send: Callable, message: dict) -> None: + if message["type"] == "lifespan.startup.complete": + self.startup_complete[path] = True + if all(self.startup_complete.values()): + await send({"type": "lifespan.startup.complete"}) + elif message["type"] == "lifespan.shutdown.complete": + self.shutdown_complete[path] = True + if all(self.shutdown_complete.values()): + await send({"type": "lifespan.shutdown.complete"}) + + +DispatcherMiddleware = AsyncioDispatcherMiddleware # Remove with version 0.11 diff --git a/debian/vendor/hypercorn/middleware/http_to_https.py b/debian/vendor/hypercorn/middleware/http_to_https.py new file mode 100644 index 0000000..542b28f --- /dev/null +++ b/debian/vendor/hypercorn/middleware/http_to_https.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from typing import Callable, Optional +from urllib.parse import urlunsplit + +from ..typing import ASGIFramework, HTTPScope, Scope, WebsocketScope, WWWScope + + +class HTTPToHTTPSRedirectMiddleware: + def __init__(self, app: ASGIFramework, host: Optional[str]) -> None: + self.app = app + self.host = host + + async def __call__(self, scope: Scope, receive: Callable, send: Callable) -> None: + if scope["type"] == "http" and scope["scheme"] == "http": + await self._send_http_redirect(scope, send) + elif scope["type"] == "websocket" and scope["scheme"] == "ws": + # If the server supports the WebSocket Denial Response + # extension we can send a redirection response, if not we + # can only deny the WebSocket connection. + if "websocket.http.response" in scope.get("extensions", {}): + await self._send_websocket_redirect(scope, send) + else: + await send({"type": "websocket.close"}) + else: + return await self.app(scope, receive, send) + + async def _send_http_redirect(self, scope: HTTPScope, send: Callable) -> None: + new_url = self._new_url("https", scope) + await send( + { + "type": "http.response.start", + "status": 307, + "headers": [(b"location", new_url.encode())], + } + ) + await send({"type": "http.response.body"}) + + async def _send_websocket_redirect(self, scope: WebsocketScope, send: Callable) -> None: + # If the HTTP version is 2 we should redirect with a https + # scheme not wss. + scheme = "wss" + if scope.get("http_version", "1.1") == "2": + scheme = "https" + + new_url = self._new_url(scheme, scope) + await send( + { + "type": "websocket.http.response.start", + "status": 307, + "headers": [(b"location", new_url.encode())], + } + ) + await send({"type": "websocket.http.response.body"}) + + def _new_url(self, scheme: str, scope: WWWScope) -> str: + host = self.host + if host is None: + for key, value in scope["headers"]: + if key == b"host": + host = value.decode("latin-1") + break + if host is None: + raise ValueError("Host to redirect to cannot be determined") + + path = scope.get("root_path", "") + scope["raw_path"].decode() + return urlunsplit((scheme, host, path, scope["query_string"].decode(), "")) diff --git a/debian/vendor/hypercorn/middleware/wsgi.py b/debian/vendor/hypercorn/middleware/wsgi.py new file mode 100644 index 0000000..8e4f61b --- /dev/null +++ b/debian/vendor/hypercorn/middleware/wsgi.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import asyncio +from functools import partial +from typing import Any, Callable, Iterable + +from ..app_wrappers import WSGIWrapper +from ..typing import ASGIReceiveCallable, ASGISendCallable, Scope, WSGIFramework + +MAX_BODY_SIZE = 2**16 + +WSGICallable = Callable[[dict, Callable], Iterable[bytes]] + + +class InvalidPathError(Exception): + pass + + +class _WSGIMiddleware: + def __init__(self, wsgi_app: WSGIFramework, max_body_size: int = MAX_BODY_SIZE) -> None: + self.wsgi_app = WSGIWrapper(wsgi_app, max_body_size) + self.max_body_size = max_body_size + + async def __call__( + self, scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable + ) -> None: + pass + + +class AsyncioWSGIMiddleware(_WSGIMiddleware): + async def __call__( + self, scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable + ) -> None: + loop = asyncio.get_event_loop() + + def _call_soon(func: Callable, *args: Any) -> Any: + future = asyncio.run_coroutine_threadsafe(func(*args), loop) + return future.result() + + await self.wsgi_app(scope, receive, send, partial(loop.run_in_executor, None), _call_soon) + + +class TrioWSGIMiddleware(_WSGIMiddleware): + async def __call__( + self, scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable + ) -> None: + import trio + + await self.wsgi_app(scope, receive, send, trio.to_thread.run_sync, trio.from_thread.run) diff --git a/debian/vendor/hypercorn/protocol/__init__.py b/debian/vendor/hypercorn/protocol/__init__.py new file mode 100755 index 0000000..e047da9 --- /dev/null +++ b/debian/vendor/hypercorn/protocol/__init__.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import Any, Awaitable, Callable, Optional, Tuple, Union + +from .h2 import H2Protocol +from .h11 import H2CProtocolRequiredError, H2ProtocolAssumedError, H11Protocol +from ..config import Config +from ..events import Event, RawData +from ..typing import AppWrapper, TaskGroup, WorkerContext + + +class ProtocolWrapper: + def __init__( + self, + app: AppWrapper, + config: Config, + context: WorkerContext, + task_group: TaskGroup, + tls: Optional[dict[str, Any]], + client: Optional[Tuple[str, int]], + server: Optional[Tuple[str, int]], + send: Callable[[Event], Awaitable[None]], + alpn_protocol: Optional[str] = None, + transport=None, + ) -> None: + self.app = app + self.config = config + self.context = context + self.task_group = task_group + self.tls = tls + self.client = client + self.server = server + self.send = send + self.protocol: Union[H11Protocol, H2Protocol] + self.transport = transport + if alpn_protocol == "h2": + self.protocol = H2Protocol( + self.app, + self.config, + self.context, + self.task_group, + self.tls, + self.client, + self.server, + self.send, + self.transport, + ) + else: + self.protocol = H11Protocol( + self.app, + self.config, + self.context, + self.task_group, + self.tls, + self.client, + self.server, + self.send, + self.transport, + ) + + async def initiate(self) -> None: + return await self.protocol.initiate() + + async def handle(self, event: Event) -> None: + try: + return await self.protocol.handle(event) + except H2ProtocolAssumedError as error: + self.protocol = H2Protocol( + self.app, + self.config, + self.context, + self.task_group, + self.tls, + self.client, + self.server, + self.send, + ) + await self.protocol.initiate() + if error.data != b"": + return await self.protocol.handle(RawData(data=error.data)) + except H2CProtocolRequiredError as error: + self.protocol = H2Protocol( + self.app, + self.config, + self.context, + self.task_group, + self.tls, + self.client, + self.server, + self.send, + ) + await self.protocol.initiate(error.headers, error.settings) + if error.data != b"": + return await self.protocol.handle(RawData(data=error.data)) diff --git a/debian/vendor/hypercorn/protocol/events.py b/debian/vendor/hypercorn/protocol/events.py new file mode 100644 index 0000000..d91d203 --- /dev/null +++ b/debian/vendor/hypercorn/protocol/events.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Tuple + + +@dataclass(frozen=True) +class Event: + stream_id: int + + +@dataclass(frozen=True) +class Request(Event): + headers: List[Tuple[bytes, bytes]] + http_version: str + method: str + raw_path: bytes + + +@dataclass(frozen=True) +class Body(Event): + data: bytes + + +@dataclass(frozen=True) +class EndBody(Event): + pass + + +@dataclass(frozen=True) +class Data(Event): + data: bytes + + +@dataclass(frozen=True) +class EndData(Event): + pass + + +@dataclass(frozen=True) +class Response(Event): + headers: List[Tuple[bytes, bytes]] + status_code: int + + +@dataclass(frozen=True) +class InformationalResponse(Event): + headers: List[Tuple[bytes, bytes]] + status_code: int + + def __post_init__(self) -> None: + if self.status_code >= 200 or self.status_code < 100: + raise ValueError(f"Status code must be 1XX not {self.status_code}") + + +@dataclass(frozen=True) +class StreamClosed(Event): + pass diff --git a/debian/vendor/hypercorn/protocol/h11.py b/debian/vendor/hypercorn/protocol/h11.py new file mode 100755 index 0000000..9d8bb04 --- /dev/null +++ b/debian/vendor/hypercorn/protocol/h11.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +from itertools import chain +from typing import Any, Awaitable, Callable, cast, Optional, Tuple, Type, Union + +import h11 + +from .events import ( + Body, + Data, + EndBody, + EndData, + Event as StreamEvent, + InformationalResponse, + Request, + Response, + StreamClosed, +) +from .http_stream import HTTPStream +from .ws_stream import WSStream +from ..config import Config +from ..events import Closed, Event, RawData, Updated +from ..typing import AppWrapper, H11SendableEvent, TaskGroup, WorkerContext + +STREAM_ID = 1 + + +class H2CProtocolRequiredError(Exception): + def __init__(self, data: bytes, request: h11.Request) -> None: + settings = "" + headers = [(b":method", request.method), (b":path", request.target)] + for name, value in request.headers: + if name.lower() == b"http2-settings": + settings = value.decode() + elif name.lower() == b"host": + headers.append((b":authority", value)) + headers.append((name, value)) + + self.data = data + self.headers = headers + self.settings = settings + + +class H2ProtocolAssumedError(Exception): + def __init__(self, data: bytes) -> None: + self.data = data + + +class H11WSConnection: + # This class matches the h11 interface, and either passes data + # through without altering it (for Data, EndData) or sends h11 + # events (Response, Body, EndBody). + our_state = None # Prevents recycling the connection + they_are_waiting_for_100_continue = False + their_state = None + trailing_data = (b"", False) + + def __init__(self, h11_connection: h11.Connection) -> None: + self.buffer = bytearray(h11_connection.trailing_data[0]) + self.h11_connection = h11_connection + + def receive_data(self, data: bytes) -> None: + self.buffer.extend(data) + + def next_event(self) -> Union[Data, Type[h11.NEED_DATA]]: + if self.buffer: + event = Data(stream_id=STREAM_ID, data=bytes(self.buffer)) + self.buffer = bytearray() + return event + else: + return h11.NEED_DATA + + def send(self, event: H11SendableEvent) -> bytes: + return self.h11_connection.send(event) + + def start_next_cycle(self) -> None: + pass + + +class H11Protocol: + def __init__( + self, + app: AppWrapper, + config: Config, + context: WorkerContext, + task_group: TaskGroup, + tls: Optional[dict[str, Any]], + client: Optional[Tuple[str, int]], + server: Optional[Tuple[str, int]], + send: Callable[[Event], Awaitable[None]], + transport=None, + ) -> None: + self.app = app + self.can_read = context.event_class() + self.client = client + self.config = config + self.connection: Union[h11.Connection, H11WSConnection] = h11.Connection( + h11.SERVER, max_incomplete_event_size=self.config.h11_max_incomplete_size + ) + self.context = context + self.send = send + self.server = server + self.tls = tls + self.stream: Optional[Union[HTTPStream, WSStream]] = None + self.task_group = task_group + self.transport = transport + + async def initiate(self) -> None: + pass + + async def handle(self, event: Event) -> None: + if isinstance(event, RawData): + self.connection.receive_data(event.data) + await self._handle_events() + elif isinstance(event, Closed): + if self.stream is not None: + await self._close_stream() + + async def stream_send(self, event: StreamEvent) -> None: + if isinstance(event, Response): + if event.status_code >= 200: + await self._send_h11_event( + h11.Response( + headers=list(chain(event.headers, self.config.response_headers("h11"))), + status_code=event.status_code, + ) + ) + else: + await self._send_h11_event( + h11.InformationalResponse( + headers=list(chain(event.headers, self.config.response_headers("h11"))), + status_code=event.status_code, + ) + ) + elif isinstance(event, InformationalResponse): + pass # Ignore for HTTP/1 + elif isinstance(event, Body): + await self._send_h11_event(h11.Data(data=event.data)) + elif isinstance(event, EndBody): + await self._send_h11_event(h11.EndOfMessage()) + elif isinstance(event, Data): + await self.send(RawData(data=event.data)) + elif isinstance(event, EndData): + pass + elif isinstance(event, StreamClosed): + await self._maybe_recycle() + + async def _handle_events(self) -> None: + while True: + if self.connection.they_are_waiting_for_100_continue: + await self._send_h11_event( + h11.InformationalResponse( + status_code=100, headers=self.config.response_headers("h11") + ) + ) + + try: + event = self.connection.next_event() + except h11.RemoteProtocolError: + if self.connection.our_state in {h11.IDLE, h11.SEND_RESPONSE}: + await self._send_error_response(400) + await self.send(Closed()) + raise + break + else: + if isinstance(event, h11.Request): + await self.send(Updated(idle=False)) + await self._check_protocol(event) + await self._create_stream(event) + elif event is h11.PAUSED: + await self.can_read.clear() + await self.can_read.wait() + elif isinstance(event, h11.ConnectionClosed) or event is h11.NEED_DATA: + break + elif self.stream is None: + break + elif isinstance(event, h11.Data): + await self.stream.handle(Body(stream_id=STREAM_ID, data=event.data)) + elif isinstance(event, h11.EndOfMessage): + await self.stream.handle(EndBody(stream_id=STREAM_ID)) + elif isinstance(event, Data): + # WebSocket pass through + await self.stream.handle(event) + + async def _create_stream(self, request: h11.Request) -> None: + upgrade_value = "" + connection_value = "" + for name, value in request.headers: + sanitised_name = name.decode("latin1").strip().lower() + if sanitised_name == "upgrade": + upgrade_value = value.decode("latin1").strip() + elif sanitised_name == "connection": + connection_value = value.decode("latin1").strip() + + connection_tokens = connection_value.lower().split(",") + if ( + any(token.strip() == "upgrade" for token in connection_tokens) + and upgrade_value.lower() == "websocket" + and request.method.decode("ascii").upper() == "GET" + ): + self.stream = WSStream( + self.app, + self.config, + self.context, + self.task_group, + self.tls, + self.client, + self.server, + self.stream_send, + STREAM_ID, + ) + self.connection = H11WSConnection(cast(h11.Connection, self.connection)) + else: + self.stream = HTTPStream( + self.app, + self.config, + self.context, + self.task_group, + self.tls, + self.client, + self.server, + self.stream_send, + STREAM_ID, + self.transport, + ) + + if self.config.h11_pass_raw_headers: + headers = request.headers.raw_items() + else: + headers = list(request.headers) + + await self.stream.handle( + Request( + stream_id=STREAM_ID, + headers=headers, + http_version=request.http_version.decode(), + method=request.method.decode("ascii").upper(), + raw_path=request.target, + ) + ) + + async def _send_h11_event(self, event: H11SendableEvent) -> None: + try: + data = self.connection.send(event) + except h11.LocalProtocolError: + if self.connection.their_state != h11.ERROR: + raise + else: + await self.send(RawData(data=data)) + + async def _send_error_response(self, status_code: int) -> None: + await self._send_h11_event( + h11.Response( + status_code=status_code, + headers=list( + chain( + [(b"content-length", b"0"), (b"connection", b"close")], + self.config.response_headers("h11"), + ) + ), + ) + ) + await self._send_h11_event(h11.EndOfMessage()) + + async def _maybe_recycle(self) -> None: + await self._close_stream() + if ( + not self.context.terminated.is_set() + and self.connection.our_state is h11.DONE + and self.connection.their_state is h11.DONE + ): + try: + self.connection.start_next_cycle() + except h11.LocalProtocolError: + await self.send(Closed()) + else: + self.response = None + self.scope = None + await self.can_read.set() + await self.send(Updated(idle=True)) + else: + await self.can_read.set() + await self.send(Closed()) + + async def _close_stream(self) -> None: + if self.stream is not None: + await self.stream.handle(StreamClosed(stream_id=STREAM_ID)) + self.stream = None + + async def _check_protocol(self, event: h11.Request) -> None: + upgrade_value = "" + has_body = False + for name, value in event.headers: + sanitised_name = name.decode("latin1").strip().lower() + if sanitised_name == "upgrade": + upgrade_value = value.decode("latin1").strip() + elif sanitised_name in {"content-length", "transfer-encoding"}: + has_body = True + + # h2c Upgrade requests with a body are a pain as the body must + # be fully recieved in HTTP/1.1 before the upgrade response + # and HTTP/2 takes over, so Hypercorn ignores the upgrade and + # responds in HTTP/1.1. Use a preflight OPTIONS request to + # initiate the upgrade if really required (or just use h2). + if upgrade_value.lower() == "h2c" and not has_body: + await self._send_h11_event( + h11.InformationalResponse( + status_code=101, + headers=self.config.response_headers("h11") + + [(b"connection", b"upgrade"), (b"upgrade", b"h2c")], + ) + ) + raise H2CProtocolRequiredError(self.connection.trailing_data[0], event) + elif event.method == b"PRI" and event.target == b"*" and event.http_version == b"2.0": + raise H2ProtocolAssumedError( + b"PRI * HTTP/2.0\r\n\r\n" + self.connection.trailing_data[0] + ) diff --git a/debian/vendor/hypercorn/protocol/h2.py b/debian/vendor/hypercorn/protocol/h2.py new file mode 100755 index 0000000..d5451fd --- /dev/null +++ b/debian/vendor/hypercorn/protocol/h2.py @@ -0,0 +1,385 @@ +from __future__ import annotations + +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, Type, Union + +import h2 +import h2.connection +import h2.events +import h2.exceptions +import priority + +from .events import ( + Body, + Data, + EndBody, + EndData, + Event as StreamEvent, + InformationalResponse, + Request, + Response, + StreamClosed, +) +from .http_stream import HTTPStream +from .ws_stream import WSStream +from ..config import Config +from ..events import Closed, Event, RawData, Updated +from ..typing import AppWrapper, Event as IOEvent, TaskGroup, WorkerContext +from ..utils import filter_pseudo_headers + +BUFFER_HIGH_WATER = 2 * 2**14 # Twice the default max frame size (two frames worth) +BUFFER_LOW_WATER = BUFFER_HIGH_WATER / 2 + + +class BufferCompleteError(Exception): + pass + + +class StreamBuffer: + def __init__(self, event_class: Type[IOEvent]) -> None: + self.buffer = bytearray() + self._complete = False + self._is_empty = event_class() + self._paused = event_class() + + async def drain(self) -> None: + await self._is_empty.wait() + + def set_complete(self) -> None: + self._complete = True + + async def close(self) -> None: + self._complete = True + self.buffer = bytearray() + await self._is_empty.set() + await self._paused.set() + + @property + def complete(self) -> bool: + return self._complete and len(self.buffer) == 0 + + async def push(self, data: bytes) -> None: + if self._complete: + raise BufferCompleteError() + self.buffer.extend(data) + await self._is_empty.clear() + if len(self.buffer) >= BUFFER_HIGH_WATER: + await self._paused.wait() + await self._paused.clear() + + async def pop(self, max_length: int) -> bytes: + length = min(len(self.buffer), max_length) + data = bytes(self.buffer[:length]) + del self.buffer[:length] + if len(data) < BUFFER_LOW_WATER: + await self._paused.set() + if len(self.buffer) == 0: + await self._is_empty.set() + return data + + +class H2Protocol: + def __init__( + self, + app: AppWrapper, + config: Config, + context: WorkerContext, + task_group: TaskGroup, + tls: Optional[dict[str, Any]], + client: Optional[Tuple[str, int]], + server: Optional[Tuple[str, int]], + send: Callable[[Event], Awaitable[None]], + transport=None, + ) -> None: + self.app = app + self.client = client + self.closed = False + self.config = config + self.context = context + self.task_group = task_group + + self.connection = h2.connection.H2Connection( + config=h2.config.H2Configuration(client_side=False, header_encoding=None) + ) + self.connection.DEFAULT_MAX_INBOUND_FRAME_SIZE = config.h2_max_inbound_frame_size + self.connection.local_settings = h2.settings.Settings( + client=False, + initial_values={ + h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS: config.h2_max_concurrent_streams, + h2.settings.SettingCodes.MAX_HEADER_LIST_SIZE: config.h2_max_header_list_size, + h2.settings.SettingCodes.ENABLE_CONNECT_PROTOCOL: 1, + }, + ) + + self.send = send + self.server = server + self.tls = tls + self.streams: Dict[int, Union[HTTPStream, WSStream]] = {} + # The below are used by the sending task + self.has_data = self.context.event_class() + self.priority = priority.PriorityTree() + self.stream_buffers: Dict[int, StreamBuffer] = {} + self.transport = transport + + @property + def idle(self) -> bool: + return len(self.streams) == 0 or all(stream.idle for stream in self.streams.values()) + + async def initiate( + self, headers: Optional[List[Tuple[bytes, bytes]]] = None, settings: Optional[str] = None + ) -> None: + if settings is not None: + self.connection.initiate_upgrade_connection(settings) + else: + self.connection.initiate_connection() + await self._flush() + if headers is not None: + event = h2.events.RequestReceived() + event.stream_id = 1 + event.headers = headers + await self._create_stream(event) + await self.streams[event.stream_id].handle(EndBody(stream_id=event.stream_id)) + self.task_group.spawn(self.send_task) + + async def send_task(self) -> None: + # This should be run in a seperate task to the rest of this + # class. This allows it seperately choose when to send, + # crucially in what order. + while not self.closed: + try: + stream_id = next(self.priority) + except priority.DeadlockError: + await self.has_data.wait() + await self.has_data.clear() + else: + await self._send_data(stream_id) + + async def _send_data(self, stream_id: int) -> None: + try: + chunk_size = min( + self.connection.local_flow_control_window(stream_id), + self.connection.max_outbound_frame_size, + ) + chunk_size = max(0, chunk_size) + data = await self.stream_buffers[stream_id].pop(chunk_size) + if data: + self.connection.send_data(stream_id, data) + await self._flush() + else: + self.priority.block(stream_id) + + if self.stream_buffers[stream_id].complete: + self.connection.end_stream(stream_id) + await self._flush() + del self.stream_buffers[stream_id] + self.priority.remove_stream(stream_id) + except (h2.exceptions.StreamClosedError, KeyError, h2.exceptions.ProtocolError): + # Stream or connection has closed whilst waiting to send + # data, not a problem - just force close it. + await self.stream_buffers[stream_id].close() + del self.stream_buffers[stream_id] + self.priority.remove_stream(stream_id) + + async def handle(self, event: Event) -> None: + if isinstance(event, RawData): + try: + events = self.connection.receive_data(event.data) + except h2.exceptions.ProtocolError: + await self._flush() + await self.send(Closed()) + raise + else: + await self._handle_events(events) + elif isinstance(event, Closed): + self.closed = True + stream_ids = list(self.streams.keys()) + for stream_id in stream_ids: + await self._close_stream(stream_id) + await self.has_data.set() + + async def stream_send(self, event: StreamEvent) -> None: + try: + if isinstance(event, (InformationalResponse, Response)): + self.connection.send_headers( + event.stream_id, + [(b":status", b"%d" % event.status_code)] + + event.headers + + self.config.response_headers("h2"), + ) + await self._flush() + elif isinstance(event, (Body, Data)): + self.priority.unblock(event.stream_id) + await self.has_data.set() + await self.stream_buffers[event.stream_id].push(event.data) + elif isinstance(event, (EndBody, EndData)): + self.stream_buffers[event.stream_id].set_complete() + self.priority.unblock(event.stream_id) + await self.has_data.set() + await self.stream_buffers[event.stream_id].drain() + elif isinstance(event, StreamClosed): + await self._close_stream(event.stream_id) + idle = len(self.streams) == 0 or all( + stream.idle for stream in self.streams.values() + ) + if idle and self.context.terminated.is_set(): + self.connection.close_connection() + await self._flush() + await self.send(Updated(idle=idle)) + elif isinstance(event, Request): + await self._create_server_push(event.stream_id, event.raw_path, event.headers) + except ( + BufferCompleteError, + KeyError, + priority.MissingStreamError, + h2.exceptions.ProtocolError, + ): + # Connection has closed whilst blocked on flow control or + # connection has advanced ahead of the last emitted event. + return + + async def _handle_events(self, events: List[h2.events.Event]) -> None: + for event in events: + if isinstance(event, h2.events.RequestReceived): + if self.context.terminated.is_set(): + self.connection.reset_stream(event.stream_id) + self.connection.update_settings( + {h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS: 0} + ) + else: + await self._create_stream(event) + await self.send(Updated(idle=False)) + elif isinstance(event, h2.events.DataReceived): + await self.streams[event.stream_id].handle( + Body(stream_id=event.stream_id, data=event.data) + ) + self.connection.acknowledge_received_data( + event.flow_controlled_length, event.stream_id + ) + elif isinstance(event, h2.events.StreamEnded): + await self.streams[event.stream_id].handle(EndBody(stream_id=event.stream_id)) + elif isinstance(event, h2.events.StreamReset): + await self._close_stream(event.stream_id) + await self._window_updated(event.stream_id) + elif isinstance(event, h2.events.WindowUpdated): + await self._window_updated(event.stream_id) + elif isinstance(event, h2.events.PriorityUpdated): + await self._priority_updated(event) + elif isinstance(event, h2.events.RemoteSettingsChanged): + if h2.settings.SettingCodes.INITIAL_WINDOW_SIZE in event.changed_settings: + await self._window_updated(None) + elif isinstance(event, h2.events.ConnectionTerminated): + await self.send(Closed()) + await self._flush() + + async def _flush(self) -> None: + data = self.connection.data_to_send() + if data != b"": + await self.send(RawData(data=data)) + + async def _window_updated(self, stream_id: Optional[int]) -> None: + if stream_id is None or stream_id == 0: + # Unblock all streams + for stream_id in list(self.stream_buffers.keys()): + self.priority.unblock(stream_id) + elif stream_id is not None and stream_id in self.stream_buffers: + self.priority.unblock(stream_id) + await self.has_data.set() + + async def _priority_updated(self, event: h2.events.PriorityUpdated) -> None: + try: + self.priority.reprioritize( + stream_id=event.stream_id, + depends_on=event.depends_on or None, + weight=event.weight, + exclusive=event.exclusive, + ) + except priority.MissingStreamError: + # Received PRIORITY frame before HEADERS frame + self.priority.insert_stream( + stream_id=event.stream_id, + depends_on=event.depends_on or None, + weight=event.weight, + exclusive=event.exclusive, + ) + self.priority.block(event.stream_id) + await self.has_data.set() + + async def _create_stream(self, request: h2.events.RequestReceived) -> None: + for name, value in request.headers: + if name == b":method": + method = value.decode("ascii").upper() + elif name == b":path": + raw_path = value + + if method == "CONNECT": + self.streams[request.stream_id] = WSStream( + self.app, + self.config, + self.context, + self.task_group, + self.tls, + self.client, + self.server, + self.stream_send, + request.stream_id, + ) + else: + self.streams[request.stream_id] = HTTPStream( + self.app, + self.config, + self.context, + self.task_group, + self.tls, + self.client, + self.server, + self.stream_send, + request.stream_id, + ) + self.stream_buffers[request.stream_id] = StreamBuffer(self.context.event_class) + try: + self.priority.insert_stream(request.stream_id) + except priority.DuplicateStreamError: + # Recieved PRIORITY frame before HEADERS frame + pass + else: + self.priority.block(request.stream_id) + + await self.streams[request.stream_id].handle( + Request( + stream_id=request.stream_id, + headers=filter_pseudo_headers(request.headers), + http_version="2", + method=method, + raw_path=raw_path, + ) + ) + + async def _create_server_push( + self, stream_id: int, path: bytes, headers: List[Tuple[bytes, bytes]] + ) -> None: + push_stream_id = self.connection.get_next_available_stream_id() + request_headers = [(b":method", b"GET"), (b":path", path)] + request_headers.extend(headers) + request_headers.extend(self.config.response_headers("h2")) + try: + self.connection.push_stream( + stream_id=stream_id, + promised_stream_id=push_stream_id, + request_headers=request_headers, + ) + await self._flush() + except h2.exceptions.ProtocolError: + # Client does not accept push promises or we are trying to + # push on a push promises request. + pass + else: + event = h2.events.RequestReceived() + event.stream_id = push_stream_id + event.headers = request_headers + await self._create_stream(event) + await self.streams[event.stream_id].handle(EndBody(stream_id=event.stream_id)) + + async def _close_stream(self, stream_id: int) -> None: + if stream_id in self.streams: + stream = self.streams.pop(stream_id) + await stream.handle(StreamClosed(stream_id=stream_id)) + await self.has_data.set() diff --git a/debian/vendor/hypercorn/protocol/h3.py b/debian/vendor/hypercorn/protocol/h3.py new file mode 100644 index 0000000..88d9a4d --- /dev/null +++ b/debian/vendor/hypercorn/protocol/h3.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from typing import Awaitable, Callable, Dict, List, Optional, Tuple, Union + +from aioquic.h3.connection import H3Connection +from aioquic.h3.events import DataReceived, HeadersReceived +from aioquic.h3.exceptions import NoAvailablePushIDError +from aioquic.quic.connection import QuicConnection +from aioquic.quic.events import QuicEvent + +from .events import ( + Body, + Data, + EndBody, + EndData, + Event as StreamEvent, + InformationalResponse, + Request, + Response, + StreamClosed, +) +from .http_stream import HTTPStream +from .ws_stream import WSStream +from ..config import Config +from ..typing import AppWrapper, TaskGroup, WorkerContext +from ..utils import filter_pseudo_headers + + +class H3Protocol: + def __init__( + self, + app: AppWrapper, + config: Config, + context: WorkerContext, + task_group: TaskGroup, + client: Optional[Tuple[str, int]], + server: Optional[Tuple[str, int]], + quic: QuicConnection, + send: Callable[[], Awaitable[None]], + ) -> None: + self.app = app + self.client = client + self.config = config + self.context = context + self.connection = H3Connection(quic) + self.send = send + self.server = server + self.streams: Dict[int, Union[HTTPStream, WSStream]] = {} + self.task_group = task_group + + async def handle(self, quic_event: QuicEvent) -> None: + for event in self.connection.handle_event(quic_event): + if isinstance(event, HeadersReceived): + if not self.context.terminated.is_set(): + await self._create_stream(event) + if event.stream_ended: + await self.streams[event.stream_id].handle( + EndBody(stream_id=event.stream_id) + ) + elif isinstance(event, DataReceived): + await self.streams[event.stream_id].handle( + Body(stream_id=event.stream_id, data=event.data) + ) + if event.stream_ended: + await self.streams[event.stream_id].handle(EndBody(stream_id=event.stream_id)) + + async def stream_send(self, event: StreamEvent) -> None: + if isinstance(event, (InformationalResponse, Response)): + self.connection.send_headers( + event.stream_id, + [(b":status", b"%d" % event.status_code)] + + event.headers + + self.config.response_headers("h3"), + ) + await self.send() + elif isinstance(event, (Body, Data)): + self.connection.send_data(event.stream_id, event.data, False) + await self.send() + elif isinstance(event, (EndBody, EndData)): + self.connection.send_data(event.stream_id, b"", True) + await self.send() + elif isinstance(event, StreamClosed): + pass # ?? + elif isinstance(event, Request): + await self._create_server_push(event.stream_id, event.raw_path, event.headers) + + async def _create_stream(self, request: HeadersReceived) -> None: + for name, value in request.headers: + if name == b":method": + method = value.decode("ascii").upper() + elif name == b":path": + raw_path = value + + if method == "CONNECT": + self.streams[request.stream_id] = WSStream( + self.app, + self.config, + self.context, + self.task_group, + True, + self.client, + self.server, + self.stream_send, + request.stream_id, + ) + else: + self.streams[request.stream_id] = HTTPStream( + self.app, + self.config, + self.context, + self.task_group, + True, + self.client, + self.server, + self.stream_send, + request.stream_id, + ) + + await self.streams[request.stream_id].handle( + Request( + stream_id=request.stream_id, + headers=filter_pseudo_headers(request.headers), + http_version="3", + method=method, + raw_path=raw_path, + ) + ) + + async def _create_server_push( + self, stream_id: int, path: bytes, headers: List[Tuple[bytes, bytes]] + ) -> None: + request_headers = [(b":method", b"GET"), (b":path", path)] + request_headers.extend(headers) + request_headers.extend(self.config.response_headers("h3")) + try: + push_stream_id = self.connection.send_push_promise( + stream_id=stream_id, headers=request_headers + ) + except NoAvailablePushIDError: + # Client does not accept push promises or we are trying to + # push on a push promises request. + pass + else: + event = HeadersReceived( + stream_id=push_stream_id, stream_ended=True, headers=request_headers + ) + await self._create_stream(event) + await self.streams[event.stream_id].handle(EndBody(stream_id=event.stream_id)) diff --git a/debian/vendor/hypercorn/protocol/http_stream.py b/debian/vendor/hypercorn/protocol/http_stream.py new file mode 100644 index 0000000..c4b7077 --- /dev/null +++ b/debian/vendor/hypercorn/protocol/http_stream.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +from enum import auto, Enum +from time import time +from typing import Any, Awaitable, Callable, Optional, Tuple +from urllib.parse import unquote + +from .events import Body, EndBody, Event, InformationalResponse, Request, Response, StreamClosed +from ..config import Config +from ..typing import ( + AppWrapper, + ASGISendEvent, + HTTPResponseStartEvent, + HTTPScope, + TaskGroup, + WorkerContext, +) +from ..utils import ( + build_and_validate_headers, + suppress_body, + UnexpectedMessageError, + valid_server_name, +) + +PUSH_VERSIONS = {"2", "3"} +EARLY_HINTS_VERSIONS = {"2", "3"} + + +class ASGIHTTPState(Enum): + # The ASGI Spec is clear that a response should not start till the + # framework has sent at least one body message hence why this + # state tracking is required. + REQUEST = auto() + RESPONSE = auto() + CLOSED = auto() + + +class HTTPStream: + def __init__( + self, + app: AppWrapper, + config: Config, + context: WorkerContext, + task_group: TaskGroup, + tls: Optional[dict[str, Any]], + client: Optional[Tuple[str, int]], + server: Optional[Tuple[str, int]], + send: Callable[[Event], Awaitable[None]], + stream_id: int, + transport=None, + ) -> None: + self.app = app + self.client = client + self.closed = False + self.config = config + self.context = context + self.response: HTTPResponseStartEvent + self.scope: HTTPScope + self.send = send + self.scheme = "https" if tls is not None else "http" + self.tls = tls + self.server = server + self.start_time: float + self.state = ASGIHTTPState.REQUEST + self.stream_id = stream_id + self.task_group = task_group + self.transport = transport + + @property + def idle(self) -> bool: + return False + + async def handle(self, event: Event) -> None: + if self.closed: + return + elif isinstance(event, Request): + self.start_time = time() + path, _, query_string = event.raw_path.partition(b"?") + self.scope = { + "type": "http", + "http_version": event.http_version, + "asgi": {"spec_version": "2.1", "version": "3.0"}, + "method": event.method, + "scheme": self.scheme, + "path": unquote(path.decode("ascii")), + "raw_path": path, + "query_string": query_string, + "root_path": self.config.root_path, + "headers": event.headers, + "client": self.client, + "server": self.server, + "extensions": {}, + } + if event.http_version in PUSH_VERSIONS: + self.scope["extensions"]["http.response.push"] = {} + + if event.http_version in EARLY_HINTS_VERSIONS: + self.scope["extensions"]["http.response.early_hint"] = {} + + if self.tls is not None: + self.scope["extensions"]["tls"] = self.tls + + if self.transport is not None: + self.scope["extensions"]["_transport"] = self.transport + + if valid_server_name(self.config, event): + self.app_put = await self.task_group.spawn_app( + self.app, self.config, self.scope, self.app_send + ) + else: + await self._send_error_response(404) + self.closed = True + + elif isinstance(event, Body): + await self.app_put( + {"type": "http.request", "body": bytes(event.data), "more_body": True} + ) + elif isinstance(event, EndBody): + await self.app_put({"type": "http.request", "body": b"", "more_body": False}) + elif isinstance(event, StreamClosed): + self.closed = True + await self.config.log.access(self.scope, None, time() - self.start_time) + if self.app_put is not None: + await self.app_put({"type": "http.disconnect"}) + + async def app_send(self, message: Optional[ASGISendEvent]) -> None: + if message is None: # ASGI App has finished sending messages + if not self.closed: + # Cleanup if required + if self.state == ASGIHTTPState.REQUEST: + await self._send_error_response(500) + await self.send(StreamClosed(stream_id=self.stream_id)) + else: + if message["type"] == "http.response.start" and self.state == ASGIHTTPState.REQUEST: + self.response = message + elif ( + message["type"] == "http.response.push" + and self.scope["http_version"] in PUSH_VERSIONS + ): + if not isinstance(message["path"], str): + raise TypeError(f"{message['path']} should be a str") + headers = [(b":scheme", self.scope["scheme"].encode())] + for name, value in self.scope["headers"]: + if name == b"host": + headers.append((b":authority", value)) + headers.extend(build_and_validate_headers(message["headers"])) + await self.send( + Request( + stream_id=self.stream_id, + headers=headers, + http_version=self.scope["http_version"], + method="GET", + raw_path=message["path"].encode(), + ) + ) + elif ( + message["type"] == "http.response.early_hint" + and self.scope["http_version"] in EARLY_HINTS_VERSIONS + and self.state == ASGIHTTPState.REQUEST + ): + headers = [(b"link", bytes(link).strip()) for link in message["links"]] + await self.send( + InformationalResponse( + stream_id=self.stream_id, + headers=headers, + status_code=103, + ) + ) + elif message["type"] == "http.response.body" and self.state in { + ASGIHTTPState.REQUEST, + ASGIHTTPState.RESPONSE, + }: + if self.state == ASGIHTTPState.REQUEST: + headers = build_and_validate_headers(self.response.get("headers", [])) + await self.send( + Response( + stream_id=self.stream_id, + headers=headers, + status_code=int(self.response["status"]), + ) + ) + self.state = ASGIHTTPState.RESPONSE + + if ( + not suppress_body(self.scope["method"], int(self.response["status"])) + and message.get("body", b"") != b"" + ): + await self.send( + Body(stream_id=self.stream_id, data=bytes(message.get("body", b""))) + ) + + if not message.get("more_body", False): + if self.state != ASGIHTTPState.CLOSED: + self.state = ASGIHTTPState.CLOSED + await self.config.log.access( + self.scope, self.response, time() - self.start_time + ) + await self.send(EndBody(stream_id=self.stream_id)) + await self.send(StreamClosed(stream_id=self.stream_id)) + else: + raise UnexpectedMessageError(self.state, message["type"]) + + async def _send_error_response(self, status_code: int) -> None: + await self.send( + Response( + stream_id=self.stream_id, + headers=[(b"content-length", b"0"), (b"connection", b"close")], + status_code=status_code, + ) + ) + await self.send(EndBody(stream_id=self.stream_id)) + self.state = ASGIHTTPState.CLOSED + await self.config.log.access( + self.scope, {"status": status_code, "headers": []}, time() - self.start_time + ) diff --git a/debian/vendor/hypercorn/protocol/quic.py b/debian/vendor/hypercorn/protocol/quic.py new file mode 100644 index 0000000..3d16e54 --- /dev/null +++ b/debian/vendor/hypercorn/protocol/quic.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from functools import partial +from typing import Awaitable, Callable, Dict, Optional, Tuple + +from aioquic.buffer import Buffer +from aioquic.h3.connection import H3_ALPN +from aioquic.quic.configuration import QuicConfiguration +from aioquic.quic.connection import QuicConnection +from aioquic.quic.events import ( + ConnectionIdIssued, + ConnectionIdRetired, + ConnectionTerminated, + ProtocolNegotiated, +) +from aioquic.quic.packet import ( + encode_quic_version_negotiation, + PACKET_TYPE_INITIAL, + pull_quic_header, +) + +from .h3 import H3Protocol +from ..config import Config +from ..events import Closed, Event, RawData +from ..typing import AppWrapper, TaskGroup, WorkerContext + + +class QuicProtocol: + def __init__( + self, + app: AppWrapper, + config: Config, + context: WorkerContext, + task_group: TaskGroup, + server: Optional[Tuple[str, int]], + send: Callable[[Event], Awaitable[None]], + ) -> None: + self.app = app + self.config = config + self.context = context + self.connections: Dict[bytes, QuicConnection] = {} + self.http_connections: Dict[QuicConnection, H3Protocol] = {} + self.send = send + self.server = server + self.task_group = task_group + + self.quic_config = QuicConfiguration(alpn_protocols=H3_ALPN, is_client=False) + self.quic_config.load_cert_chain(certfile=config.certfile, keyfile=config.keyfile) + + @property + def idle(self) -> bool: + return len(self.connections) == 0 and len(self.http_connections) == 0 + + async def handle(self, event: Event) -> None: + if isinstance(event, RawData): + try: + header = pull_quic_header(Buffer(data=event.data), host_cid_length=8) + except ValueError: + return + if ( + header.version is not None + and header.version not in self.quic_config.supported_versions + ): + data = encode_quic_version_negotiation( + source_cid=header.destination_cid, + destination_cid=header.source_cid, + supported_versions=self.quic_config.supported_versions, + ) + await self.send(RawData(data=data, address=event.address)) + return + + connection = self.connections.get(header.destination_cid) + if ( + connection is None + and len(event.data) >= 1200 + and header.packet_type == PACKET_TYPE_INITIAL + and not self.context.terminated.is_set() + ): + connection = QuicConnection( + configuration=self.quic_config, + original_destination_connection_id=header.destination_cid, + ) + self.connections[header.destination_cid] = connection + self.connections[connection.host_cid] = connection + + if connection is not None: + connection.receive_datagram(event.data, event.address, now=self.context.time()) + await self._handle_events(connection, event.address) + elif isinstance(event, Closed): + pass + + async def send_all(self, connection: QuicConnection) -> None: + for data, address in connection.datagrams_to_send(now=self.context.time()): + await self.send(RawData(data=data, address=address)) + + async def _handle_events( + self, connection: QuicConnection, client: Optional[Tuple[str, int]] = None + ) -> None: + event = connection.next_event() + while event is not None: + if isinstance(event, ConnectionTerminated): + pass + elif isinstance(event, ProtocolNegotiated): + self.http_connections[connection] = H3Protocol( + self.app, + self.config, + self.context, + self.task_group, + client, + self.server, + connection, + partial(self.send_all, connection), + ) + elif isinstance(event, ConnectionIdIssued): + self.connections[event.connection_id] = connection + elif isinstance(event, ConnectionIdRetired): + del self.connections[event.connection_id] + + if connection in self.http_connections: + await self.http_connections[connection].handle(event) + + event = connection.next_event() + + await self.send_all(connection) + + timer = connection.get_timer() + if timer is not None: + self.task_group.spawn(self._handle_timer, timer, connection) + + async def _handle_timer(self, timer: float, connection: QuicConnection) -> None: + wait = max(0, timer - self.context.time()) + await self.context.sleep(wait) + if connection._close_at is not None: + connection.handle_timer(now=self.context.time()) + await self._handle_events(connection, None) diff --git a/debian/vendor/hypercorn/protocol/ws_stream.py b/debian/vendor/hypercorn/protocol/ws_stream.py new file mode 100644 index 0000000..3f22ee3 --- /dev/null +++ b/debian/vendor/hypercorn/protocol/ws_stream.py @@ -0,0 +1,375 @@ +from __future__ import annotations + +from enum import auto, Enum +from io import BytesIO, StringIO +from time import time +from typing import Any, Awaitable, Callable, Iterable, List, Optional, Tuple, Union +from urllib.parse import unquote + +from wsproto.connection import Connection, ConnectionState, ConnectionType +from wsproto.events import ( + BytesMessage, + CloseConnection, + Event as WSProtoEvent, + Message, + Ping, + TextMessage, +) +from wsproto.extensions import Extension, PerMessageDeflate +from wsproto.frame_protocol import CloseReason +from wsproto.handshake import server_extensions_handshake, WEBSOCKET_VERSION +from wsproto.utilities import generate_accept_token, split_comma_header + +from .events import Body, Data, EndBody, EndData, Event, Request, Response, StreamClosed +from ..config import Config +from ..typing import ( + AppWrapper, + ASGISendEvent, + TaskGroup, + WebsocketAcceptEvent, + WebsocketResponseBodyEvent, + WebsocketResponseStartEvent, + WebsocketScope, + WorkerContext, +) +from ..utils import ( + build_and_validate_headers, + suppress_body, + UnexpectedMessageError, + valid_server_name, +) + + +class ASGIWebsocketState(Enum): + # Hypercorn supports the ASGI websocket HTTP response extension, + # which allows HTTP responses rather than acceptance. + HANDSHAKE = auto() + CONNECTED = auto() + RESPONSE = auto() + CLOSED = auto() + HTTPCLOSED = auto() + + +class FrameTooLargeError(Exception): + pass + + +class Handshake: + def __init__(self, headers: List[Tuple[bytes, bytes]], http_version: str) -> None: + self.http_version = http_version + self.connection_tokens: Optional[List[str]] = None + self.extensions: Optional[List[str]] = None + self.key: Optional[bytes] = None + self.subprotocols: Optional[List[str]] = None + self.upgrade: Optional[bytes] = None + self.version: Optional[bytes] = None + for name, value in headers: + name = name.lower() + if name == b"connection": + self.connection_tokens = split_comma_header(value) + elif name == b"sec-websocket-extensions": + self.extensions = split_comma_header(value) + elif name == b"sec-websocket-key": + self.key = value + elif name == b"sec-websocket-protocol": + self.subprotocols = split_comma_header(value) + elif name == b"sec-websocket-version": + self.version = value + elif name == b"upgrade": + self.upgrade = value + + def is_valid(self) -> bool: + if self.http_version < "1.1": + return False + elif self.http_version == "1.1": + if self.key is None: + return False + if self.connection_tokens is None or not any( + token.lower() == "upgrade" for token in self.connection_tokens + ): + return False + if self.upgrade.lower() != b"websocket": + return False + + if self.version != WEBSOCKET_VERSION: + return False + return True + + def accept( + self, + subprotocol: Optional[str], + additional_headers: Iterable[Tuple[bytes, bytes]], + ) -> Tuple[int, List[Tuple[bytes, bytes]], Connection]: + headers = [] + if subprotocol is not None: + if self.subprotocols is None or subprotocol not in self.subprotocols: + raise Exception("Invalid Subprotocol") + else: + headers.append((b"sec-websocket-protocol", subprotocol.encode())) + + extensions: List[Extension] = [PerMessageDeflate()] + accepts = None + if self.extensions is not None: + accepts = server_extensions_handshake(self.extensions, extensions) + + if accepts: + headers.append((b"sec-websocket-extensions", accepts)) + + if self.key is not None: + headers.append((b"sec-websocket-accept", generate_accept_token(self.key))) + + status_code = 200 + if self.http_version == "1.1": + headers.extend([(b"upgrade", b"WebSocket"), (b"connection", b"Upgrade")]) + status_code = 101 + + for name, value in additional_headers: + if b"sec-websocket-protocol" == name or name.startswith(b":"): + raise Exception(f"Invalid additional header, {name.decode()}") + + headers.append((name, value)) + + return status_code, headers, Connection(ConnectionType.SERVER, extensions) + + +class WebsocketBuffer: + def __init__(self, max_length: int) -> None: + self.value: Optional[Union[BytesIO, StringIO]] = None + self.length = 0 + self.max_length = max_length + + def extend(self, event: Message) -> None: + if self.value is None: + if isinstance(event, TextMessage): + self.value = StringIO() + else: + self.value = BytesIO() + self.length += self.value.write(event.data) + if self.length > self.max_length: + raise FrameTooLargeError() + + def clear(self) -> None: + self.value = None + self.length = 0 + + def to_message(self) -> dict: + return { + "type": "websocket.receive", + "bytes": self.value.getvalue() if isinstance(self.value, BytesIO) else None, + "text": self.value.getvalue() if isinstance(self.value, StringIO) else None, + } + + +class WSStream: + def __init__( + self, + app: AppWrapper, + config: Config, + context: WorkerContext, + task_group: TaskGroup, + tls: Optional[dict[str, Any]], + client: Optional[Tuple[str, int]], + server: Optional[Tuple[str, int]], + send: Callable[[Event], Awaitable[None]], + stream_id: int, + ) -> None: + self.app = app + self.app_put: Optional[Callable] = None + self.buffer = WebsocketBuffer(config.websocket_max_message_size) + self.client = client + self.closed = False + self.config = config + self.context = context + self.task_group = task_group + self.response: WebsocketResponseStartEvent + self.scope: WebsocketScope + self.send = send + # RFC 8441 for HTTP/2 says use http or https, ASGI says ws or wss + self.scheme = "wss" if tls is not None else "ws" + self.server = server + self.start_time: float + self.state = ASGIWebsocketState.HANDSHAKE + self.stream_id = stream_id + + self.connection: Connection + self.handshake: Handshake + + @property + def idle(self) -> bool: + return self.state in {ASGIWebsocketState.CLOSED, ASGIWebsocketState.HTTPCLOSED} + + async def handle(self, event: Event) -> None: + if self.closed: + return + elif isinstance(event, Request): + self.start_time = time() + self.handshake = Handshake(event.headers, event.http_version) + path, _, query_string = event.raw_path.partition(b"?") + self.scope = { + "type": "websocket", + "asgi": {"spec_version": "2.3", "version": "3.0"}, + "scheme": self.scheme, + "http_version": event.http_version, + "path": unquote(path.decode("ascii")), + "raw_path": path, + "query_string": query_string, + "root_path": self.config.root_path, + "headers": event.headers, + "client": self.client, + "server": self.server, + "subprotocols": self.handshake.subprotocols or [], + "extensions": {"websocket.http.response": {}}, + } + + if not valid_server_name(self.config, event): + await self._send_error_response(404) + self.closed = True + elif not self.handshake.is_valid(): + await self._send_error_response(400) + self.closed = True + else: + self.app_put = await self.task_group.spawn_app( + self.app, self.config, self.scope, self.app_send + ) + await self.app_put({"type": "websocket.connect"}) + elif isinstance(event, (Body, Data)): + self.connection.receive_data(event.data) + await self._handle_events() + elif isinstance(event, StreamClosed): + self.closed = True + if self.app_put is not None: + if self.state in {ASGIWebsocketState.HTTPCLOSED, ASGIWebsocketState.CLOSED}: + code = CloseReason.NORMAL_CLOSURE.value + else: + code = CloseReason.ABNORMAL_CLOSURE.value + await self.app_put({"type": "websocket.disconnect", "code": code}) + + async def app_send(self, message: Optional[ASGISendEvent]) -> None: + if self.closed: + # Allow app to finish after close + return + + if message is None: # ASGI App has finished sending messages + # Cleanup if required + if self.state == ASGIWebsocketState.HANDSHAKE: + await self._send_error_response(500) + await self.config.log.access( + self.scope, {"status": 500, "headers": []}, time() - self.start_time + ) + elif self.state == ASGIWebsocketState.CONNECTED: + await self._send_wsproto_event(CloseConnection(code=CloseReason.INTERNAL_ERROR)) + await self.send(StreamClosed(stream_id=self.stream_id)) + else: + if message["type"] == "websocket.accept" and self.state == ASGIWebsocketState.HANDSHAKE: + await self._accept(message) + elif ( + message["type"] == "websocket.http.response.start" + and self.state == ASGIWebsocketState.HANDSHAKE + ): + self.response = message + elif message["type"] == "websocket.http.response.body" and self.state in { + ASGIWebsocketState.HANDSHAKE, + ASGIWebsocketState.RESPONSE, + }: + await self._send_rejection(message) + elif message["type"] == "websocket.send" and self.state == ASGIWebsocketState.CONNECTED: + event: WSProtoEvent + if message.get("bytes") is not None: + event = BytesMessage(data=bytes(message["bytes"])) + elif not isinstance(message["text"], str): + raise TypeError(f"{message['text']} should be a str") + else: + event = TextMessage(data=message["text"]) + await self._send_wsproto_event(event) + elif ( + message["type"] == "websocket.close" and self.state == ASGIWebsocketState.HANDSHAKE + ): + self.state = ASGIWebsocketState.HTTPCLOSED + await self._send_error_response(403) + elif message["type"] == "websocket.close": + self.state = ASGIWebsocketState.CLOSED + await self._send_wsproto_event( + CloseConnection( + code=int(message.get("code", CloseReason.NORMAL_CLOSURE)), + reason=message.get("reason"), + ) + ) + await self.send(EndData(stream_id=self.stream_id)) + else: + raise UnexpectedMessageError(self.state, message["type"]) + + async def _handle_events(self) -> None: + for event in self.connection.events(): + if isinstance(event, Message): + try: + self.buffer.extend(event) + except FrameTooLargeError: + await self._send_wsproto_event( + CloseConnection(code=CloseReason.MESSAGE_TOO_BIG) + ) + break + + if event.message_finished: + await self.app_put(self.buffer.to_message()) + self.buffer.clear() + elif isinstance(event, Ping): + await self._send_wsproto_event(event.response()) + elif isinstance(event, CloseConnection): + if self.connection.state == ConnectionState.REMOTE_CLOSING: + await self._send_wsproto_event(event.response()) + await self.send(StreamClosed(stream_id=self.stream_id)) + + async def _send_error_response(self, status_code: int) -> None: + await self.send( + Response( + stream_id=self.stream_id, + status_code=status_code, + headers=[(b"content-length", b"0"), (b"connection", b"close")], + ) + ) + await self.send(EndBody(stream_id=self.stream_id)) + await self.config.log.access( + self.scope, {"status": status_code, "headers": []}, time() - self.start_time + ) + + async def _send_wsproto_event(self, event: WSProtoEvent) -> None: + data = self.connection.send(event) + await self.send(Data(stream_id=self.stream_id, data=data)) + + async def _accept(self, message: WebsocketAcceptEvent) -> None: + self.state = ASGIWebsocketState.CONNECTED + status_code, headers, self.connection = self.handshake.accept( + message.get("subprotocol"), message.get("headers", []) + ) + await self.send( + Response(stream_id=self.stream_id, status_code=status_code, headers=headers) + ) + await self.config.log.access( + self.scope, {"status": status_code, "headers": []}, time() - self.start_time + ) + if self.config.websocket_ping_interval is not None: + self.task_group.spawn(self._send_pings) + + async def _send_rejection(self, message: WebsocketResponseBodyEvent) -> None: + body_suppressed = suppress_body("GET", self.response["status"]) + if self.state == ASGIWebsocketState.HANDSHAKE: + headers = build_and_validate_headers(self.response["headers"]) + await self.send( + Response( + stream_id=self.stream_id, + status_code=int(self.response["status"]), + headers=headers, + ) + ) + self.state = ASGIWebsocketState.RESPONSE + if not body_suppressed: + await self.send(Body(stream_id=self.stream_id, data=bytes(message.get("body", b"")))) + if not message.get("more_body", False): + self.state = ASGIWebsocketState.HTTPCLOSED + await self.send(EndBody(stream_id=self.stream_id)) + await self.config.log.access(self.scope, self.response, time() - self.start_time) + + async def _send_pings(self) -> None: + while not self.closed: + await self._send_wsproto_event(Ping()) + await self.context.sleep(self.config.websocket_ping_interval) diff --git a/debian/vendor/hypercorn/py.typed b/debian/vendor/hypercorn/py.typed new file mode 100644 index 0000000..f5642f7 --- /dev/null +++ b/debian/vendor/hypercorn/py.typed @@ -0,0 +1 @@ +Marker diff --git a/debian/vendor/hypercorn/run.py b/debian/vendor/hypercorn/run.py new file mode 100644 index 0000000..a6d2fb0 --- /dev/null +++ b/debian/vendor/hypercorn/run.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import platform +import signal +import time +from multiprocessing import get_context +from multiprocessing.context import BaseContext +from multiprocessing.process import BaseProcess +from multiprocessing.synchronize import Event as EventType +from pickle import PicklingError +from typing import Any, List + +from .config import Config, Sockets +from .typing import WorkerFunc +from .utils import load_application, wait_for_changes, write_pid_file + + +def run(config: Config) -> None: + if config.pid_path is not None: + write_pid_file(config.pid_path) + + worker_func: WorkerFunc + if config.worker_class == "asyncio": + from .asyncio.run import asyncio_worker + + worker_func = asyncio_worker + elif config.worker_class == "uvloop": + from .asyncio.run import uvloop_worker + + worker_func = uvloop_worker + elif config.worker_class == "trio": + from .trio.run import trio_worker + + worker_func = trio_worker + else: + raise ValueError(f"No worker of class {config.worker_class} exists") + + sockets = config.create_sockets() + + if config.use_reloader and config.workers == 0: + raise RuntimeError("Cannot reload without workers") + + if config.use_reloader or config.workers == 0: + # Load the application so that the correct paths are checked for + # changes, but only when the reloader is being used. + load_application(config.application_path, config.wsgi_max_body_size) + + if config.workers == 0: + worker_func(config, sockets) + else: + ctx = get_context("spawn") + + active = True + while active: + # Ignore SIGINT before creating the processes, so that they + # inherit the signal handling. This means that the shutdown + # function controls the shutdown. + signal.signal(signal.SIGINT, signal.SIG_IGN) + + shutdown_event = ctx.Event() + processes = start_processes(config, worker_func, sockets, shutdown_event, ctx) + + def shutdown(*args: Any) -> None: + nonlocal active, shutdown_event + shutdown_event.set() + active = False + + for signal_name in {"SIGINT", "SIGTERM", "SIGBREAK"}: + if hasattr(signal, signal_name): + signal.signal(getattr(signal, signal_name), shutdown) + + if config.use_reloader: + wait_for_changes(shutdown_event) + shutdown_event.set() + # Recreate the sockets to be used again in the next + # iteration of the loop. + sockets = config.create_sockets() + else: + active = False + + for process in processes: + process.join() + for process in processes: + process.terminate() + + for sock in sockets.secure_sockets: + sock.close() + for sock in sockets.insecure_sockets: + sock.close() + + +def start_processes( + config: Config, + worker_func: WorkerFunc, + sockets: Sockets, + shutdown_event: EventType, + ctx: BaseContext, +) -> List[BaseProcess]: + processes = [] + for _ in range(config.workers): + process = ctx.Process( # type: ignore + target=worker_func, + kwargs={"config": config, "shutdown_event": shutdown_event, "sockets": sockets}, + ) + process.daemon = True + try: + process.start() + except PicklingError as error: + raise RuntimeError( + "Cannot pickle the config, see https://docs.python.org/3/library/pickle.html#pickle-picklable" # noqa: E501 + ) from error + processes.append(process) + if platform.system() == "Windows": + time.sleep(0.1) + return processes diff --git a/debian/vendor/hypercorn/statsd.py b/debian/vendor/hypercorn/statsd.py new file mode 100644 index 0000000..9cd7647 --- /dev/null +++ b/debian/vendor/hypercorn/statsd.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from .logging import Logger + +if TYPE_CHECKING: + from .config import Config + from .typing import ResponseSummary, WWWScope + +METRIC_VAR = "metric" +VALUE_VAR = "value" +MTYPE_VAR = "mtype" +GAUGE_TYPE = "gauge" +COUNTER_TYPE = "counter" +HISTOGRAM_TYPE = "histogram" + + +class StatsdLogger(Logger): + def __init__(self, config: "Config") -> None: + super().__init__(config) + self.dogstatsd_tags = config.dogstatsd_tags + self.prefix = config.statsd_prefix + if len(self.prefix) and self.prefix[-1] != ".": + self.prefix += "." + + async def critical(self, message: str, *args: Any, **kwargs: Any) -> None: + await super().critical(message, *args, **kwargs) + await self.increment("hypercorn.log.critical", 1) + + async def error(self, message: str, *args: Any, **kwargs: Any) -> None: + await super().error(message, *args, **kwargs) + await self.increment("hypercorn.log.error", 1) + + async def warning(self, message: str, *args: Any, **kwargs: Any) -> None: + await super().warning(message, *args, **kwargs) + await self.increment("hypercorn.log.warning", 1) + + async def info(self, message: str, *args: Any, **kwargs: Any) -> None: + await super().info(message, *args, **kwargs) + + async def debug(self, message: str, *args: Any, **kwargs: Any) -> None: + await super().debug(message, *args, **kwargs) + + async def exception(self, message: str, *args: Any, **kwargs: Any) -> None: + await super().exception(message, *args, **kwargs) + await self.increment("hypercorn.log.exception", 1) + + async def log(self, level: int, message: str, *args: Any, **kwargs: Any) -> None: + try: + extra = kwargs.get("extra", None) + if extra is not None: + metric = extra.get(METRIC_VAR, None) + value = extra.get(VALUE_VAR, None) + type_ = extra.get(MTYPE_VAR, None) + if metric and value and type_: + if type_ == GAUGE_TYPE: + await self.gauge(metric, value) + elif type_ == COUNTER_TYPE: + await self.increment(metric, value) + elif type_ == HISTOGRAM_TYPE: + await self.histogram(metric, value) + + if message: + await super().log(level, message, *args, **kwargs) + except Exception: + await super().warning("Failed to log to statsd", exc_info=True) + + async def access( + self, request: "WWWScope", response: "ResponseSummary", request_time: float + ) -> None: + await super().access(request, response, request_time) + await self.histogram("hypercorn.request.duration", request_time * 1_000) + await self.increment("hypercorn.requests", 1) + await self.increment(f"hypercorn.request.status.{response['status']}", 1) + + async def gauge(self, name: str, value: int) -> None: + await self._send(f"{self.prefix}{name}:{value}|g") + + async def increment(self, name: str, value: int, sampling_rate: float = 1.0) -> None: + await self._send(f"{self.prefix}{name}:{value}|c|@{sampling_rate}") + + async def decrement(self, name: str, value: int, sampling_rate: float = 1.0) -> None: + await self._send(f"{self.prefix}{name}:-{value}|c|@{sampling_rate}") + + async def histogram(self, name: str, value: float) -> None: + await self._send(f"{self.prefix}{name}:{value}|ms") + + async def _send(self, message: str) -> None: + if self.dogstatsd_tags: + message = f"{message}|#{self.dogstatsd_tags}" + await self._socket_send(message.encode("ascii")) + + async def _socket_send(self, message: bytes) -> None: + raise NotImplementedError() diff --git a/debian/vendor/hypercorn/trio/__init__.py b/debian/vendor/hypercorn/trio/__init__.py new file mode 100644 index 0000000..44a2eb9 --- /dev/null +++ b/debian/vendor/hypercorn/trio/__init__.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import warnings +from typing import Awaitable, Callable, Literal, Optional + +import trio + +from .run import worker_serve +from ..config import Config +from ..typing import Framework +from ..utils import wrap_app + + +async def serve( + app: Framework, + config: Config, + *, + shutdown_trigger: Optional[Callable[..., Awaitable[None]]] = None, + task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED, + mode: Optional[Literal["asgi", "wsgi"]] = None, +) -> None: + """Serve an ASGI framework app given the config. + + This allows for a programmatic way to serve an ASGI framework, it + can be used via, + + .. code-block:: python + + trio.run(serve, app, config) + + It is assumed that the event-loop is configured before calling + this function, therefore configuration values that relate to loop + setup or process setup are ignored. + + Arguments: + app: The ASGI application to serve. + config: A Hypercorn configuration object. + shutdown_trigger: This should return to trigger a graceful + shutdown. + mode: Specify if the app is WSGI or ASGI. + """ + if config.debug: + warnings.warn("The config `debug` has no affect when using serve", Warning) + if config.workers != 1: + warnings.warn("The config `workers` has no affect when using serve", Warning) + + await worker_serve( + wrap_app(app, config.wsgi_max_body_size, mode), + config, + shutdown_trigger=shutdown_trigger, + task_status=task_status, + ) diff --git a/debian/vendor/hypercorn/trio/lifespan.py b/debian/vendor/hypercorn/trio/lifespan.py new file mode 100644 index 0000000..a45fc52 --- /dev/null +++ b/debian/vendor/hypercorn/trio/lifespan.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import trio + +from ..config import Config +from ..typing import AppWrapper, ASGIReceiveEvent, ASGISendEvent, LifespanScope +from ..utils import LifespanFailureError, LifespanTimeoutError + + +class UnexpectedMessageError(Exception): + pass + + +class Lifespan: + def __init__(self, app: AppWrapper, config: Config) -> None: + self.app = app + self.config = config + self.startup = trio.Event() + self.shutdown = trio.Event() + self.app_send_channel, self.app_receive_channel = trio.open_memory_channel( + config.max_app_queue_size + ) + self.supported = True + + async def handle_lifespan( + self, *, task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED + ) -> None: + task_status.started() + scope: LifespanScope = { + "type": "lifespan", + "asgi": {"spec_version": "2.0", "version": "3.0"}, + } + try: + await self.app( + scope, + self.asgi_receive, + self.asgi_send, + trio.to_thread.run_sync, + trio.from_thread.run, + ) + except LifespanFailureError: + # Lifespan failures should crash the server + raise + except Exception: + self.supported = False + if not self.startup.is_set(): + await self.config.log.warning( + "ASGI Framework Lifespan error, continuing without Lifespan support" + ) + elif not self.shutdown.is_set(): + await self.config.log.exception( + "ASGI Framework Lifespan error, shutdown without Lifespan support" + ) + else: + await self.config.log.exception("ASGI Framework Lifespan errored after shutdown.") + finally: + self.startup.set() + self.shutdown.set() + await self.app_send_channel.aclose() + await self.app_receive_channel.aclose() + + async def wait_for_startup(self) -> None: + if not self.supported: + return + + await self.app_send_channel.send({"type": "lifespan.startup"}) + try: + with trio.fail_after(self.config.startup_timeout): + await self.startup.wait() + except trio.TooSlowError as error: + raise LifespanTimeoutError("startup") from error + + async def wait_for_shutdown(self) -> None: + if not self.supported: + return + + await self.app_send_channel.send({"type": "lifespan.shutdown"}) + try: + with trio.fail_after(self.config.shutdown_timeout): + await self.shutdown.wait() + except trio.TooSlowError as error: + raise LifespanTimeoutError("startup") from error + + async def asgi_receive(self) -> ASGIReceiveEvent: + return await self.app_receive_channel.receive() + + async def asgi_send(self, message: ASGISendEvent) -> None: + if message["type"] == "lifespan.startup.complete": + self.startup.set() + elif message["type"] == "lifespan.shutdown.complete": + self.shutdown.set() + elif message["type"] == "lifespan.startup.failed": + raise LifespanFailureError("startup", message.get("message", "")) + elif message["type"] == "lifespan.shutdown.failed": + raise LifespanFailureError("shutdown", message.get("message", "")) + else: + raise UnexpectedMessageError(message["type"]) diff --git a/debian/vendor/hypercorn/trio/run.py b/debian/vendor/hypercorn/trio/run.py new file mode 100644 index 0000000..d8721bb --- /dev/null +++ b/debian/vendor/hypercorn/trio/run.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import sys +from functools import partial +from multiprocessing.synchronize import Event as EventType +from typing import Awaitable, Callable, Optional + +import trio + +from .lifespan import Lifespan +from .statsd import StatsdLogger +from .tcp_server import TCPServer +from .udp_server import UDPServer +from .worker_context import WorkerContext +from ..config import Config, Sockets +from ..typing import AppWrapper +from ..utils import ( + check_multiprocess_shutdown_event, + load_application, + raise_shutdown, + repr_socket_addr, + ShutdownError, +) + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + + +async def worker_serve( + app: AppWrapper, + config: Config, + *, + sockets: Optional[Sockets] = None, + shutdown_trigger: Optional[Callable[..., Awaitable[None]]] = None, + task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED, +) -> None: + config.set_statsd_logger_class(StatsdLogger) + + lifespan = Lifespan(app, config) + context = WorkerContext() + + async with trio.open_nursery() as lifespan_nursery: + await lifespan_nursery.start(lifespan.handle_lifespan) + await lifespan.wait_for_startup() + + async with trio.open_nursery() as server_nursery: + if sockets is None: + sockets = config.create_sockets() + for sock in sockets.secure_sockets: + sock.listen(config.backlog) + for sock in sockets.insecure_sockets: + sock.listen(config.backlog) + + ssl_context = config.create_ssl_context() + listeners = [] + binds = [] + for sock in sockets.secure_sockets: + listeners.append( + trio.SSLListener( + trio.SocketListener(trio.socket.from_stdlib_socket(sock)), + ssl_context, + https_compatible=True, + ) + ) + bind = repr_socket_addr(sock.family, sock.getsockname()) + binds.append(f"https://{bind}") + await config.log.info(f"Running on https://{bind} (CTRL + C to quit)") + + for sock in sockets.insecure_sockets: + listeners.append(trio.SocketListener(trio.socket.from_stdlib_socket(sock))) + bind = repr_socket_addr(sock.family, sock.getsockname()) + binds.append(f"http://{bind}") + await config.log.info(f"Running on http://{bind} (CTRL + C to quit)") + + for sock in sockets.quic_sockets: + await server_nursery.start(UDPServer(app, config, context, sock).run) + bind = repr_socket_addr(sock.family, sock.getsockname()) + await config.log.info(f"Running on https://{bind} (QUIC) (CTRL + C to quit)") + + task_status.started(binds) + try: + async with trio.open_nursery(strict_exception_groups=True) as nursery: + if shutdown_trigger is not None: + nursery.start_soon(raise_shutdown, shutdown_trigger) + + nursery.start_soon( + partial( + trio.serve_listeners, + partial(TCPServer, app, config, context), + listeners, + handler_nursery=server_nursery, + ), + ) + + await trio.sleep_forever() + except BaseExceptionGroup as error: + _, other_errors = error.split((ShutdownError, KeyboardInterrupt)) + if other_errors is not None: + raise other_errors + finally: + await context.terminated.set() + server_nursery.cancel_scope.deadline = trio.current_time() + config.graceful_timeout + + await lifespan.wait_for_shutdown() + lifespan_nursery.cancel_scope.cancel() + + +def trio_worker( + config: Config, sockets: Optional[Sockets] = None, shutdown_event: Optional[EventType] = None +) -> None: + if sockets is not None: + for sock in sockets.secure_sockets: + sock.listen(config.backlog) + for sock in sockets.insecure_sockets: + sock.listen(config.backlog) + app = load_application(config.application_path, config.wsgi_max_body_size) + + shutdown_trigger = None + if shutdown_event is not None: + shutdown_trigger = partial(check_multiprocess_shutdown_event, shutdown_event, trio.sleep) + + trio.run(partial(worker_serve, app, config, sockets=sockets, shutdown_trigger=shutdown_trigger)) diff --git a/debian/vendor/hypercorn/trio/statsd.py b/debian/vendor/hypercorn/trio/statsd.py new file mode 100644 index 0000000..db04176 --- /dev/null +++ b/debian/vendor/hypercorn/trio/statsd.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import trio + +from ..config import Config +from ..statsd import StatsdLogger as Base + + +class StatsdLogger(Base): + def __init__(self, config: Config) -> None: + super().__init__(config) + self.address = tuple(config.statsd_host.rsplit(":", 1)) + self.socket = trio.socket.socket(trio.socket.AF_INET, trio.socket.SOCK_DGRAM) + + async def _socket_send(self, message: bytes) -> None: + await self.socket.sendto(message, self.address) diff --git a/debian/vendor/hypercorn/trio/task_group.py b/debian/vendor/hypercorn/trio/task_group.py new file mode 100644 index 0000000..044ff85 --- /dev/null +++ b/debian/vendor/hypercorn/trio/task_group.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import sys +from types import TracebackType +from typing import Any, Awaitable, Callable, Optional + +import trio + +from ..config import Config +from ..typing import AppWrapper, ASGIReceiveCallable, ASGIReceiveEvent, ASGISendEvent, Scope + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + + +async def _handle( + app: AppWrapper, + config: Config, + scope: Scope, + receive: ASGIReceiveCallable, + send: Callable[[Optional[ASGISendEvent]], Awaitable[None]], + sync_spawn: Callable, + call_soon: Callable, +) -> None: + try: + await app(scope, receive, send, sync_spawn, call_soon) + except trio.Cancelled: + raise + except BaseExceptionGroup as error: + _, other_errors = error.split(trio.Cancelled) + if other_errors is not None: + await config.log.exception("Error in ASGI Framework") + await send(None) + else: + raise + except Exception: + await config.log.exception("Error in ASGI Framework") + finally: + await send(None) + + +class TaskGroup: + def __init__(self) -> None: + self._nursery: Optional[trio._core._run.Nursery] = None + self._nursery_manager: Optional[trio._core._run.NurseryManager] = None + + async def spawn_app( + self, + app: AppWrapper, + config: Config, + scope: Scope, + send: Callable[[Optional[ASGISendEvent]], Awaitable[None]], + ) -> Callable[[ASGIReceiveEvent], Awaitable[None]]: + app_send_channel, app_receive_channel = trio.open_memory_channel(config.max_app_queue_size) + self._nursery.start_soon( + _handle, + app, + config, + scope, + app_receive_channel.receive, + send, + trio.to_thread.run_sync, + trio.from_thread.run, + ) + return app_send_channel.send + + def spawn(self, func: Callable, *args: Any) -> None: + self._nursery.start_soon(func, *args) + + async def __aenter__(self) -> TaskGroup: + self._nursery_manager = trio.open_nursery() + self._nursery = await self._nursery_manager.__aenter__() + return self + + async def __aexit__(self, exc_type: type, exc_value: BaseException, tb: TracebackType) -> None: + await self._nursery_manager.__aexit__(exc_type, exc_value, tb) + self._nursery_manager = None + self._nursery = None diff --git a/debian/vendor/hypercorn/trio/tcp_server.py b/debian/vendor/hypercorn/trio/tcp_server.py new file mode 100644 index 0000000..e723f50 --- /dev/null +++ b/debian/vendor/hypercorn/trio/tcp_server.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import ssl +from math import inf +from typing import Any, Generator, Optional + +import trio + +from .task_group import TaskGroup +from .worker_context import WorkerContext +from ..config import Config +from ..events import Closed, Event, RawData, Updated +from ..protocol import ProtocolWrapper +from ..typing import AppWrapper +from ..utils import parse_socket_addr + +MAX_RECV = 2**16 + + +class TCPServer: + def __init__( + self, app: AppWrapper, config: Config, context: WorkerContext, stream: trio.abc.Stream + ) -> None: + self.app = app + self.config = config + self.context = context + self.protocol: ProtocolWrapper + self.send_lock = trio.Lock() + self.idle_lock = trio.Lock() + self.stream = stream + + self._idle_handle: Optional[trio.CancelScope] = None + + def __await__(self) -> Generator[Any, None, None]: + return self.run().__await__() + + async def run(self) -> None: + try: + try: + with trio.fail_after(self.config.ssl_handshake_timeout): + await self.stream.do_handshake() + except (trio.BrokenResourceError, trio.TooSlowError): + return # Handshake failed + alpn_protocol = self.stream.selected_alpn_protocol() + socket = self.stream.transport_stream.socket + + tls = {"alpn_protocol": alpn_protocol} + client_certificate = self.stream.getpeercert(binary_form=False) + if client_certificate: + tls["client_cert_name"] = ", ".join( + [f"{part[0][0]}={part[0][1]}" for part in client_certificate["subject"]] + ) + except AttributeError: # Not SSL + alpn_protocol = "http/1.1" + socket = self.stream.socket + tls = None + + try: + client = parse_socket_addr(socket.family, socket.getpeername()) + server = parse_socket_addr(socket.family, socket.getsockname()) + + async with TaskGroup() as task_group: + self._task_group = task_group + self.protocol = ProtocolWrapper( + self.app, + self.config, + self.context, + task_group, + tls, + client, + server, + self.protocol_send, + alpn_protocol, + self.stream, + ) + await self.protocol.initiate() + await self._start_idle() + await self._read_data() + except OSError: + pass + finally: + await self._close() + + async def protocol_send(self, event: Event) -> None: + if isinstance(event, RawData): + async with self.send_lock: + try: + with trio.CancelScope() as cancel_scope: + cancel_scope.shield = True + await self.stream.send_all(event.data) + except (trio.BrokenResourceError, trio.ClosedResourceError): + await self.protocol.handle(Closed()) + elif isinstance(event, Closed): + await self._close() + await self.protocol.handle(Closed()) + elif isinstance(event, Updated): + if event.idle: + await self._start_idle() + else: + await self._stop_idle() + + async def _read_data(self) -> None: + while True: + try: + with trio.fail_after(self.config.read_timeout or inf): + data = await self.stream.receive_some(MAX_RECV) + except ( + trio.ClosedResourceError, + trio.BrokenResourceError, + trio.TooSlowError, + ): + break + else: + await self.protocol.handle(RawData(data)) + if data == b"": + break + await self.protocol.handle(Closed()) + + async def _close(self) -> None: + try: + await self.stream.send_eof() + except ( + trio.BrokenResourceError, + AttributeError, + trio.BusyResourceError, + trio.ClosedResourceError, + ): + # They're already gone, nothing to do + # Or it is a SSL stream + pass + await self.stream.aclose() + + async def _initiate_server_close(self) -> None: + await self.protocol.handle(Closed()) + await self.stream.aclose() + + async def _start_idle(self) -> None: + async with self.idle_lock: + if self._idle_handle is None: + self._idle_handle = await self._task_group._nursery.start(self._run_idle) + + async def _stop_idle(self) -> None: + async with self.idle_lock: + if self._idle_handle is not None: + self._idle_handle.cancel() + self._idle_handle = None + + async def _run_idle( + self, + task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED, + ) -> None: + cancel_scope = trio.CancelScope() + task_status.started(cancel_scope) + with cancel_scope: + with trio.move_on_after(self.config.keep_alive_timeout): + await self.context.terminated.wait() + + cancel_scope.shield = True + await self._initiate_server_close() diff --git a/debian/vendor/hypercorn/trio/udp_server.py b/debian/vendor/hypercorn/trio/udp_server.py new file mode 100644 index 0000000..b8d4530 --- /dev/null +++ b/debian/vendor/hypercorn/trio/udp_server.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import trio + +from .task_group import TaskGroup +from .worker_context import WorkerContext +from ..config import Config +from ..events import Event, RawData +from ..typing import AppWrapper +from ..utils import parse_socket_addr + +MAX_RECV = 2**16 + + +class UDPServer: + def __init__( + self, + app: AppWrapper, + config: Config, + context: WorkerContext, + socket: trio.socket.socket, + ) -> None: + self.app = app + self.config = config + self.context = context + self.socket = trio.socket.from_stdlib_socket(socket) + + async def run( + self, task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED + ) -> None: + from ..protocol.quic import QuicProtocol # h3/Quic is an optional part of Hypercorn + + task_status.started() + server = parse_socket_addr(self.socket.family, self.socket.getsockname()) + async with TaskGroup() as task_group: + self.protocol = QuicProtocol( + self.app, self.config, self.context, task_group, server, self.protocol_send + ) + + while not self.context.terminated.is_set() or not self.protocol.idle: + data, address = await self.socket.recvfrom(MAX_RECV) + await self.protocol.handle(RawData(data=data, address=address)) + + async def protocol_send(self, event: Event) -> None: + if isinstance(event, RawData): + await self.socket.sendto(event.data, event.address) diff --git a/debian/vendor/hypercorn/trio/worker_context.py b/debian/vendor/hypercorn/trio/worker_context.py new file mode 100644 index 0000000..bcfa1a5 --- /dev/null +++ b/debian/vendor/hypercorn/trio/worker_context.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import Type, Union + +import trio + +from ..typing import Event + + +class EventWrapper: + def __init__(self) -> None: + self._event = trio.Event() + + async def clear(self) -> None: + self._event = trio.Event() + + async def wait(self) -> None: + await self._event.wait() + + async def set(self) -> None: + self._event.set() + + def is_set(self) -> bool: + return self._event.is_set() + + +class WorkerContext: + event_class: Type[Event] = EventWrapper + + def __init__(self) -> None: + self.terminated = self.event_class() + + @staticmethod + async def sleep(wait: Union[float, int]) -> None: + return await trio.sleep(wait) + + @staticmethod + def time() -> float: + return trio.current_time() diff --git a/debian/vendor/hypercorn/typing.py b/debian/vendor/hypercorn/typing.py new file mode 100644 index 0000000..1299a77 --- /dev/null +++ b/debian/vendor/hypercorn/typing.py @@ -0,0 +1,338 @@ +from __future__ import annotations + +from multiprocessing.synchronize import Event as EventType +from types import TracebackType +from typing import ( + Any, + Awaitable, + Callable, + Dict, + Iterable, + Literal, + Optional, + Protocol, + Tuple, + Type, + TypedDict, + Union, +) + +import h2.events +import h11 + +from .config import Config, Sockets + +H11SendableEvent = Union[h11.Data, h11.EndOfMessage, h11.InformationalResponse, h11.Response] + +WorkerFunc = Callable[[Config, Optional[Sockets], Optional[EventType]], None] + + +class ASGIVersions(TypedDict, total=False): + spec_version: str + version: Union[Literal["2.0"], Literal["3.0"]] + + +class HTTPScope(TypedDict): + type: Literal["http"] + asgi: ASGIVersions + http_version: str + method: str + scheme: str + path: str + raw_path: bytes + query_string: bytes + root_path: str + headers: Iterable[Tuple[bytes, bytes]] + client: Optional[Tuple[str, int]] + server: Optional[Tuple[str, Optional[int]]] + extensions: Dict[str, dict] + + +class WebsocketScope(TypedDict): + type: Literal["websocket"] + asgi: ASGIVersions + http_version: str + scheme: str + path: str + raw_path: bytes + query_string: bytes + root_path: str + headers: Iterable[Tuple[bytes, bytes]] + client: Optional[Tuple[str, int]] + server: Optional[Tuple[str, Optional[int]]] + subprotocols: Iterable[str] + extensions: Dict[str, dict] + + +class LifespanScope(TypedDict): + type: Literal["lifespan"] + asgi: ASGIVersions + + +WWWScope = Union[HTTPScope, WebsocketScope] +Scope = Union[HTTPScope, WebsocketScope, LifespanScope] + + +class HTTPRequestEvent(TypedDict): + type: Literal["http.request"] + body: bytes + more_body: bool + + +class HTTPResponseStartEvent(TypedDict): + type: Literal["http.response.start"] + status: int + headers: Iterable[Tuple[bytes, bytes]] + + +class HTTPResponseBodyEvent(TypedDict): + type: Literal["http.response.body"] + body: bytes + more_body: bool + + +class HTTPServerPushEvent(TypedDict): + type: Literal["http.response.push"] + path: str + headers: Iterable[Tuple[bytes, bytes]] + + +class HTTPEarlyHintEvent(TypedDict): + type: Literal["http.response.early_hint"] + links: Iterable[bytes] + + +class HTTPDisconnectEvent(TypedDict): + type: Literal["http.disconnect"] + + +class WebsocketConnectEvent(TypedDict): + type: Literal["websocket.connect"] + + +class WebsocketAcceptEvent(TypedDict): + type: Literal["websocket.accept"] + subprotocol: Optional[str] + headers: Iterable[Tuple[bytes, bytes]] + + +class WebsocketReceiveEvent(TypedDict): + type: Literal["websocket.receive"] + bytes: Optional[bytes] + text: Optional[str] + + +class WebsocketSendEvent(TypedDict): + type: Literal["websocket.send"] + bytes: Optional[bytes] + text: Optional[str] + + +class WebsocketResponseStartEvent(TypedDict): + type: Literal["websocket.http.response.start"] + status: int + headers: Iterable[Tuple[bytes, bytes]] + + +class WebsocketResponseBodyEvent(TypedDict): + type: Literal["websocket.http.response.body"] + body: bytes + more_body: bool + + +class WebsocketDisconnectEvent(TypedDict): + type: Literal["websocket.disconnect"] + code: int + + +class WebsocketCloseEvent(TypedDict): + type: Literal["websocket.close"] + code: int + reason: Optional[str] + + +class LifespanStartupEvent(TypedDict): + type: Literal["lifespan.startup"] + + +class LifespanShutdownEvent(TypedDict): + type: Literal["lifespan.shutdown"] + + +class LifespanStartupCompleteEvent(TypedDict): + type: Literal["lifespan.startup.complete"] + + +class LifespanStartupFailedEvent(TypedDict): + type: Literal["lifespan.startup.failed"] + message: str + + +class LifespanShutdownCompleteEvent(TypedDict): + type: Literal["lifespan.shutdown.complete"] + + +class LifespanShutdownFailedEvent(TypedDict): + type: Literal["lifespan.shutdown.failed"] + message: str + + +ASGIReceiveEvent = Union[ + HTTPRequestEvent, + HTTPDisconnectEvent, + WebsocketConnectEvent, + WebsocketReceiveEvent, + WebsocketDisconnectEvent, + LifespanStartupEvent, + LifespanShutdownEvent, +] + + +ASGISendEvent = Union[ + HTTPResponseStartEvent, + HTTPResponseBodyEvent, + HTTPServerPushEvent, + HTTPEarlyHintEvent, + HTTPDisconnectEvent, + WebsocketAcceptEvent, + WebsocketSendEvent, + WebsocketResponseStartEvent, + WebsocketResponseBodyEvent, + WebsocketCloseEvent, + LifespanStartupCompleteEvent, + LifespanStartupFailedEvent, + LifespanShutdownCompleteEvent, + LifespanShutdownFailedEvent, +] + + +ASGIReceiveCallable = Callable[[], Awaitable[ASGIReceiveEvent]] +ASGISendCallable = Callable[[ASGISendEvent], Awaitable[None]] + +ASGIFramework = Callable[ + [ + Scope, + ASGIReceiveCallable, + ASGISendCallable, + ], + Awaitable[None], +] +WSGIFramework = Callable[[dict, Callable], Iterable[bytes]] +Framework = Union[ASGIFramework, WSGIFramework] + + +class H2SyncStream(Protocol): + scope: dict + + def data_received(self, data: bytes) -> None: + ... + + def ended(self) -> None: + ... + + def reset(self) -> None: + ... + + def close(self) -> None: + ... + + async def handle_request( + self, + event: h2.events.RequestReceived, + scheme: str, + client: Tuple[str, int], + server: Tuple[str, int], + ) -> None: + ... + + +class H2AsyncStream(Protocol): + scope: dict + + async def data_received(self, data: bytes) -> None: + ... + + async def ended(self) -> None: + ... + + async def reset(self) -> None: + ... + + async def close(self) -> None: + ... + + async def handle_request( + self, + event: h2.events.RequestReceived, + scheme: str, + client: Tuple[str, int], + server: Tuple[str, int], + ) -> None: + ... + + +class Event(Protocol): + def __init__(self) -> None: + ... + + async def clear(self) -> None: + ... + + async def set(self) -> None: + ... + + async def wait(self) -> None: + ... + + def is_set(self) -> bool: + ... + + +class WorkerContext(Protocol): + event_class: Type[Event] + terminated: Event + + @staticmethod + async def sleep(wait: Union[float, int]) -> None: + ... + + @staticmethod + def time() -> float: + ... + + +class TaskGroup(Protocol): + async def spawn_app( + self, + app: AppWrapper, + config: Config, + scope: Scope, + send: Callable[[Optional[ASGISendEvent]], Awaitable[None]], + ) -> Callable[[ASGIReceiveEvent], Awaitable[None]]: + ... + + def spawn(self, func: Callable, *args: Any) -> None: + ... + + async def __aenter__(self) -> TaskGroup: + ... + + async def __aexit__(self, exc_type: type, exc_value: BaseException, tb: TracebackType) -> None: + ... + + +class ResponseSummary(TypedDict): + status: int + headers: Iterable[Tuple[bytes, bytes]] + + +class AppWrapper(Protocol): + async def __call__( + self, + scope: Scope, + receive: ASGIReceiveCallable, + send: ASGISendCallable, + sync_spawn: Callable, + call_soon: Callable, + ) -> None: + ... diff --git a/debian/vendor/hypercorn/utils.py b/debian/vendor/hypercorn/utils.py new file mode 100644 index 0000000..5629ff7 --- /dev/null +++ b/debian/vendor/hypercorn/utils.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import inspect +import os +import socket +import sys +import time +from enum import Enum +from importlib import import_module +from multiprocessing.synchronize import Event as EventType +from pathlib import Path +from typing import ( + Any, + Awaitable, + Callable, + cast, + Dict, + Iterable, + List, + Literal, + Optional, + Tuple, + TYPE_CHECKING, +) + +from .app_wrappers import ASGIWrapper, WSGIWrapper +from .config import Config +from .typing import AppWrapper, ASGIFramework, Framework, WSGIFramework + +if TYPE_CHECKING: + from .protocol.events import Request + + +class ShutdownError(Exception): + pass + + +class NoAppError(Exception): + pass + + +class LifespanTimeoutError(Exception): + def __init__(self, stage: str) -> None: + super().__init__( + f"Timeout whilst awaiting {stage}. Your application may not support the ASGI Lifespan " + f"protocol correctly, alternatively the {stage}_timeout configuration is incorrect." + ) + + +class LifespanFailureError(Exception): + def __init__(self, stage: str, message: str) -> None: + super().__init__(f"Lifespan failure in {stage}. '{message}'") + + +class UnexpectedMessageError(Exception): + def __init__(self, state: Enum, message_type: str) -> None: + super().__init__(f"Unexpected message type, {message_type} given the state {state}") + + +class FrameTooLargeError(Exception): + pass + + +def suppress_body(method: str, status_code: int) -> bool: + return method == "HEAD" or 100 <= status_code < 200 or status_code in {204, 304} + + +def build_and_validate_headers(headers: Iterable[Tuple[bytes, bytes]]) -> List[Tuple[bytes, bytes]]: + # Validates that the header name and value are bytes + validated_headers: List[Tuple[bytes, bytes]] = [] + for name, value in headers: + if name[0] == b":"[0]: + raise ValueError("Pseudo headers are not valid") + validated_headers.append((bytes(name).strip(), bytes(value).strip())) + return validated_headers + + +def filter_pseudo_headers(headers: List[Tuple[bytes, bytes]]) -> List[Tuple[bytes, bytes]]: + filtered_headers: List[Tuple[bytes, bytes]] = [(b"host", b"")] # Placeholder + authority = None + host = b"" + for name, value in headers: + if name == b":authority": # h2 & h3 libraries validate this is present + authority = value + elif name == b"host": + host = value + elif name[0] != b":"[0]: + filtered_headers.append((name, value)) + filtered_headers[0] = (b"host", authority if authority is not None else host) + return filtered_headers + + +def load_application(path: str, wsgi_max_body_size: int) -> AppWrapper: + mode: Optional[Literal["asgi", "wsgi"]] = None + if ":" not in path: + module_name, app_name = path, "app" + elif path.count(":") == 2: + mode, module_name, app_name = path.split(":", 2) # type: ignore + if mode not in {"asgi", "wsgi"}: + raise ValueError("Invalid mode, must be 'asgi', or 'wsgi'") + else: + module_name, app_name = path.split(":", 1) + + module_path = Path(module_name).resolve() + sys.path.insert(0, str(module_path.parent)) + if module_path.is_file(): + import_name = module_path.with_suffix("").name + else: + import_name = module_path.name + try: + module = import_module(import_name) + except ModuleNotFoundError as error: + if error.name == import_name: + raise NoAppError(f"Cannot load application from '{path}', module not found.") + else: + raise + try: + app = eval(app_name, vars(module)) + except NameError: + raise NoAppError(f"Cannot load application from '{path}', application not found.") + else: + return wrap_app(app, wsgi_max_body_size, mode) + + +def wrap_app( + app: Framework, wsgi_max_body_size: int, mode: Optional[Literal["asgi", "wsgi"]] +) -> AppWrapper: + if mode is None: + mode = "asgi" if is_asgi(app) else "wsgi" + if mode == "asgi": + return ASGIWrapper(cast(ASGIFramework, app)) + else: + return WSGIWrapper(cast(WSGIFramework, app), wsgi_max_body_size) + + +def wait_for_changes(shutdown_event: EventType) -> None: + last_updates: Dict[Path, float] = {} + for module in list(sys.modules.values()): + filename = getattr(module, "__file__", None) + if filename is None: + continue + path = Path(filename) + try: + last_updates[Path(filename)] = path.stat().st_mtime + except (FileNotFoundError, NotADirectoryError): + pass + + while not shutdown_event.is_set(): + time.sleep(1) + + for index, (path, last_mtime) in enumerate(last_updates.items()): + if index % 10 == 0: + # Yield to the event loop + time.sleep(0) + + try: + mtime = path.stat().st_mtime + except FileNotFoundError: + return + else: + if mtime > last_mtime: + return + else: + last_updates[path] = mtime + + +async def raise_shutdown(shutdown_event: Callable[..., Awaitable]) -> None: + await shutdown_event() + raise ShutdownError() + + +async def check_multiprocess_shutdown_event( + shutdown_event: EventType, sleep: Callable[[float], Awaitable[Any]] +) -> None: + while True: + if shutdown_event.is_set(): + return + await sleep(0.1) + + +def write_pid_file(pid_path: str) -> None: + with open(pid_path, "w") as file_: + file_.write(f"{os.getpid()}") + + +def parse_socket_addr(family: int, address: tuple) -> Optional[Tuple[str, int]]: + if family == socket.AF_INET: + return address # type: ignore + elif family == socket.AF_INET6: + return (address[0], address[1]) + else: + return None + + +def repr_socket_addr(family: int, address: tuple) -> str: + if family == socket.AF_INET: + return f"{address[0]}:{address[1]}" + elif family == socket.AF_INET6: + return f"[{address[0]}]:{address[1]}" + elif family == socket.AF_UNIX: + return f"unix:{address}" + else: + return f"{address}" + + +def valid_server_name(config: Config, request: "Request") -> bool: + if len(config.server_names) == 0: + return True + + host = "" + for name, value in request.headers: + if name.lower() == b"host": + host = value.decode() + break + return host in config.server_names + + +def is_asgi(app: Any) -> bool: + if inspect.iscoroutinefunction(app): + return True + elif hasattr(app, "__call__"): + return inspect.iscoroutinefunction(app.__call__) + return False diff --git a/dev-requirements.txt b/dev-requirements.txt index f5a4db5..a4ec9fe 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,21 +1,23 @@ -mock==3.0.5 -coverage~=5.0;python_version<="2.7" -coverage~=6.0;python_version>="3.6" -tornado==5.1.1;python_version<="2.7" -tornado==6.1.0;python_version>="3.6" +h2==4.1.0 +build==1.2.1 +coverage==7.6.4 PySocks==1.7.1 -# https://github.com/Anorov/PySocks/issues/131 -win-inet-pton==1.1.0 -pytest==4.6.9; python_version<"3.10" -pytest==6.2.4; python_version>="3.10" -pytest-timeout==1.4.2 -pytest-freezegun==0.4.2 -flaky==3.7.0 -trustme==0.7.0 -cryptography==3.2.1;python_version<"3.6" -cryptography==38.0.3;python_version>="3.6" -python-dateutil==2.8.1 +pytest==8.0.2 +pytest-timeout==2.1.0 +pyOpenSSL==24.2.1 +idna==3.7 +trustme==1.2.0 +cryptography==43.0.1 +towncrier==23.6.0 +pytest-memray==1.7.0;python_version<"3.14" and sys_platform!="win32" and implementation_name=="cpython" +trio==0.26.2 +# https://github.com/pallets/quart/pull/369 +Quart @ git+https://github.com/pallets/quart@67110bf383d8973bce1619e957b4b6ea088ad9f2 +quart-trio==0.11.1 +# https://github.com/pgjones/hypercorn/issues/62 +# https://github.com/pgjones/hypercorn/issues/168 +# https://github.com/pgjones/hypercorn/issues/169 +hypercorn @ git+https://github.com/urllib3/hypercorn@urllib3-changes +httpx==0.25.2 +pytest-socket==0.7.0 -# https://github.com/GrahamDumpleton/wrapt/issues/189 -wrapt==1.12.1; python_version<="2.7" and sys_platform=="win32" -gcp-devrel-py-tools==0.0.16 diff --git a/docs/_static/banner_github.svg b/docs/_static/banner_github.svg new file mode 100644 index 0000000..069aa19 --- /dev/null +++ b/docs/_static/banner_github.svg @@ -0,0 +1,13 @@ + + + Layer 1 + + + + + + + + + + \ No newline at end of file diff --git a/docs/advanced-usage.rst b/docs/advanced-usage.rst index d38e719..f6ab70b 100644 --- a/docs/advanced-usage.rst +++ b/docs/advanced-usage.rst @@ -11,10 +11,13 @@ The :class:`~poolmanager.PoolManager` class automatically handles creating :class:`~connectionpool.ConnectionPool` instances for each host as needed. By default, it will keep a maximum of 10 :class:`~connectionpool.ConnectionPool` instances. If you're making requests to many different hosts it might improve -performance to increase this number:: +performance to increase this number. - >>> import urllib3 - >>> http = urllib3.PoolManager(num_pools=50) +.. code-block:: python + + import urllib3 + + http = urllib3.PoolManager(num_pools=50) However, keep in mind that this does increase memory and socket consumption. @@ -23,12 +26,15 @@ of individual :class:`~connection.HTTPConnection` instances. These connections are used during an individual request and returned to the pool when the request is complete. By default only one connection will be saved for re-use. If you are making many requests to the same host simultaneously it might improve -performance to increase this number:: +performance to increase this number. - >>> import urllib3 - >>> http = urllib3.PoolManager(maxsize=10) +.. code-block:: python + + import urllib3 + + http = urllib3.PoolManager(maxsize=10) # Alternatively - >>> http = urllib3.HTTPConnectionPool('google.com', maxsize=10) + pool = urllib3.HTTPConnectionPool("google.com", maxsize=10) The behavior of the pooling for :class:`~connectionpool.ConnectionPool` is different from :class:`~poolmanager.PoolManager`. By default, if a new @@ -37,11 +43,14 @@ connection will be created. However, this connection will not be saved if more than ``maxsize`` connections exist. This means that ``maxsize`` does not determine the maximum number of connections that can be open to a particular host, just the maximum number of connections to keep in the pool. However, if you specify ``block=True`` then there can be at most ``maxsize`` connections -open to a particular host:: +open to a particular host. + +.. code-block:: python + + http = urllib3.PoolManager(maxsize=10, block=True) - >>> http = urllib3.PoolManager(maxsize=10, block=True) # Alternatively - >>> http = urllib3.HTTPConnectionPool('google.com', maxsize=10, block=True) + pool = urllib3.HTTPConnectionPool("google.com", maxsize=10, block=True) Any new requests will block until a connection is available from the pool. This is a great way to prevent flooding a host with too many connections in @@ -81,49 +90,79 @@ that urllib3 will only read from the socket when data is requested. :meth:`~response.HTTPResponse.stream` lets you iterate over chunks of the response content. - >>> import urllib3 - >>> http = urllib3.PoolManager() - >>> r = http.request( - ... 'GET', - ... 'http://httpbin.org/bytes/1024', - ... preload_content=False) - >>> for chunk in r.stream(32): - ... print(chunk) - b'...' - b'...' - ... - >>> r.release_conn() +.. code-block:: python + + import urllib3 + + resp = urllib3.request( + "GET", + "https://httpbin.org/bytes/1024", + preload_content=False + ) + + for chunk in resp.stream(32): + print(chunk) + # b"\x9e\xa97'\x8e\x1eT .... + + resp.release_conn() However, you can also treat the :class:`~response.HTTPResponse` instance as -a file-like object. This allows you to do buffering:: +a file-like object. This allows you to do buffering: - >>> r = http.request( - ... 'GET', - ... 'http://httpbin.org/bytes/1024', - ... preload_content=False) - >>> r.read(4) - b'\x88\x1f\x8b\xe5' +.. code-block:: python + + import urllib3 + + resp = urllib3.request( + "GET", + "https://httpbin.org/bytes/1024", + preload_content=False + ) + + print(resp.read(4)) + # b"\x88\x1f\x8b\xe5" Calls to :meth:`~response.HTTPResponse.read()` will block until more response data is available. - >>> import io - >>> reader = io.BufferedReader(r, 8) - >>> reader.read(4) - >>> r.release_conn() +.. code-block:: python + + import io + import urllib3 + + resp = urllib3.request( + "GET", + "https://httpbin.org/bytes/1024", + preload_content=False + ) + + reader = io.BufferedReader(resp, 8) + print(reader.read(4)) + # b"\xbf\x9c\xd6" + + resp.release_conn() You can use this file-like object to do things like decode the content using -:mod:`codecs`:: - - >>> import codecs - >>> reader = codecs.getreader('utf-8') - >>> r = http.request( - ... 'GET', - ... 'http://httpbin.org/ip', - ... preload_content=False) - >>> json.load(reader(r)) - {'origin': '127.0.0.1'} - >>> r.release_conn() +:mod:`codecs`: + +.. code-block:: python + + import codecs + import json + import urllib3 + + reader = codecs.getreader("utf-8") + + resp = urllib3.request( + "GET", + "https://httpbin.org/ip", + preload_content=False + ) + + print(json.load(reader(resp))) + # {"origin": "127.0.0.1"} + + resp.release_conn() .. _proxies: @@ -131,11 +170,14 @@ Proxies ------- You can use :class:`~poolmanager.ProxyManager` to tunnel requests through an -HTTP proxy:: +HTTP proxy: + +.. code-block:: python - >>> import urllib3 - >>> proxy = urllib3.ProxyManager('http://localhost:3128/') - >>> proxy.request('GET', 'http://google.com/') + import urllib3 + + proxy = urllib3.ProxyManager("https://localhost:3128/") + proxy.request("GET", "https://google.com/") The usage of :class:`~poolmanager.ProxyManager` is the same as :class:`~poolmanager.PoolManager`. @@ -156,23 +198,23 @@ urllib3's behavior will be different depending on your proxy and destination: * HTTP proxy + HTTP destination Your request will be forwarded with the `absolute URI - `_. + `_. * HTTP proxy + HTTPS destination A TCP tunnel will be established with a `HTTP - CONNECT `_. Afterward a + CONNECT `_. Afterward a TLS connection will be established with the destination and your request will be sent. * HTTPS proxy + HTTP destination A TLS connection will be established to the proxy and later your request will be forwarded with the `absolute URI - `_. + `_. * HTTPS proxy + HTTPS destination A TLS-in-TLS tunnel will be established. An initial TLS connection will be established to the proxy, then an `HTTP CONNECT - `_ will be sent to + `_ will be sent to establish a TCP connection to the destination and finally a second TLS connection will be established to the destination. You can customize the :class:`ssl.SSLContext` used for the proxy TLS connection through the @@ -180,7 +222,7 @@ urllib3's behavior will be different depending on your proxy and destination: class. For HTTPS proxies we also support forwarding your requests to HTTPS destinations with -an `absolute URI `_ if the +an `absolute URI `_ if the ``use_forwarding_for_https`` argument is set to ``True``. We strongly recommend you **only use this option with trusted or corporate proxies** as the proxy will have full visibility of your requests. @@ -200,7 +242,7 @@ starts with ``http://`` instead of ``https://``: # Do this: http = urllib3.ProxyManager("http://...") - + # Not this: http = urllib3.ProxyManager("https://...") @@ -218,11 +260,11 @@ and not ``https://``: $ env | grep "_PROXY" HTTP_PROXY=http://127.0.0.1:8888 HTTPS_PROXY=https://127.0.0.1:8888 # <--- This setting is the problem! - + # Make the fix in your current session and test your script $ export HTTPS_PROXY="http://127.0.0.1:8888" $ python test-proxy.py # This should now pass. - + # Persist your change in your shell 'profile' (~/.bashrc, ~/.profile, ~/.bash_profile, etc) # You may need to logout and log back in to ensure this works across all programs. $ vim ~/.bashrc @@ -264,16 +306,21 @@ SOCKS Proxies For SOCKS, you can use :class:`~contrib.socks.SOCKSProxyManager` to connect to SOCKS4 or SOCKS5 proxies. In order to use SOCKS proxies you will need to install `PySocks `_ or install urllib3 with -the ``socks`` extra:: +the ``socks`` extra: + +.. code-block:: bash python -m pip install urllib3[socks] Once PySocks is installed, you can use -:class:`~contrib.socks.SOCKSProxyManager`:: +:class:`~contrib.socks.SOCKSProxyManager`: - >>> from urllib3.contrib.socks import SOCKSProxyManager - >>> proxy = SOCKSProxyManager('socks5h://localhost:8889/') - >>> proxy.request('GET', 'http://google.com/') +.. code-block:: python + + from urllib3.contrib.socks import SOCKSProxyManager + + proxy = SOCKSProxyManager("socks5h://localhost:8889/") + proxy.request("GET", "https://google.com/") .. note:: It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in @@ -290,12 +337,17 @@ Instead of using `certifi `_ you can provide your own certificate authority bundle. This is useful for cases where you've generated your own certificates or when you're using a private certificate authority. Just provide the full path to the certificate bundle when creating a -:class:`~poolmanager.PoolManager`:: +:class:`~poolmanager.PoolManager`: - >>> import urllib3 - >>> http = urllib3.PoolManager( - ... cert_reqs='CERT_REQUIRED', - ... ca_certs='/path/to/your/certificate_bundle') +.. code-block:: python + + import urllib3 + + http = urllib3.PoolManager( + cert_reqs="CERT_REQUIRED", + ca_certs="/path/to/your/certificate_bundle" + ) + resp = http.request("GET", "https://example.com") When you specify your own certificate bundle only requests that can be verified with that bundle will succeed. It's recommended to use a separate @@ -317,20 +369,22 @@ Normally, urllib3 takes care of setting and checking these values for you when you connect to a host by name. However, it's sometimes useful to set a connection's expected Host header and certificate hostname (subject), especially when you are connecting without using name resolution. For example, -you could connect to a server by IP using HTTPS like so:: - - >>> import urllib3 - >>> pool = urllib3.HTTPSConnectionPool( - ... "10.0.0.10", - ... assert_hostname="example.org", - ... server_hostname="example.org" - ... ) - >>> pool.urlopen( - ... "GET", - ... "/", - ... headers={"Host": "example.org"}, - ... assert_same_host=False - ... ) +you could connect to a server by IP using HTTPS like so: + +.. code-block:: python + + import urllib3 + + pool = urllib3.HTTPSConnectionPool( + "104.154.89.105", + server_hostname="badssl.com" + ) + pool.request( + "GET", + "/", + headers={"Host": "badssl.com"}, + assert_same_host=False + ) Note that when you use a connection in this way, you must specify @@ -341,6 +395,25 @@ address that you would like to use. The IP may be for a private interface, or you may want to use a specific host under round-robin DNS. +.. _assert_hostname: + +Verifying TLS against a different host +-------------------------------------- + +If the server you're connecting to presents a different certificate than the +hostname or the SNI hostname, you can use ``assert_hostname``: + +.. code-block:: python + + import urllib3 + + pool = urllib3.HTTPSConnectionPool( + "wrong.host.badssl.com", + assert_hostname="badssl.com", + ) + pool.request("GET", "/") + + .. _ssl_client: Client Certificates @@ -349,24 +422,62 @@ Client Certificates You can also specify a client certificate. This is useful when both the server and the client need to verify each other's identity. Typically these certificates are issued from the same authority. To use a client certificate, -provide the full path when creating a :class:`~poolmanager.PoolManager`:: +provide the full path when creating a :class:`~poolmanager.PoolManager`: + +.. code-block:: python - >>> http = urllib3.PoolManager( - ... cert_file='/path/to/your/client_cert.pem', - ... cert_reqs='CERT_REQUIRED', - ... ca_certs='/path/to/your/certificate_bundle') + http = urllib3.PoolManager( + cert_file="/path/to/your/client_cert.pem", + cert_reqs="CERT_REQUIRED", + ca_certs="/path/to/your/certificate_bundle" + ) If you have an encrypted client certificate private key you can use -the ``key_password`` parameter to specify a password to decrypt the key. :: +the ``key_password`` parameter to specify a password to decrypt the key. - >>> http = urllib3.PoolManager( - ... cert_file='/path/to/your/client_cert.pem', - ... cert_reqs='CERT_REQUIRED', - ... key_file='/path/to/your/client.key', - ... key_password='keyfile_password') +.. code-block:: python + + http = urllib3.PoolManager( + cert_file="/path/to/your/client_cert.pem", + cert_reqs="CERT_REQUIRED", + key_file="/path/to/your/client.key", + key_password="keyfile_password" + ) If your key isn't encrypted the ``key_password`` parameter isn't required. +TLS minimum and maximum versions +-------------------------------- + +When the configured TLS versions by urllib3 aren't compatible with the TLS versions that +the server is willing to use you'll likely see an error like this one: + +.. code-block:: + + SSLError(1, '[SSL: UNSUPPORTED_PROTOCOL] unsupported protocol (_ssl.c:1124)') + +Starting in v2.0 by default urllib3 uses TLS 1.2 and later so servers that only support TLS 1.1 +or earlier will not work by default with urllib3. + +To fix the issue you'll need to use the ``ssl_minimum_version`` option along with the `TLSVersion enum`_ +in the standard library ``ssl`` module to configure urllib3 to accept a wider range of TLS versions. + +For the best security it's a good idea to set this value to the version of TLS that's being used by the +server. For example if the server requires TLS 1.0 you'd configure urllib3 like so: + +.. code-block:: python + + import ssl + import urllib3 + + http = urllib3.PoolManager( + ssl_minimum_version=ssl.TLSVersion.TLSv1 + ) + # This request works! + resp = http.request("GET", "https://tls-v1-0.badssl.com:1010") + +.. _TLSVersion enum: https://docs.python.org/3/library/ssl.html#ssl.TLSVersion + .. _ssl_mac: .. _certificate_validation_and_mac_os_x: @@ -397,75 +508,28 @@ be resolved in different ways. This happens when a request is made to an HTTPS URL without certificate verification enabled. Follow the :ref:`certificate verification ` guide to resolve this warning. -* :class:`~exceptions.InsecurePlatformWarning` - This happens on Python 2 platforms that have an outdated :mod:`ssl` module. - These older :mod:`ssl` modules can cause some insecure requests to succeed - where they should fail and secure requests to fail where they should - succeed. Follow the :ref:`pyOpenSSL ` guide to resolve this - warning. - -.. _sni_warning: - -* :class:`~exceptions.SNIMissingWarning` - This happens on Python 2 versions older than 2.7.9. These older versions - lack `SNI `_ support. - This can cause servers to present a certificate that the client thinks is - invalid. Follow the :ref:`pyOpenSSL ` guide to resolve this - warning. .. _disable_ssl_warnings: Making unverified HTTPS requests is **strongly** discouraged, however, if you understand the risks and wish to disable these warnings, you can use :func:`~urllib3.disable_warnings`: -.. code-block:: pycon +.. code-block:: python - >>> import urllib3 - >>> urllib3.disable_warnings() + import urllib3 + + urllib3.disable_warnings() Alternatively you can capture the warnings with the standard :mod:`logging` module: -.. code-block:: pycon +.. code-block:: python - >>> logging.captureWarnings(True) + logging.captureWarnings(True) Finally, you can suppress the warnings at the interpreter level by setting the ``PYTHONWARNINGS`` environment variable or by using the `-W flag `_. -Google App Engine ------------------ - -urllib3 supports `Google App Engine `_ with -some caveats. - -If you're using the `Flexible environment -`_, you do not have to do -any configuration- urllib3 will just work. However, if you're using the -`Standard environment `_ then -you either have to use :mod:`urllib3.contrib.appengine`'s -:class:`~urllib3.contrib.appengine.AppEngineManager` or use the `Sockets API -`_ - -To use :class:`~urllib3.contrib.appengine.AppEngineManager`: - -.. code-block:: pycon - - >>> from urllib3.contrib.appengine import AppEngineManager - >>> http = AppEngineManager() - >>> http.request('GET', 'https://google.com/') - -To use the Sockets API, add the following to your app.yaml and use -:class:`~urllib3.poolmanager.PoolManager` as usual: - -.. code-block:: yaml - - env_variables: - GAE_USE_SOCKETS_HTTPLIB : 'true' - -For more details on the limitations and gotchas, see -:mod:`urllib3.contrib.appengine`. - Brotli Encoding --------------- @@ -481,15 +545,50 @@ You may also request the package be installed via the ``urllib3[brotli]`` extra: Here's an example using brotli encoding via the ``Accept-Encoding`` header: -.. code-block:: pycon +.. code-block:: python + + import urllib3 + + urllib3.request( + "GET", + "https://www.google.com/", + headers={"Accept-Encoding": "br"} + ) + +Zstandard Encoding +------------------ + +`Zstandard `_ +is a compression algorithm created by Facebook with better compression +than brotli, gzip and deflate (see `benchmarks `_) +and is supported by urllib3 if the `zstandard package `_ is installed. +You may also request the package be installed via the ``urllib3[zstd]`` extra: + +.. code-block:: bash + + $ python -m pip install urllib3[zstd] + +.. note:: + + Zstandard support in urllib3 requires using v0.18.0 or later of the ``zstandard`` package. + If the version installed is less than v0.18.0 then Zstandard support won't be enabled. + +Here's an example using zstd encoding via the ``Accept-Encoding`` header: + +.. code-block:: python + + import urllib3 + + urllib3.request( + "GET", + "https://www.facebook.com/", + headers={"Accept-Encoding": "zstd"} + ) - >>> from urllib3 import PoolManager - >>> http = PoolManager() - >>> http.request('GET', 'https://www.google.com/', headers={'Accept-Encoding': 'br'}) Decrypting Captured TLS Sessions with Wireshark ----------------------------------------------- -Python 3.8 and higher support logging of TLS pre-master secrets. +Python supports logging of TLS pre-master secrets. With these secrets tools like `Wireshark `_ can decrypt captured network traffic. @@ -501,3 +600,34 @@ To enable this simply define environment variable `SSLKEYLOGFILE`: Then configure the key logfile in `Wireshark `_, see `Wireshark TLS Decryption `_ for instructions. + +Custom SSL Contexts +------------------- + +You can exercise fine-grained control over the urllib3 SSL configuration by +providing a :class:`ssl.SSLContext ` object. For purposes +of compatibility, we recommend you obtain one from +:func:`~urllib3.util.create_urllib3_context`. + +Once you have a context object, you can mutate it to achieve whatever effect +you'd like. For example, the code below loads the default SSL certificates, sets +the :data:`ssl.OP_ENABLE_MIDDLEBOX_COMPAT` +flag that isn't set by default, and then makes a HTTPS request: + +.. code-block:: python + + import ssl + + from urllib3 import PoolManager + from urllib3.util import create_urllib3_context + + ctx = create_urllib3_context() + ctx.load_default_certs() + ctx.options |= ssl.OP_ENABLE_MIDDLEBOX_COMPAT + + with PoolManager(ssl_context=ctx) as pool: + pool.request("GET", "https://www.google.com/") + +Note that this is different from passing an ``options`` argument to +:func:`~urllib3.util.create_urllib3_context` because we don't overwrite +the default options: we only add a new one. diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..26a877a --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1,5 @@ +========= +Changelog +========= + +.. include:: ../CHANGES.rst diff --git a/docs/conf.py b/docs/conf.py index 65e693d..0ab1b3a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations import os import sys @@ -11,22 +11,17 @@ root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, root_path) -# Mock some expensive/platform-specific modules so build will work. -# (https://read-the-docs.readthedocs.io/en/latest/faq.html#\ -# i-get-import-errors-on-libraries-that-depend-on-c-modules) -import mock +# https://docs.readthedocs.io/en/stable/builds.html#build-environment +if "READTHEDOCS" in os.environ: + import glob + if glob.glob("../changelog/*.*.rst"): + print("-- Found changes; running towncrier --", flush=True) + import subprocess -class MockModule(mock.Mock): - @classmethod - def __getattr__(cls, name): - return MockModule() - - -MOCK_MODULES = ("ntlm",) - -sys.modules.update((mod_name, MockModule()) for mod_name in MOCK_MODULES) - + subprocess.run( + ["towncrier", "--yes", "--date", "not released yet"], cwd="..", check=True + ) import urllib3 @@ -37,10 +32,18 @@ def __getattr__(cls, name): # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ "sphinx.ext.autodoc", + "sphinx_copybutton", "sphinx.ext.doctest", "sphinx.ext.intersphinx", + "sphinxext.opengraph", ] +# Open Graph metadata +ogp_title = "urllib3 documentation" +ogp_type = "website" +ogp_social_cards = {"image": "images/logo.png", "line_color": "#F09837"} +ogp_description = "urllib3 is a user-friendly HTTP client library for Python." + # Test code blocks only when explicitly specified doctest_test_doctest_blocks = "" @@ -55,7 +58,7 @@ def __getattr__(cls, name): # General information about the project. project = "urllib3" -copyright = "{year}, Andrey Petrov".format(year=date.today().year) +copyright = f"{date.today().year}, Andrey Petrov" # The short X.Y version. version = urllib3.__version__ @@ -69,6 +72,9 @@ def __getattr__(cls, name): # The name of the Pygments (syntax highlighting) style to use. pygments_style = "friendly" +# The base URL with a proper language and version. +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "/") + # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = "furo" @@ -78,8 +84,8 @@ def __getattr__(cls, name): html_theme_options = { "announcement": """ - Support urllib3 on GitHub Sponsors + href=\"https://opencollective.com/urllib3/updates/urllib3-is-fundraising-for-http-2-support\"> + urllib3 is fundraising for HTTP/2 support! """, "sidebar_hide_name": True, @@ -88,3 +94,33 @@ def __getattr__(cls, name): } intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} + +# Show typehints as content of the function or method +autodoc_typehints = "description" + +# Warn about all references to unknown targets +nitpicky = True +# Except for these ones, which we expect to point to unknown targets: +nitpick_ignore = [ + ("py:class", "_TYPE_SOCKS_OPTIONS"), + ("py:class", "_TYPE_SOCKET_OPTIONS"), + ("py:class", "_TYPE_TIMEOUT"), + ("py:class", "_TYPE_FIELD_VALUE"), + ("py:class", "_TYPE_BODY"), + ("py:class", "_HttplibHTTPResponse"), + ("py:class", "_HttplibHTTPMessage"), + ("py:class", "TracebackType"), + ("py:class", "email.errors.MessageDefect"), + ("py:class", "MessageDefect"), + ("py:class", "http.client.HTTPMessage"), + ("py:class", "RequestHistory"), + ("py:class", "SSLTransportType"), + ("py:class", "VerifyMode"), + ("py:class", "_ssl._SSLContext"), + ("py:class", "urllib3._collections.HTTPHeaderDict"), + ("py:class", "urllib3._collections.RecentlyUsedContainer"), + ("py:class", "urllib3._request_methods.RequestMethods"), + ("py:class", "urllib3.contrib.socks._TYPE_SOCKS_OPTIONS"), + ("py:class", "urllib3.util.timeout._TYPE_DEFAULT"), + ("py:class", "BaseHTTPConnection"), +] diff --git a/docs/contributing.rst b/docs/contributing.rst index a4c97ff..b4c3ac3 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -13,8 +13,10 @@ If you wish to add a new feature or fix a bug: to start making your changes. #. Write a test which shows that the bug was fixed or that the feature works as expected. -#. Format your changes with black using command `$ nox -rs format` and lint your - changes using command `nox -rs lint`. +#. Format your changes with black using command ``nox -rs format`` and lint your + changes using command ``nox -rs lint``. +#. Add a `changelog entry + `__. #. Send a pull request and bug the maintainer until it gets merged and published. @@ -34,18 +36,21 @@ We use some external dependencies, multiple interpreters and code coverage analysis while running test suite. Our ``noxfile.py`` handles much of this for you:: - $ nox --reuse-existing-virtualenvs --sessions test-2.7 test-3.7 + $ nox --reuse-existing-virtualenvs --sessions test-3.12 test-pypy3.10 [ Nox will create virtualenv if needed, install the specified dependencies, and run the commands in order.] - nox > Running session test-2.7 - ....... - ....... - nox > Session test-2.7 was successful. - ....... - ....... - nox > Running session test-3.7 - ....... - ....... - nox > Session test-3.7 was successful. + + +Note that for nox to test different interpreters, the interpreters must be on the +``PATH`` first. Check with ``which`` to see if the interpreter is on the ``PATH`` +like so:: + + + $ which python3.12 + ~/.pyenv/versions/3.12.1/bin/python3.12 + + $ which pypy3.10 + ~/.pyenv/versions/pypy3.10-7.3.13/bin/pypy3.10 + There is also a nox command for running all of our tests and multiple python versions.:: @@ -61,13 +66,12 @@ suite:: [ Nox will create virtualenv if needed, install the specified dependencies, and run the commands in order.] ....... ....... - nox > Session test-2.7 was successful. - nox > Session test-3.6 was successful. - nox > Session test-3.7 was successful. - nox > Session test-3.8 was successful. nox > Session test-3.9 was successful. nox > Session test-3.10 was successful. nox > Session test-3.11 was successful. + nox > Session test-3.12 was successful. + nox > Session test-3.13 was successful. + nox > Session test-3.14 was successful. nox > Session test-pypy was successful. Our test suite `runs continuously on GitHub Actions @@ -75,13 +79,13 @@ Our test suite `runs continuously on GitHub Actions To run specific tests or quickly re-run without nox recreating the env, do the following:: - $ nox --reuse-existing-virtualenvs --sessions test-3.8 -- pyTestArgument1 pyTestArgument2 pyTestArgumentN + $ nox --reuse-existing-virtualenvs --sessions test-3.13 -- pyTestArgument1 pyTestArgument2 pyTestArgumentN [ Nox will create virtualenv, install the specified dependencies, and run the commands in order.] - nox > Running session test-3.8 - nox > Re-using existing virtual environment at .nox/test-3-8. + nox > Running session test-3.13 + nox > Re-using existing virtual environment at .nox/test-3-13. ....... ....... - nox > Session test-3.8 was successful. + nox > Session test-3.13 was successful. After the ``--`` indicator, any arguments will be passed to pytest. To specify an exact test case the following syntax also works: @@ -128,6 +132,23 @@ This program is an experiment so if you have positive or negative feedback on th Note that this program isn't a "bug bounty" program, we don't distribute funds to reporters of bugs or security vulnerabilities at this time. +Running local proxies +--------------------- + +If the feature you are developing involves a proxy, you can rely on scripts we have developed to run a proxy locally. + +Run an HTTP proxy locally: + +.. code-block:: bash + + $ python -m dummyserver.proxy + +Run an HTTPS proxy locally: + +.. code-block:: bash + + $ python -m dummyserver.https_proxy + Contributing to documentation ----------------------------- @@ -170,7 +191,8 @@ A release candidate can be created by any contributor. - Announce intent to release on Discord, see if anyone wants to include last minute changes. -- Update ``urllib3/_version.py`` with the proper version number +- Run ``towncrier build`` to update ``CHANGES.rst`` with the release notes, adjust as + necessary. - Commit the changes to a ``release-X.Y.Z`` branch. - Create a pull request and append ``&expand=1&template=release.md`` to the URL before submitting in order to include our release checklist in the pull request description. diff --git a/docs/index.rst b/docs/index.rst index d6edf3c..1407431 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,12 +6,14 @@ urllib3 :maxdepth: 3 For Enterprise - v2-roadmap + Community Discord + v2-migration-guide sponsors user-guide advanced-usage reference/index contributing + changelog urllib3 is a powerful, *user-friendly* HTTP client for Python. :ref:`Much of the Python ecosystem already uses ` urllib3 and you should too. @@ -24,21 +26,20 @@ standard libraries: - Client-side TLS/SSL verification. - File uploads with multipart encoding. - Helpers for retrying requests and dealing with HTTP redirects. -- Support for gzip, deflate, and brotli encoding. +- Support for gzip, deflate, brotli, and zstd encoding. - Proxy support for HTTP and SOCKS. - 100% test coverage. urllib3 is powerful and easy to use: -.. code-block:: python +.. code-block:: pycon - >>> import urllib3 - >>> http = urllib3.PoolManager() - >>> r = http.request('GET', 'http://httpbin.org/robots.txt') - >>> r.status - 200 - >>> r.data - 'User-agent: *\nDisallow: /deny\n' + >>> import urllib3 + >>> resp = urllib3.request("GET", "https://httpbin.org/robots.txt") + >>> resp.status + 200 + >>> resp.data + b"User-agent: *\nDisallow: /deny\n" For Enterprise -------------- @@ -87,7 +88,6 @@ Alternatively, you can grab the latest source code from `GitHub `_ is a Python distribution for the browser and Node.js based on WebAssembly and `Emscripten `_. +This technology also underpins the `PyScript framework `_ and `Jupyterlite `_, so should work in those environments too. + +Starting in version 2.2.0 urllib3 supports being used in a Pyodide runtime utilizing +the `JavaScript fetch API `_ +or falling back on `XMLHttpRequest `_ +if the fetch API isn't available (such as when cross-origin isolation +isn't active). This means you can use Python libraries to make HTTP requests from your browser! + +Because urllib3's Emscripten support is API-compatible, this means that +libraries that depend on urllib3 may now be usable from Emscripten and Pyodide environments, too. + + .. warning:: + + **Support for Emscripten and Pyodide is experimental**. Report all bugs to the `urllib3 issue tracker `_. + Currently Node.js support is very experimental - see the description below. + +It's recommended to `run Pyodide in a Web Worker `_ +in order to take full advantage of features like the fetch API which enables streaming of HTTP response bodies. + +Getting started +--------------- + +Using urllib3 with Pyodide means you need to `get started with Pyodide first `_. +The Pyodide project provides a `useful online REPL `_ to try in your browser without +any setup or installation to test out the code examples below. + +One minor note - when running Pyodide code from JavaScript, if you use ``pyodide.runPythonAsync`` rather +than ``pyodide.runPython``, urllib3 can sometimes run more efficiently. It is generally always worth using +``runPythonAsync``. + +urllib3's Emscripten support is automatically enabled if ``sys.platform`` is ``"emscripten"``, so no setup is required beyond installation and importing the module. + +urllib3 is packaged with the default Pyodide build, so you should be able to use it as normal. + + .. code-block:: python + + import urllib3 + resp = urllib3.request("GET", "https://httpbin.org/anything") + + print(resp.status) # 200 + print(resp.headers) # HTTPHeaderDict(...) + print(resp.json()) # {"headers": {"Accept": "*/*", ...}, ...} + +Because `Requests `_ is built on urllib3, Requests also works out of the box: + + .. code-block:: python + + import requests + resp = requests.request("GET", "https://httpbin.org/anything") + + print(resp.status_code) # 200 + print(resp.headers) + +Features +-------- + +Because we use JavaScript APIs under the hood, it's not possible to use all of urllib3 features. +Features which are usable with Emscripten support are: + +* Requests over HTTP and HTTPS +* Timeouts +* Retries +* Streaming (with Web Workers and Cross-Origin Isolation) +* Redirects +* Decompressing response bodies + +Features which don't work with Emscripten: + +* Proxies, both forwarding and tunneling +* Customizing TLS and certificates (uses browsers' configuration) +* Configuring low-level socket options or source address + +Streaming with Web Workers +-------------------------- +To access the fetch API and do HTTP response streaming with urllib3 +you must be running the code within a Web Worker and set specific HTTP headers +for the serving website to enable `Cross-Origin Isolation `_. + +You can verify whether a given environment is cross-origin isolated by evaluating the global ``crossOriginIsolated`` JavaScript property. + +Node.js support +--------------- +Node.js support uses a relatively new feature in WebAssembly known as JavaScript Promise Integration. +To use urllib3 in Node.js, you need to use Node.js version 20 or newer and may need to call Node.js with +the ``--experimental-wasm-stack-switching`` command line parameter. \ No newline at end of file diff --git a/docs/reference/contrib/index.rst b/docs/reference/contrib/index.rst index 94cac5e..bb8bc08 100644 --- a/docs/reference/contrib/index.rst +++ b/docs/reference/contrib/index.rst @@ -6,8 +6,6 @@ prime time or that require optional third-party dependencies. .. toctree:: - appengine - ntlmpool + emscripten pyopenssl - securetransport socks diff --git a/docs/reference/contrib/ntlmpool.rst b/docs/reference/contrib/ntlmpool.rst deleted file mode 100644 index 08277e9..0000000 --- a/docs/reference/contrib/ntlmpool.rst +++ /dev/null @@ -1,7 +0,0 @@ -NTLM Authentication -=================== - -.. automodule:: urllib3.contrib.ntlmpool - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/reference/contrib/pyopenssl.rst b/docs/reference/contrib/pyopenssl.rst index c727bcd..03cda71 100644 --- a/docs/reference/contrib/pyopenssl.rst +++ b/docs/reference/contrib/pyopenssl.rst @@ -1,8 +1,5 @@ PyOpenSSL ========= -.. warning:: - DEPRECATED: This module is deprecated and will be removed in a future 2.x release. - Read more in this `issue `_. .. automodule:: urllib3.contrib.pyopenssl :members: diff --git a/docs/reference/contrib/securetransport.rst b/docs/reference/contrib/securetransport.rst deleted file mode 100644 index 12a6ddc..0000000 --- a/docs/reference/contrib/securetransport.rst +++ /dev/null @@ -1,28 +0,0 @@ -macOS SecureTransport -===================== - -`SecureTranport `_ -support for urllib3 via ctypes. - -This makes platform-native TLS available to urllib3 users on macOS without the -use of a compiler. This is an important feature because the Python Package -Index is moving to become a TLSv1.2-or-higher server, and the default OpenSSL -that ships with macOS is not capable of doing TLSv1.2. The only way to resolve -this is to give macOS users an alternative solution to the problem, and that -solution is to use SecureTransport. - -We use ctypes here because this solution must not require a compiler. That's -because Pip is not allowed to require a compiler either. - -This code is a bastardised version of the code found in Will Bond's -`oscrypto `_ library. An enormous debt -is owed to him for blazing this trail for us. For that reason, this code -should be considered to be covered both by urllib3's license and by -`oscrypto's `_. - -To use this module, simply import and inject it: - -.. code-block:: python - - import urllib3.contrib.securetransport - urllib3.contrib.securetransport.inject_into_urllib3() diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 2d21e3c..582b8f7 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -3,12 +3,12 @@ API Reference .. toctree:: + urllib3.request urllib3.poolmanager urllib3.connectionpool urllib3.connection urllib3.exceptions urllib3.response urllib3.fields - urllib3.request urllib3.util contrib/index diff --git a/docs/reference/urllib3.connection.rst b/docs/reference/urllib3.connection.rst index 472ddc5..8bd7b25 100644 --- a/docs/reference/urllib3.connection.rst +++ b/docs/reference/urllib3.connection.rst @@ -1,6 +1,8 @@ Connections =========== +.. automodule:: urllib3.connection + .. autoclass:: urllib3.connection.HTTPConnection :members: :exclude-members: putrequest @@ -9,3 +11,7 @@ Connections .. autoclass:: urllib3.connection.HTTPSConnection :members: :show-inheritance: + +.. autoclass:: urllib3.connection.ProxyConfig + :members: + :show-inheritance: diff --git a/docs/reference/urllib3.connectionpool.rst b/docs/reference/urllib3.connectionpool.rst index 9b10144..45dee02 100644 --- a/docs/reference/urllib3.connectionpool.rst +++ b/docs/reference/urllib3.connectionpool.rst @@ -1,6 +1,8 @@ Connection Pools ================ +.. automodule:: urllib3.connectionpool + .. autoclass:: urllib3.HTTPConnectionPool :members: :undoc-members: @@ -15,3 +17,5 @@ Connection Pools :members: :undoc-members: :show-inheritance: + +.. autofunction:: urllib3.connectionpool.connection_from_url diff --git a/docs/reference/urllib3.exceptions.rst b/docs/reference/urllib3.exceptions.rst index f139f5e..84603a7 100644 --- a/docs/reference/urllib3.exceptions.rst +++ b/docs/reference/urllib3.exceptions.rst @@ -1,7 +1,9 @@ -Exceptions -========== +Exceptions and Warnings +======================= .. automodule:: urllib3.exceptions :members: :undoc-members: :show-inheritance: + +.. autofunction:: urllib3.disable_warnings diff --git a/docs/reference/urllib3.poolmanager.rst b/docs/reference/urllib3.poolmanager.rst index d796daf..200f140 100644 --- a/docs/reference/urllib3.poolmanager.rst +++ b/docs/reference/urllib3.poolmanager.rst @@ -5,8 +5,14 @@ Pool Manager :members: :undoc-members: :show-inheritance: + :inherited-members: .. autoclass:: urllib3.ProxyManager :members: :undoc-members: :show-inheritance: + +.. autoclass:: urllib3.poolmanager.PoolKey + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/reference/urllib3.request.rst b/docs/reference/urllib3.request.rst index 39a1236..ea39de8 100644 --- a/docs/reference/urllib3.request.rst +++ b/docs/reference/urllib3.request.rst @@ -1,7 +1,4 @@ -Request Methods -=============== +urllib3.request() +================= -.. automodule:: urllib3.request - :members: - :undoc-members: - :show-inheritance: +.. autofunction:: urllib3.request diff --git a/docs/reference/urllib3.response.rst b/docs/reference/urllib3.response.rst index aa87d34..d00b8af 100644 --- a/docs/reference/urllib3.response.rst +++ b/docs/reference/urllib3.response.rst @@ -4,10 +4,20 @@ Response and Decoders Response -------- +.. autoclass:: urllib3.response.BaseHTTPResponse + :members: + :undoc-members: + :show-inheritance: + .. autoclass:: urllib3.response.HTTPResponse :members: :undoc-members: :show-inheritance: + :inherited-members: json + + .. autoattribute:: auto_close + .. autoattribute:: status + .. autoattribute:: headers Decoders -------- @@ -19,4 +29,5 @@ representation. .. autoclass:: urllib3.response.BrotliDecoder .. autoclass:: urllib3.response.DeflateDecoder .. autoclass:: urllib3.response.GzipDecoder +.. autoclass:: urllib3.response.ZstdDecoder .. autoclass:: urllib3.response.MultiDecoder diff --git a/docs/reference/urllib3.util.rst b/docs/reference/urllib3.util.rst index cb51215..d837b85 100644 --- a/docs/reference/urllib3.util.rst +++ b/docs/reference/urllib3.util.rst @@ -4,11 +4,11 @@ Utilities Useful methods for working with :mod:`http.client`, completely decoupled from code specific to **urllib3**. -At the very core, just like its predecessors, :mod:`urllib3` is built on top of +At the very core, just like its predecessors, urllib3 is built on top of :mod:`http.client` -- the lowest level HTTP library included in the Python standard library. -To aid the limited functionality of the :mod:`http.client` module, :mod:`urllib3` +To aid the limited functionality of the :mod:`http.client` module, urllib3 provides various helper methods which are used with the higher level components but can also be used independently. diff --git a/docs/requirements.txt b/docs/requirements.txt index 9cd179b..3edab4d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,6 @@ -r ../dev-requirements.txt -sphinx>3.0.0 -requests>=2 +sphinx>=7.2.6 +requests furo +sphinx-copybutton +sphinxext-opengraph \ No newline at end of file diff --git a/docs/sponsors.rst b/docs/sponsors.rst index a08b536..33e9ce7 100644 --- a/docs/sponsors.rst +++ b/docs/sponsors.rst @@ -7,45 +7,6 @@ benefits from this library. Your contribution will go towards adding new features to urllib3 and making sure all functionality continues to meet our high quality standards. - -v2.0 Sponsor Perks ------------------- - -.. important:: - - `Get in contact `_ for additional - details on sponsorship and perks before making a contribution - through `GitHub Sponsors `_ if you have questions. - - -Silver v2.0 Sponsor Perks -~~~~~~~~~~~~~~~~~~~~~~~~~ - -- Your organization name and URL permanently added - to the **Sponsors and Grants** section below -- Thank you within the v2.0 release announcement - and on Twitter from urllib3 maintainers - -➤ `Contribute to the "Silver v2.0 Sponsor" tier `_ -on Open Collective. - - -Gold v2.0 Sponsor Perks -~~~~~~~~~~~~~~~~~~~~~~~~ - -- Organization logo and URL listed on top of the v2.0 Roadmap -- Call with one or more urllib3 maintainer(s) to discuss - the v2.0 release and how it impacts your organization -- Your organization will be thanked within the v2.0 release - announcement, within all blog posts and public updates related to v2.0 - development, and multiple thank-you's on Twitter from - urllib3 maintainers throughout v2.0 development -- All perks from the **Silver v2.0 Sponsors Perks** above - -➤ `Contribute to the "Gold v2.0 Sponsor" tier `_ -on Open Collective. - - Sponsors and Grants ------------------- @@ -61,6 +22,8 @@ We also welcome sponsorship in the form of time. We greatly appreciate companies who encourage employees to contribute on an ongoing basis during their work hours. Let us know and we'll be glad to add you to our sponsors list. +* `Spotify `_ (June 2nd, 2022) + * `GitCoin Grants `_ (2019-2020), sponsored `@sethmlarson `_ and `@pquentin `_ diff --git a/docs/user-guide.rst b/docs/user-guide.rst index abd5323..5c78c8a 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -18,43 +18,64 @@ Making Requests First things first, import the urllib3 module: -.. code-block:: pycon +.. code-block:: python - >>> import urllib3 + import urllib3 You'll need a :class:`~poolmanager.PoolManager` instance to make requests. This object handles all of the details of connection pooling and thread safety so that you don't have to: -.. code-block:: pycon +.. code-block:: python - >>> http = urllib3.PoolManager() + http = urllib3.PoolManager() -To make a request use :meth:`~poolmanager.PoolManager.request`: +To make a request use :meth:`~urllib3.PoolManager.request`: -.. code-block:: pycon +.. code-block:: python - >>> r = http.request('GET', 'http://httpbin.org/robots.txt') - >>> r.data - b'User-agent: *\nDisallow: /deny\n' + import urllib3 + + # Creating a PoolManager instance for sending requests. + http = urllib3.PoolManager() + + # Sending a GET request and getting back response as HTTPResponse object. + resp = http.request("GET", "https://httpbin.org/robots.txt") + + # Print the returned data. + print(resp.data) + # b"User-agent: *\nDisallow: /deny\n" ``request()`` returns a :class:`~response.HTTPResponse` object, the :ref:`response_content` section explains how to handle various responses. -You can use :meth:`~poolmanager.PoolManager.request` to make requests using any +You can use :meth:`~urllib3.PoolManager.request` to make requests using any HTTP verb: -.. code-block:: pycon +.. code-block:: python - >>> r = http.request( - ... 'POST', - ... 'http://httpbin.org/post', - ... fields={'hello': 'world'} - ... ) + import urllib3 + + http = urllib3.PoolManager() + resp = http.request( + "POST", + "https://httpbin.org/post", + fields={"hello": "world"} # Add custom form fields + ) + + print(resp.data) + # b"{\n "form": {\n "hello": "world"\n }, ... } The :ref:`request_data` section covers sending other kinds of requests data, including JSON, files, and binary data. +.. note:: For quick scripts and experiments you can also use a top-level ``urllib3.request()``. + It uses a module-global ``PoolManager`` instance. + Because of that, its side effects could be shared across dependencies relying on it. + To avoid side effects, create a new ``PoolManager`` instance and use it instead. + In addition, the method does not accept the low-level ``**urlopen_kw`` keyword arguments. + System CA certificates are loaded on default. + .. _response_content: Response Content @@ -64,28 +85,50 @@ The :class:`~response.HTTPResponse` object provides :attr:`~response.HTTPResponse.status`, :attr:`~response.HTTPResponse.data`, and :attr:`~response.HTTPResponse.headers` attributes: -.. code-block:: pycon +.. code-block:: python - >>> r = http.request('GET', 'http://httpbin.org/ip') - >>> r.status - 200 - >>> r.data - b'{\n "origin": "104.232.115.37"\n}\n' - >>> r.headers - HTTPHeaderDict({'Content-Length': '33', ...}) + import urllib3 + + # Making the request (The request function returns HTTPResponse object) + resp = urllib3.request("GET", "https://httpbin.org/ip") + + print(resp.status) + # 200 + print(resp.data) + # b"{\n "origin": "104.232.115.37"\n}\n" + print(resp.headers) + # HTTPHeaderDict({"Content-Length": "32", ...}) + +.. _json_content: JSON Content ~~~~~~~~~~~~ +JSON content can be loaded by :meth:`~response.HTTPResponse.json` +method of the response: + +.. code-block:: python + + import urllib3 + + resp = urllib3.request("GET", "https://httpbin.org/ip") -JSON content can be loaded by decoding and deserializing the -:attr:`~response.HTTPResponse.data` attribute of the request: + print(resp.json()) + # {"origin": "127.0.0.1"} -.. code-block:: pycon +Alternatively, Custom JSON libraries such as `orjson` can be used to encode data, +retrieve data by decoding and deserializing the :attr:`~response.HTTPResponse.data` +attribute of the request: - >>> import json - >>> r = http.request('GET', 'http://httpbin.org/ip') - >>> json.loads(r.data.decode('utf-8')) - {'origin': '127.0.0.1'} +.. code-block:: python + + import orjson + import urllib3 + + encoded_data = orjson.dumps({"attribute": "value"}) + resp = urllib3.request(method="POST", url="http://httpbin.org/post", body=encoded_data) + + print(orjson.loads(resp.data)["json"]) + # {'attribute': 'value'} Binary Content ~~~~~~~~~~~~~~ @@ -93,11 +136,14 @@ Binary Content The :attr:`~response.HTTPResponse.data` attribute of the response is always set to a byte string representing the response content: -.. code-block:: pycon +.. code-block:: python - >>> r = http.request('GET', 'http://httpbin.org/bytes/8') - >>> r.data - b'\xaa\xa5H?\x95\xe9\x9b\x11' + import urllib3 + + resp = urllib3.request("GET", "https://httpbin.org/bytes/8") + + print(resp.data) + # b"\xaa\xa5H?\x95\xe9\x9b\x11" .. note:: For larger responses, it's sometimes better to :ref:`stream ` the response. @@ -110,13 +156,22 @@ directly with :class:`~response.HTTPResponse` data. Making these two interfaces together requires using the :attr:`~response.HTTPResponse.auto_close` attribute by setting it to ``False``. By default HTTP responses are closed after reading all bytes, this disables that behavior: -.. code-block:: pycon +.. code-block:: python + + import io + import urllib3 - >>> import io - >>> r = http.request('GET', 'https://example.com', preload_content=False) - >>> r.auto_close = False - >>> for line in io.TextIOWrapper(r): - >>> print(line) + resp = urllib3.request("GET", "https://example.com", preload_content=False) + resp.auto_close = False + + for line in io.TextIOWrapper(resp): + print(line) + # + # + # + # .... + # + # .. _request_data: @@ -126,48 +181,120 @@ Request Data Headers ~~~~~~~ -You can specify headers as a dictionary in the ``headers`` argument in :meth:`~poolmanager.PoolManager.request`: +You can specify headers as a dictionary in the ``headers`` argument in :meth:`~urllib3.PoolManager.request`: + +.. code-block:: python + + import urllib3 + + resp = urllib3.request( + "GET", + "https://httpbin.org/headers", + headers={ + "X-Something": "value" + } + ) + + print(resp.json()["headers"]) + # {"X-Something": "value", ...} + +Or you can use the ``HTTPHeaderDict`` class to create multi-valued HTTP headers: + +.. code-block:: python + + import urllib3 + + # Create an HTTPHeaderDict and add headers + headers = urllib3.HTTPHeaderDict() + headers.add("Accept", "application/json") + headers.add("Accept", "text/plain") + + # Make the request using the headers + resp = urllib3.request( + "GET", + "https://httpbin.org/headers", + headers=headers + ) + + print(resp.json()["headers"]) + # {"Accept": "application/json, text/plain", ...} + +Cookies +~~~~~~~ + +Cookies are specified using the ``Cookie`` header with a string containing +the ``;`` delimited key-value pairs: + +.. code-block:: python + + import urllib3 + + resp = urllib3.request( + "GET", + "https://httpbin.org/cookies", + headers={ + "Cookie": "session=f3efe9db; id=30" + } + ) + + print(resp.json()) + # {"cookies": {"id": "30", "session": "f3efe9db"}} -.. code-block:: pycon +Note that the ``Cookie`` header will be stripped if the server redirects to a +different host. - >>> r = http.request( - ... 'GET', - ... 'http://httpbin.org/headers', - ... headers={ - ... 'X-Something': 'value' - ... } - ... ) - >>> json.loads(r.data.decode('utf-8'))['headers'] - {'X-Something': 'value', ...} +Cookies provided by the server are stored in the ``Set-Cookie`` header: + +.. code-block:: python + + import urllib3 + + resp = urllib3.request( + "GET", + "https://httpbin.org/cookies/set/session/f3efe9db", + redirect=False + ) + + print(resp.headers["Set-Cookie"]) + # session=f3efe9db; Path=/ Query Parameters ~~~~~~~~~~~~~~~~ For ``GET``, ``HEAD``, and ``DELETE`` requests, you can simply pass the arguments as a dictionary in the ``fields`` argument to -:meth:`~poolmanager.PoolManager.request`: +:meth:`~urllib3.PoolManager.request`: + +.. code-block:: python + + import urllib3 -.. code-block:: pycon + resp = urllib3.request( + "GET", + "https://httpbin.org/get", + fields={"arg": "value"} + ) - >>> r = http.request( - ... 'GET', - ... 'http://httpbin.org/get', - ... fields={'arg': 'value'} - ... ) - >>> json.loads(r.data.decode('utf-8'))['args'] - {'arg': 'value'} + print(resp.json()["args"]) + # {"arg": "value"} For ``POST`` and ``PUT`` requests, you need to manually encode query parameters in the URL: -.. code-block:: pycon +.. code-block:: python - >>> from urllib.parse import urlencode - >>> encoded_args = urlencode({'arg': 'value'}) - >>> url = 'http://httpbin.org/post?' + encoded_args - >>> r = http.request('POST', url) - >>> json.loads(r.data.decode('utf-8'))['args'] - {'arg': 'value'} + from urllib.parse import urlencode + import urllib3 + + # Encode the args into url grammar. + encoded_args = urlencode({"arg": "value"}) + + # Create a URL with args encoded. + url = "https://httpbin.org/post?" + encoded_args + resp = urllib3.request("POST", url) + + print(resp.json()["args"]) + # {"arg": "value"} .. _form_data: @@ -177,39 +304,47 @@ Form Data For ``PUT`` and ``POST`` requests, urllib3 will automatically form-encode the dictionary in the ``fields`` argument provided to -:meth:`~poolmanager.PoolManager.request`: +:meth:`~urllib3.PoolManager.request`: -.. code-block:: pycon +.. code-block:: python - >>> r = http.request( - ... 'POST', - ... 'http://httpbin.org/post', - ... fields={'field': 'value'} - ... ) - >>> json.loads(r.data.decode('utf-8'))['form'] - {'field': 'value'} + import urllib3 + + resp = urllib3.request( + "POST", + "https://httpbin.org/post", + fields={"field": "value"} + ) + + print(resp.json()["form"]) + # {"field": "value"} + +.. _json: JSON ~~~~ -You can send a JSON request by specifying the encoded data as the ``body`` -argument and setting the ``Content-Type`` header when calling -:meth:`~poolmanager.PoolManager.request`: +To send JSON in the body of a request, provide the data in the ``json`` argument to +:meth:`~urllib3.PoolManager.request` and urllib3 will automatically encode the data +using the ``json`` module with ``UTF-8`` encoding. +In addition, when ``json`` is provided, the ``"Content-Type"`` in headers is set to +``"application/json"`` if not specified otherwise. + +.. code-block:: python -.. code-block:: pycon + import urllib3 - >>> import json - >>> data = {'attribute': 'value'} - >>> encoded_data = json.dumps(data).encode('utf-8') - >>> r = http.request( - ... 'POST', - ... 'http://httpbin.org/post', - ... body=encoded_data, - ... headers={'Content-Type': 'application/json'} - ... ) - >>> json.loads(r.data.decode('utf-8'))['json'] - {'attribute': 'value'} + resp = urllib3.request( + "POST", + "https://httpbin.org/post", + json={"attribute": "value"}, + headers={"Content-Type": "application/json"} + ) + print(resp.json()) + # {'headers': {'Content-Type': 'application/json', ...}, + # 'data': '{"attribute":"value"}', 'json': {'attribute': 'value'}, ...} + Files & Binary Data ~~~~~~~~~~~~~~~~~~~ @@ -217,49 +352,59 @@ For uploading files using ``multipart/form-data`` encoding you can use the same approach as :ref:`form_data` and specify the file field as a tuple of ``(file_name, file_data)``: -.. code-block:: pycon - - >>> with open('example.txt') as fp: - ... file_data = fp.read() - >>> r = http.request( - ... 'POST', - ... 'http://httpbin.org/post', - ... fields={ - ... 'filefield': ('example.txt', file_data), - ... } - ... ) - >>> json.loads(r.data.decode('utf-8'))['files'] - {'filefield': '...'} +.. code-block:: python + + import urllib3 + + # Reading the text file from local storage. + with open("example.txt") as fp: + file_data = fp.read() + + # Sending the request. + resp = urllib3.request( + "POST", + "https://httpbin.org/post", + fields={ + "filefield": ("example.txt", file_data), + } + ) + + print(resp.json()["files"]) + # {"filefield": "..."} While specifying the filename is not strictly required, it's recommended in order to match browser behavior. You can also pass a third item in the tuple to specify the file's MIME type explicitly: -.. code-block:: pycon +.. code-block:: python - >>> r = http.request( - ... 'POST', - ... 'http://httpbin.org/post', - ... fields={ - ... 'filefield': ('example.txt', file_data, 'text/plain'), - ... } - ... ) + resp = urllib3.request( + "POST", + "https://httpbin.org/post", + fields={ + "filefield": ("example.txt", file_data, "text/plain"), + } + ) For sending raw binary data simply specify the ``body`` argument. It's also recommended to set the ``Content-Type`` header: -.. code-block:: pycon +.. code-block:: python + + import urllib3 + + with open("/home/samad/example.jpg", "rb") as fp: + binary_data = fp.read() - >>> with open('example.jpg', 'rb') as fp: - ... binary_data = fp.read() - >>> r = http.request( - ... 'POST', - ... 'http://httpbin.org/post', - ... body=binary_data, - ... headers={'Content-Type': 'image/jpeg'} - ... ) - >>> json.loads(r.data.decode('utf-8'))['data'] - b'...' + resp = urllib3.request( + "POST", + "https://httpbin.org/post", + body=binary_data, + headers={"Content-Type": "image/jpeg"} + ) + + print(resp.json()["data"]) + # data:application/octet-stream;base64,... .. _ssl: @@ -268,9 +413,9 @@ Certificate Verification .. note:: *New in version 1.25:* - HTTPS connections are now verified by default (``cert_reqs = 'CERT_REQUIRED'``). + HTTPS connections are now verified by default (``cert_reqs = "CERT_REQUIRED"``). -While you can disable certification verification by setting ``cert_reqs = 'CERT_NONE'``, it is highly recommend to leave it on. +While you can disable certification verification by setting ``cert_reqs = "CERT_NONE"``, it is highly recommend to leave it on. Unless otherwise specified urllib3 will try to load the default system certificate stores. The most reliable cross-platform method is to use the `certifi `_ @@ -280,36 +425,37 @@ package which provides Mozilla's root certificate bundle: $ python -m pip install certifi -You can also install certifi along with urllib3 by using the ``secure`` -extra: - -.. code-block:: bash - - $ python -m pip install urllib3[secure] - -.. warning:: If you're using Python 2 you may need additional packages. See the :ref:`section below ` for more details. - Once you have certificates, you can create a :class:`~poolmanager.PoolManager` that verifies certificates when making requests: -.. code-block:: pycon +.. code-block:: python - >>> import certifi - >>> import urllib3 - >>> http = urllib3.PoolManager( - ... cert_reqs='CERT_REQUIRED', - ... ca_certs=certifi.where() - ... ) + import certifi + import urllib3 + + http = urllib3.PoolManager( + cert_reqs="CERT_REQUIRED", + ca_certs=certifi.where() + ) The :class:`~poolmanager.PoolManager` will automatically handle certificate verification and will raise :class:`~exceptions.SSLError` if verification fails: -.. code-block:: pycon +.. code-block:: python + + import certifi + import urllib3 + + http = urllib3.PoolManager( + cert_reqs="CERT_REQUIRED", + ca_certs=certifi.where() + ) - >>> http.request('GET', 'https://google.com') - (No exception) - >>> http.request('GET', 'https://expired.badssl.com') - urllib3.exceptions.SSLError ... + http.request("GET", "https://httpbin.org/") + # (No exception) + + http.request("GET", "https://expired.badssl.com") + # urllib3.exceptions.SSLError ... .. note:: You can use OS-provided certificates if desired. Just specify the full path to the certificate bundle as the ``ca_certs`` argument instead of @@ -317,210 +463,187 @@ verification and will raise :class:`~exceptions.SSLError` if verification fails: at ``/etc/ssl/certs/ca-certificates.crt``. Other operating systems can be `difficult `_. -.. _ssl_py2: - -Certificate Verification in Python 2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Older versions of Python 2 are built with an :mod:`ssl` module that lacks -:ref:`SNI support ` and can lag behind security updates. For these reasons it's recommended to use -`pyOpenSSL `_. - -If you install urllib3 with the ``secure`` extra, all required packages for -certificate verification on Python 2 will be installed: - -.. code-block:: bash - - $ python -m pip install urllib3[secure] - -If you want to install the packages manually, you will need ``pyOpenSSL``, -``cryptography``, ``idna``, and ``certifi``. - -.. note:: If you are not using macOS or Windows, note that `cryptography - `_ requires additional system packages - to compile. See `building cryptography on Linux - `_ - for the list of packages required. - -Once installed, you can tell urllib3 to use pyOpenSSL by using :mod:`urllib3.contrib.pyopenssl`: - -.. code-block:: pycon - - >>> import urllib3.contrib.pyopenssl - >>> urllib3.contrib.pyopenssl.inject_into_urllib3() - -Finally, you can create a :class:`~poolmanager.PoolManager` that verifies -certificates when performing requests: - -.. code-block:: pycon - - >>> import certifi - >>> import urllib3 - >>> http = urllib3.PoolManager( - ... cert_reqs='CERT_REQUIRED', - ... ca_certs=certifi.where() - ... ) - -If you do not wish to use pyOpenSSL, you can simply omit the call to -:func:`urllib3.contrib.pyopenssl.inject_into_urllib3`. urllib3 will fall back -to the standard-library :mod:`ssl` module. You may experience -:ref:`several warnings ` when doing this. - -.. warning:: If you do not use pyOpenSSL, Python must be compiled with ssl - support for certificate verification to work. It is uncommon, but it is - possible to compile Python without SSL support. See this - `StackOverflow thread `_ - for more details. - - If you are on Google App Engine, you must explicitly enable SSL - support in your ``app.yaml``: - - .. code-block:: yaml - - libraries: - - name: ssl - version: latest - Using Timeouts -------------- Timeouts allow you to control how long (in seconds) requests are allowed to run before being aborted. In simple cases, you can specify a timeout as a ``float`` -to :meth:`~poolmanager.PoolManager.request`: +to :meth:`~urllib3.PoolManager.request`: + +.. code-block:: python + + import urllib3 + + resp = urllib3.request( + "GET", + "https://httpbin.org/delay/3", + timeout=4.0 + ) -.. code-block:: pycon + print(type(resp)) + # - >>> http.request( - ... 'GET', 'http://httpbin.org/delay/3', timeout=4.0 - ... ) - - >>> http.request( - ... 'GET', 'http://httpbin.org/delay/3', timeout=2.5 - ... ) - MaxRetryError caused by ReadTimeoutError + # This request will take more time to process than timeout. + urllib3.request( + "GET", + "https://httpbin.org/delay/3", + timeout=2.5 + ) + # MaxRetryError caused by ReadTimeoutError For more granular control you can use a :class:`~util.timeout.Timeout` instance which lets you specify separate connect and read timeouts: -.. code-block:: pycon +.. code-block:: python - >>> http.request( - ... 'GET', - ... 'http://httpbin.org/delay/3', - ... timeout=urllib3.Timeout(connect=1.0) - ... ) - - >>> http.request( - ... 'GET', - ... 'http://httpbin.org/delay/3', - ... timeout=urllib3.Timeout(connect=1.0, read=2.0) - ... ) - MaxRetryError caused by ReadTimeoutError + import urllib3 + + resp = urllib3.request( + "GET", + "https://httpbin.org/delay/3", + timeout=urllib3.Timeout(connect=1.0) + ) + + print(type(resp)) + # + + urllib3.request( + "GET", + "https://httpbin.org/delay/3", + timeout=urllib3.Timeout(connect=1.0, read=2.0) + ) + # MaxRetryError caused by ReadTimeoutError If you want all requests to be subject to the same timeout, you can specify the timeout at the :class:`~urllib3.poolmanager.PoolManager` level: -.. code-block:: pycon +.. code-block:: python - >>> http = urllib3.PoolManager(timeout=3.0) - >>> http = urllib3.PoolManager( - ... timeout=urllib3.Timeout(connect=1.0, read=2.0) - ... ) + import urllib3 + + http = urllib3.PoolManager(timeout=3.0) + + http = urllib3.PoolManager( + timeout=urllib3.Timeout(connect=1.0, read=2.0) + ) You still override this pool-level timeout by specifying ``timeout`` to -:meth:`~poolmanager.PoolManager.request`. +:meth:`~urllib3.PoolManager.request`. Retrying Requests ----------------- urllib3 can automatically retry idempotent requests. This same mechanism also handles redirects. You can control the retries using the ``retries`` parameter -to :meth:`~poolmanager.PoolManager.request`. By default, urllib3 will retry +to :meth:`~urllib3.PoolManager.request`. By default, urllib3 will retry requests 3 times and follow up to 3 redirects. To change the number of retries just specify an integer: -.. code-block:: pycon +.. code-block:: python + + import urllib3 - >>> http.requests('GET', 'http://httpbin.org/ip', retries=10) + urllib3.request("GET", "https://httpbin.org/ip", retries=10) To disable all retry and redirect logic specify ``retries=False``: -.. code-block:: pycon +.. code-block:: python + + import urllib3 - >>> http.request( - ... 'GET', 'http://nxdomain.example.com', retries=False - ... ) - NewConnectionError - >>> r = http.request( - ... 'GET', 'http://httpbin.org/redirect/1', retries=False - ... ) - >>> r.status - 302 + urllib3.request( + "GET", + "https://nxdomain.example.com", + retries=False + ) + # NewConnectionError + + resp = urllib3.request( + "GET", + "https://httpbin.org/redirect/1", + retries=False + ) + + print(resp.status) + # 302 To disable redirects but keep the retrying logic, specify ``redirect=False``: -.. code-block:: pycon +.. code-block:: python - >>> r = http.request( - ... 'GET', 'http://httpbin.org/redirect/1', redirect=False - ... ) - >>> r.status - 302 + resp = urllib3.request( + "GET", + "https://httpbin.org/redirect/1", + redirect=False + ) + + print(resp.status) + # 302 For more granular control you can use a :class:`~util.retry.Retry` instance. This class allows you far greater control of how requests are retried. For example, to do a total of 3 retries, but limit to only 2 redirects: -.. code-block:: pycon +.. code-block:: python - >>> http.request( - ... 'GET', - ... 'http://httpbin.org/redirect/3', - ... retries=urllib3.Retry(3, redirect=2) - ... ) - MaxRetryError + urllib3.request( + "GET", + "https://httpbin.org/redirect/3", + retries=urllib3.Retry(3, redirect=2) + ) + # MaxRetryError You can also disable exceptions for too many redirects and just return the ``302`` response: -.. code-block:: pycon +.. code-block:: python - >>> r = http.request( - ... 'GET', - ... 'http://httpbin.org/redirect/3', - ... retries=urllib3.Retry( - ... redirect=2, raise_on_redirect=False) - ... ) - >>> r.status - 302 + resp = urllib3.request( + "GET", + "https://httpbin.org/redirect/3", + retries=urllib3.Retry( + redirect=2, + raise_on_redirect=False + ) + ) + + print(resp.status) + # 302 If you want all requests to be subject to the same retry policy, you can specify the retry at the :class:`~urllib3.poolmanager.PoolManager` level: -.. code-block:: pycon +.. code-block:: python + + import urllib3 - >>> http = urllib3.PoolManager(retries=False) - >>> http = urllib3.PoolManager( - ... retries=urllib3.Retry(5, redirect=2) - ... ) + http = urllib3.PoolManager(retries=False) + + http = urllib3.PoolManager( + retries=urllib3.Retry(5, redirect=2) + ) You still override this pool-level retry policy by specifying ``retries`` to -:meth:`~poolmanager.PoolManager.request`. +:meth:`~urllib3.PoolManager.request`. Errors & Exceptions ------------------- urllib3 wraps lower-level exceptions, for example: -.. code-block:: pycon +.. code-block:: python + + import urllib3 + + try: + urllib3.request("GET","https://nx.example.com", retries=False) - >>> try: - ... http.request('GET', 'nx.example.com', retries=False) - ... except urllib3.exceptions.NewConnectionError: - ... print('Connection failed.') + except urllib3.exceptions.NewConnectionError: + print("Connection failed.") + # Connection failed. See :mod:`~urllib3.exceptions` for the full list of all exceptions. @@ -531,6 +654,6 @@ If you are using the standard library :mod:`logging` module urllib3 will emit several logs. In some cases this can be undesirable. You can use the standard logger interface to change the log level for urllib3's logger: -.. code-block:: pycon +.. code-block:: python - >>> logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) diff --git a/docs/v2-migration-guide.rst b/docs/v2-migration-guide.rst new file mode 100644 index 0000000..dae10d2 --- /dev/null +++ b/docs/v2-migration-guide.rst @@ -0,0 +1,400 @@ +v2.0 Migration Guide +==================== + +**urllib3 v2.0 is now available!** Read below for how to get started and what is contained in the new major release. + +**🚀 Migrating from 1.x to 2.0** +-------------------------------- + +We're maintaining **functional API compatibility for most users** to make the +migration an easy choice for almost everyone. Most changes are either to default +configurations, supported Python versions, or internal implementation details. +So unless you're in a specific situation you should notice no changes! 🎉 + +.. note:: + + If you have difficulty migrating to v2.0 or following this guide + you can `open an issue on GitHub `_ + or reach out in `our community Discord channel `_. + + +Timeline for deprecations and breaking changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The 2.x initial release schedule will look like this: + +* urllib3 ``v2.0.0-alpha1`` will be released in November 2022. This release + contains **minor breaking changes and deprecation warnings for other breaking changes**. + There may be other pre-releases to address fixes before v2.0.0 is released. +* urllib3 ``v2.0.0`` will be released in early 2023 after some initial integration testing + against dependent packages and fixing of bug reports. +* urllib3 ``v2.1.0`` will be released in the summer of 2023 with **all breaking changes + being warned about in v2.0.0**. + +.. warning:: + + Please take the ``DeprecationWarnings`` you receive when migrating from v1.x to v2.0 seriously + as they will become errors after 2.1.0 is released. + + +What are the important changes? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here's a short summary of which changes in urllib3 v2.0 are most important: + +- Python version must be **3.7 or later** (previously supported Python 2.7, 3.5, and 3.6). +- Removed support for non-OpenSSL TLS libraries (like LibreSSL and wolfSSL). +- Removed support for OpenSSL versions older than 1.1.1. +- Removed support for Python implementations that aren't CPython or PyPy3 (previously supported Google App Engine, Jython). +- Removed the ``urllib3.contrib.ntlmpool`` module. +- Deprecated the ``urllib3.contrib.pyopenssl``, ``urllib3.contrib.securetransport`` modules, will be removed in v2.1.0. +- Deprecated the ``urllib3[secure]`` extra, will be removed in v2.1.0. +- Deprecated the ``HTTPResponse.getheaders()`` method in favor of ``HTTPResponse.headers``, will be removed in v2.1.0. +- Deprecated the ``HTTPResponse.getheader(name, default)`` method in favor of ``HTTPResponse.headers.get(name, default)``, will be removed in v2.1.0. +- Deprecated URLs without a scheme (ie 'https://') and will be raising an error in a future version of urllib3. +- Changed the default minimum TLS version to TLS 1.2 (previously was TLS 1.0). +- Changed the default request body encoding from 'ISO-8859-1' to 'UTF-8'. +- Removed support for verifying certificate hostnames via ``commonName``, now only ``subjectAltName`` is used. +- Removed the default set of TLS ciphers, instead now urllib3 uses the list of ciphers configured by the system. + +For a full list of changes you can look at `the changelog `_. + + +Migrating as a package maintainer? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you're a maintainer of a package that uses urllib3 under the hood then this section is for you. +You may have already seen an issue opened from someone on our team about the upcoming release. + +The primary goal for migrating to urllib3 v2.x should be to ensure your package supports **both urllib3 v1.26.x and v2.0 for some time**. +This is to reduce the chance that diamond dependencies are introduced into your users' dependencies which will then cause issues +with them upgrading to the latest version of **your package**. + +The first step to supporting urllib3 v2.0 is to make sure the version v2.x not being excluded by ``install_requires``. You should +ensure your package allows for both urllib3 1.26.x and 2.0 to be used: + +.. code-block:: python + + # setup.py (setuptools) + setup( + ... + install_requires=["urllib3>=1.26,<3"] + ) + + # pyproject.toml (hatch) + [project] + dependencies = [ + "urllib3>=1.26,<3" + ] + +Next you should try installing urllib3 v2.0 locally and run your test suite. + +.. code-block:: bash + + $ python -m pip install -U --pre 'urllib3>=2.0.0a1' + + +Because there are many ``DeprecationWarnings`` you should ensure that you're +able to see those warnings when running your test suite. To do so you can add +the following to your test setup to ensure even ``DeprecationWarnings`` are +output to the terminal: + +.. code-block:: bash + + # Set PYTHONWARNING=default to show all warnings. + $ export PYTHONWARNINGS="default" + + # Run your test suite and look for failures. + # Pytest automatically prints all warnings. + $ pytest tests/ + +or you can opt-in within your Python code: + +.. code-block:: python + + # You can change warning filters according to the filter rules: + # https://docs.python.org/3/library/warnings.html#warning-filter + import warnings + warnings.filterwarnings("default", category=DeprecationWarning) + +Any failures or deprecation warnings you receive should be fixed as urllib3 v2.1.0 will remove all +deprecated features. Many deprecation warnings will make suggestions about what to do to avoid the deprecated feature. + +Warnings will look something like this: + +.. code-block:: bash + + DeprecationWarning: 'ssl_version' option is deprecated and will be removed + in urllib3 v2.1.0. Instead use 'ssl_minimum_version' + +Continue removing deprecation warnings until there are no more. After this you can publish a new release of your package +that supports both urllib3 v1.26.x and v2.x. + +.. note:: + + If you're not able to support both 1.26.x and v2.0 of urllib3 at the same time with your package please + `open an issue on GitHub `_ or reach out in + `our community Discord channel `_. + + +Migrating as an application developer? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you're someone who writes Python but doesn't ship as a package (things like web services, data science, tools, and more) this section is for you. + +Python environments only allow for one version of a dependency to be installed per environment which means +that **all of your dependencies using urllib3 need to support v2.0 for you to upgrade**. + +The best way to visualize relationships between your dependencies is using `pipdeptree `_ and ``$ pipdeptree --reverse``: + +.. code-block:: bash + + # From inside your Python environment: + $ python -m pip install pipdeptree + # We only care about packages requiring urllib3 + $ pipdeptree --reverse | grep "requires: urllib3" + + - botocore==1.29.8 [requires: urllib3>=1.25.4,<2] + - requests==2.28.1 [requires: urllib3>=1.21.1,<2] + +Reading the output from above, there are two packages which depend on urllib3: ``botocore`` and ``requests``. +The versions of these two packages both require urllib3 that is less than v2.0 (ie ``<2``). + +Because both of these packages require urllib3 before v2.0 the new version of urllib3 can't be installed +by default. There are ways to force installing the newer version of urllib3 v2.0 (ie pinning to ``urllib3==2.0.0``) +which you can do to test your application. + +It's important to know that even if you don't upgrade all of your services to 2.x +immediately you will `receive security fixes on the 1.26.x release stream <#security-fixes-for-urllib3-v1-26-x>` for some time. + + +Security fixes for urllib3 v1.26.x +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Thanks to support from `Tidelift `_ +we're able to continue supporting the v1.26.x release stream with +security fixes for the foreseeable future 💖 + +However, upgrading is still recommended as **no new feature developments or non-critical +bug fixes will be shipped to the 1.26.x release stream**. + +If your organization relies on urllib3 and is interested in continuing support you can learn +more about the `Tidelift Subscription for Enterprise `_. + +**🤔 Common upgrading issues** +------------------------------- + +ssl module is compiled with OpenSSL 1.0.2.k-fips +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: text + + ImportError: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'OpenSSL 1.0.2k-fips 26 Jan 2017'. + See: https://github.com/urllib3/urllib3/issues/2168 + +Remediation depends on your system: + +- **AWS Lambda**: Upgrade to the Python3.10 runtime as it uses OpenSSL 1.1.1. Alternatively, you can + use a `custom Docker image + `_ and ensure you + use a Python build that uses OpenSSL 1.1.1 or later. +- **Amazon Linux 2**: Upgrade to `Amazon Linux 2023 + `_. Alternatively, you can install OpenSSL 1.1.1 + on Amazon Linux 2 using ``yum install openssl11 openssl11-devel`` and then install Python with a + tool like pyenv. +- **Red Hat Enterpritse Linux 7 (RHEL 7)**: Upgrade to RHEL 8 or RHEL 9. +- **Read the Docs**: Upgrade your `configuration file to use Ubuntu 22.04 + `_ by using ``os: ubuntu-22.04`` in the + ``build`` section. Feel free to use the `urllib3 configuration + `_ as an inspiration. + +docker.errors.dockerexception: error while fetching server api version: request() got an unexpected keyword argument 'chunked' +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Upgrade to ``docker==6.1.0`` that is compatible with urllib3 2.0. + +ImportError: cannot import name 'gaecontrib' from 'requests_toolbelt._compat' +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To be compatible with urllib3 2.0, Requests Toolbelt released version 1.0.0 without Google App +Engine Standard Python 2.7 support. Most users that reported this issue were using the `Pyrebase +`_ library that provides an API for the Firebase API. This +library is unmaintained, but `replacements exist +`_. + +``ImportError: cannot import name 'DEFAULT_CIPHERS' from 'urllib3.util.ssl_'`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This likely happens because you're using botocore which `does not support urllib3 2.0 yet +`_. The good news is that botocore explicitly declares +in its dependencies that it only supports ``urllib3<2``. Make sure to use a recent pip. That way, pip +will install urllib3 1.26.x until botocore starts supporting urllib3 2.0. + +If you're deploying to an AWS environment such as Lambda or a host using Amazon Linux 2, +you'll need to explicitly pin to ``urllib3<2`` in your project to ensure urllib3 2.0 isn't +brought into your environment. Otherwise, this may result in unintended side effects with +the default boto3 installation. + +AttributeError: module 'urllib3.connectionpool' has no attribute 'VerifiedHTTPSConnection' +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``VerifiedHTTPSConnection`` class has always been documented to be in the +:mod:`~urllib3.connection` module. It used to be possible to import it from +:mod:`~urllib3.connectionpool` but that was acccidental and is no longer possible due to a +refactoring in urllib3 2.0. + +Note that the new name of this class is :class:`~urllib3.connection.HTTPSConnection`. It can be used +starting from urllib3 1.25.9. + +AttributeError: 'HTTPResponse' object has no attribute 'strict' +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``strict`` parameter is unneeded with Python 3 and should be removed. + +Pinning urllib3<2 +~~~~~~~~~~~~~~~~~ + +If the advice from the above sections did not help, you can pin urllib3 to 1.26.x by installing +``urllib3<2``. Please do **not** specify ``urllib3==1.26.15`` to make sure you continue getting +1.26.x updates! + +While urllib3 1.26.x is still supported, it won't get new features or bug fixes, just security +updates. Consider opening a tracking issue to unpin urllib3 in the future to not stay on 1.26.x +indefinitely. For more details on the recommended way to handle your dependencies in general, see +`Semantic Versioning Will Not Save You `_. The +second half even uses urllib3 2.0 as an example! + + +**💪 User-friendly features** +----------------------------- + +urllib3 has always billed itself as a **user-friendly HTTP client library**. +In the spirit of being even more user-friendly we've added two features +which should make using urllib3 for tinkering sessions, throw-away scripts, +and smaller projects a breeze! + +urllib3.request() +~~~~~~~~~~~~~~~~~ + +Previously the highest-level API available for urllib3 was a ``PoolManager``, +but for many cases configuring a poolmanager is extra steps for no benefit. +To make using urllib3 as simple as possible we've added a top-level function +for sending requests from a global poolmanager instance: + +.. code-block:: python + + >>> import urllib3 + >>> resp = urllib3.request("GET", "https://example.com") + >>> resp.status + 200 + +JSON support for requests and responses +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +JSON is everywhere – and now it's in urllib3, too! + +If you'd like to send JSON in a request body or deserialize a response body +from JSON into Python objects you can now use the new ``json=`` parameter +for requests and ``HTTPResponse.json()`` method on responses: + +.. code-block:: python + + import urllib3 + + # Send a request with a JSON body. + # This adds 'Content-Type: application/json' by default. + resp = urllib3.request( + "POST", "https://example.api.com", + json={"key": "value"} + ) + + # Receive a JSON body in the response. + resp = urllib3.request("GET", "https://xkcd.com/2347/info.0.json") + + # There's always an XKCD... + resp.json() + { + "num": 2347, + "img": "https://imgs.xkcd.com/comics/dependency.png", + "title": "Dependency", + ... + } + + +**✨ Optimized for Python 3.7+** +-------------------------------- + +In v2.0 we'll be specifically targeting +CPython 3.7+ and PyPy 7.0+ (compatible with CPython 3.7) +and dropping support for Python versions 2.7, 3.5, and 3.6. + +By dropping end-of-life Python versions we're able to optimize +the codebase for Python 3.7+ by using new features to improve +performance and reduce the amount of code that needs to be executed +in order to support legacy versions. + + +**📜 Type-hinted APIs** +----------------------- + +You're finally able to run Mypy or other type-checkers +on code using urllib3. This also means that for IDEs +that support type hints you'll receive better suggestions +from auto-complete. No more confusion with ``**kwargs``! + +We've also added API interfaces like ``BaseHTTPResponse`` +and ``BaseHTTPConnection`` to ensure that when you're sub-classing +an interface you're only using supported public APIs to ensure +compatibility and minimize breakages down the road. + +.. note:: + + If you're one of the rare few who is subclassing connections + or responses you should take a closer look at detailed changes + in `the changelog `_. + + +**🔐 Modern security by default** +--------------------------------- + +HTTPS requires TLS 1.2+ +~~~~~~~~~~~~~~~~~~~~~~~ + +Greater than 95% of websites support TLS 1.2 or above. +At this point we're comfortable switching the default +minimum TLS version to be 1.2 to ensure high security +for users without breaking services. + +Dropping TLS 1.0 and 1.1 by default means you +won't be vulnerable to TLS downgrade attacks +if a vulnerability in TLS 1.0 or 1.1 were discovered in +the future. Extra security for free! By dropping TLS 1.0 +and TLS 1.1 we also tighten the list of ciphers we need +to support to ensure high security for data traveling +over the wire. + +If you still need to use TLS 1.0 or 1.1 in your application +you can still upgrade to v2.0, you'll only need to set +``ssl_minimum_version`` to the proper value to continue using +legacy TLS versions. + + +Stop verifying commonName in certificates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Dropping support the long deprecated ``commonName`` +field on certificates in favor of only verifying +``subjectAltName`` to put us in line with browsers and +other HTTP client libraries and to improve security for our users. + + +Certificate verification via SSLContext +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default certificate verification is handled by urllib3 +to support legacy Python versions, but now we can +rely on Python's certificate verification instead! This +should result in a speedup for verifying certificates +and means that any improvements made to certificate +verification in Python or OpenSSL will be immediately +available. diff --git a/docs/v2-roadmap.rst b/docs/v2-roadmap.rst deleted file mode 100644 index 9dcebec..0000000 --- a/docs/v2-roadmap.rst +++ /dev/null @@ -1,177 +0,0 @@ -v2.0 Roadmap -============ - -.. important:: - - We're seeking `sponsors and supporters for urllib3 v2.0 on Open Collective `_. - There's a lot of work to be done for our small team and we want to make sure - development can get completed on-time while also fairly compensating contributors - for the additional effort required for a large release like ``v2.0``. - - Additional information available within the :doc:`sponsors` section of our documentation. - - -**🚀 Functional API Compatibility** ------------------------------------ - -We're maintaining **99% functional API compatibility** to make the -migration an easy choice for most users. Migration from v1.x to v2.x -should be the simplest major version upgrade you've ever completed. - -Most changes are either to default configurations, supported Python versions, -and internal implementation details. So unless you're in a specific situation -you should notice no changes! 🎉 - - -v1.26.x Security and Bug Fixes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Thanks to support from `Tidelift `_ -we're able to continue supporting v1.26.x releases with -both security and bug fixes for the forseeable future 💖 - -If your organization relies on urllib3 and is interested in continuing support you can learn -more about the `Tidelift Subscription for Enterprise `_. - - -**🔐 Modern Security by Default** ---------------------------------- - -HTTPS requires TLS 1.2+ -~~~~~~~~~~~~~~~~~~~~~~~ - -Greater than 95% of websites support TLS 1.2 or above. -At this point we're comfortable switching the default -minimum TLS version to be 1.2 to ensure high security -for users without breaking services. - -Dropping TLS 1.0 and 1.1 by default means you -won't be vulnerable to TLS downgrade attacks -if a vulnerability in TLS 1.0 or 1.1 were discovered in -the future. Extra security for free! By dropping TLS 1.0 -and TLS 1.1 we also tighten the list of ciphers we need -to support to ensure high security for data traveling -over the wire. - -If you still need to use TLS 1.0 or 1.1 in your application -you can still upgrade to v2.0, you'll only need to set -``ssl_version`` to the proper values to continue using -legacy TLS versions. - - -Stop Verifying CommonName in Certificates -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Dropping support the long deprecated ``commonName`` -field on certificates in favor of only verifying -``subjectAltName`` to put us in line with browsers and -other HTTP client libraries and to improve security for our users. - - -Certificate Verification via SSLContext -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -By default certificate verification is handled by urllib3 -to support legacy Python versions, but now we can -rely on Python's certificate verification instead! This -should result in a speedup for verifying certificates -and means that any improvements made to certificate -verification in Python or OpenSSL will be immediately -available. - - -**✨ Optimized for Python 3.7+** --------------------------------- - -In v2.0 we'll be specifically be targeting -CPython 3.7+ and PyPy 7.3.10+ (compatible with CPython 3.8) -and dropping support Python versions 2.7 and 3.6. - -By dropping end-of-life Python versions we're able to optimize -the codebase for Python 3.7+ by using new features to improve -performance and reduce the amount of code that needs to be executed -in order to support legacy versions. - - -**🔮 Tracing** --------------- - -Currently with urllib3 it's tough to get low-level insights into what -how your HTTP client is performing and what your connection information -looks like. In v2.0 we'll be adding tracing and telemetry information -to HTTP response objects including: - -- Connection ID -- IP Address resolved by DNS -- Request Method, Target, and Headers -- TLS Version and Cipher -- Certificate Fingerprint, subjectAltName, and Validity Information -- Timings for DNS, Request Data, First Byte in Response - - -**📜 Type-Hinted APIs** ------------------------ - -You'll finally be able to run Mypy or other type-checkers -on code using urllib3. This also means that for IDEs -that support type hints you'll receive better suggestions -from auto-complete. No more confusing with ``**kwargs``! - -We'll also add API interfaces to ensure that when -you're sub-classing an interface you're only using -supported public APIs to ensure compatibility and -minimize breakages down the road. - - -**🎁 ...and many more features!** ---------------------------------- - -- Top-level ``urllib3.request()`` API -- Streaming ``multipart/form-encoded`` Request Data -- More Powerful and Configurable Retry Logic - -If there's a feature you don't see here but would like to see -in urllib3 v2.0, there's an open GitHub issue for making -feature suggestions. - - -**📅 Release and Migration Schedule** -------------------------------------- - -We're aiming for all ``v2.x`` features to be released in **mid-to-late 2021**. - -Here's what the release and migration schedule will look like leading up -to v2.0 being released: - -- Development of ``v2.x`` breaking changes starts. -- Release ``v1.26.0`` with deprecation warnings for ``v2.0.0`` breaking changes. - This will be the last non-patch release within the ``v1.x`` stream. -- Release ``v2.0.0-alpha1`` once all breaking changes have been completed. - We'll wait for users to report issues, bugs, and unexpected - breakages at this stage to ensure the release ``v2.0.0`` goes smoothly. -- Development of remaining ``v2.x`` features starts. -- Release ``v2.0.0`` which will be identical to ``v2.0.0-alpha1``. -- Release ``v2.1.0`` with remaining ``v2.x`` features. - -Deprecation warnings within ``v1.26.x`` will be opt-in by default. - -**More detailed Application Migration Guide coming soon.** - -For Package Maintainers -~~~~~~~~~~~~~~~~~~~~~~~ - -Since this is the first major release in almost 9 years some users may -be caught off-guard by a new major release of urllib3. We're mitigating this by -trying to make ``v2.x`` API-compatible with ``v1.x``. - -If your application or library uses urllib3 and you'd like to be extra -cautious about not breaking your users, you can pin urllib3 like so -until you ensure compatibility with ``v2.x``: - -.. code-block:: python - - # 'install_requires' or 'requirements.txt' - "urllib3>=1.25,<2" - -We'd really appreciate testing compatibility -and providing feedback on ``v2.0.0-alpha1`` once released. diff --git a/dummyserver/app.py b/dummyserver/app.py new file mode 100644 index 0000000..97b1b23 --- /dev/null +++ b/dummyserver/app.py @@ -0,0 +1,476 @@ +from __future__ import annotations + +import collections +import contextlib +import datetime +import email.utils +import gzip +import mimetypes +import zlib +from collections.abc import AsyncGenerator, Iterator +from io import BytesIO +from pathlib import Path + +import trio +from quart import Response, make_response, request +from quart.typing import ResponseReturnValue +from quart_trio import QuartTrio + +hypercorn_app = QuartTrio(__name__) + +# Globals are not safe in Flask/Quart but work for our Hypercorn use case +RETRY_TEST_NAMES: collections.Counter[str] = collections.Counter() +LAST_RETRY_AFTER_REQ: datetime.datetime = datetime.datetime.min + + +pyodide_testing_app = QuartTrio(__name__) +DEFAULT_HEADERS = [ + # Allow cross-origin requests for emscripten + ("Access-Control-Allow-Origin", "*"), + ("Cross-Origin-Opener-Policy", "same-origin"), + ("Cross-Origin-Embedder-Policy", "require-corp"), + ("Feature-Policy", "sync-xhr *;"), + ("Access-Control-Allow-Headers", "*"), +] + + +@hypercorn_app.route("/") +@pyodide_testing_app.route("/") +@pyodide_testing_app.route("/index") +async def index() -> ResponseReturnValue: + return await make_response("Dummy server!") + + +@hypercorn_app.route("/alpn_protocol") +async def alpn_protocol() -> ResponseReturnValue: + """Return the requester's certificate.""" + alpn_protocol = request.scope["extensions"]["tls"]["alpn_protocol"] + return await make_response(alpn_protocol) + + +@hypercorn_app.route("/certificate") +async def certificate() -> ResponseReturnValue: + """Return the requester's certificate.""" + subject = request.scope["extensions"]["tls"]["client_cert_name"] + subject_as_dict = dict(part.split("=") for part in subject.split(", ")) + return await make_response(subject_as_dict) + + +@hypercorn_app.route("/specific_method", methods=["GET", "POST", "PUT"]) +@pyodide_testing_app.route("/specific_method", methods=["GET", "POST", "PUT"]) +async def specific_method() -> ResponseReturnValue: + "Confirm that the request matches the desired method type" + method_param = (await request.values).get("method", "") + + if request.method.upper() == method_param.upper(): + return await make_response("", 200) + else: + return await make_response( + f"Wrong method: {method_param} != {request.method}", 400 + ) + + +@hypercorn_app.route("/upload", methods=["POST"]) +async def upload() -> ResponseReturnValue: + "Confirm that the uploaded file conforms to specification" + params = await request.form + param = params.get("upload_param") + filename_param = params.get("upload_filename") + size = int(params.get("upload_size", "0")) + files_ = (await request.files).getlist(param) + assert files_ is not None + + if len(files_) != 1: + return await make_response( + f"Expected 1 file for '{param}', not {len(files_)}", 400 + ) + + file_ = files_[0] + # data is short enough to read synchronously without blocking the event loop + with contextlib.closing(file_.stream) as stream: + data = stream.read() + + if int(size) != len(data): + return await make_response(f"Wrong size: {int(size)} != {len(data)}", 400) + + if filename_param != file_.filename: + return await make_response( + f"Wrong filename: {filename_param} != {file_.filename}", 400 + ) + + return await make_response() + + +@hypercorn_app.route("/chunked") +async def chunked() -> ResponseReturnValue: + def generate() -> Iterator[str]: + for _ in range(4): + yield "123" + + return await make_response(generate()) + + +@hypercorn_app.route("/chunked_gzip") +async def chunked_gzip() -> ResponseReturnValue: + def generate() -> Iterator[bytes]: + compressor = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) + + for uncompressed in [b"123"] * 4: + yield compressor.compress(uncompressed) + yield compressor.flush() + + return await make_response(generate(), 200, [("Content-Encoding", "gzip")]) + + +@hypercorn_app.route("/keepalive") +async def keepalive() -> ResponseReturnValue: + if request.args.get("close", b"0") == b"1": + headers = [("Connection", "close")] + return await make_response("Closing", 200, headers) + + headers = [("Connection", "keep-alive")] + return await make_response("Keeping alive", 200, headers) + + +@hypercorn_app.route("/echo", methods=["GET", "POST", "PUT"]) +async def echo() -> ResponseReturnValue: + "Echo back the params" + if request.method == "GET": + return await make_response(request.query_string) + + return await make_response(await request.get_data()) + + +@hypercorn_app.route("/echo_json", methods=["POST"]) +@pyodide_testing_app.route("/echo_json", methods=["POST", "OPTIONS"]) +async def echo_json() -> ResponseReturnValue: + "Echo back the JSON" + if request.method == "OPTIONS": + return await make_response("", 200) + data = await request.get_data() + return await make_response(data, 200, request.headers) + + +@hypercorn_app.route("/echo_uri/") +@hypercorn_app.route("/echo_uri", defaults={"rest": ""}) +async def echo_uri(rest: str) -> ResponseReturnValue: + "Echo back the requested URI" + assert request.full_path is not None + return await make_response(request.full_path) + + +@hypercorn_app.route("/echo_params") +async def echo_params() -> ResponseReturnValue: + "Echo back the query parameters" + await request.get_data() + echod = sorted((k, v) for k, v in request.args.items()) + return await make_response(repr(echod)) + + +@hypercorn_app.route("/headers", methods=["GET", "POST"]) +async def headers() -> ResponseReturnValue: + return await make_response(dict(request.headers.items())) + + +@hypercorn_app.route("/headers_and_params") +async def headers_and_params() -> ResponseReturnValue: + return await make_response( + { + "headers": dict(request.headers), + "params": request.args, + } + ) + + +@hypercorn_app.route("/multi_headers", methods=["GET", "POST"]) +async def multi_headers() -> ResponseReturnValue: + return await make_response({"headers": list(request.headers)}) + + +@hypercorn_app.route("/multi_redirect") +async def multi_redirect() -> ResponseReturnValue: + "Performs a redirect chain based on ``redirect_codes``" + params = request.args + codes = params.get("redirect_codes", "200") + head, tail = codes.split(",", 1) if "," in codes else (codes, None) + assert head is not None + status = head + if not tail: + return await make_response("Done redirecting", status) + + headers = [("Location", f"/multi_redirect?redirect_codes={tail}")] + return await make_response("", status, headers) + + +@hypercorn_app.route("/encodingrequest") +async def encodingrequest() -> ResponseReturnValue: + "Check for UA accepting gzip/deflate encoding" + data = b"hello, world!" + encoding = request.headers.get("Accept-Encoding", "") + headers = [] + if encoding == "gzip": + headers = [("Content-Encoding", "gzip")] + file_ = BytesIO() + with contextlib.closing(gzip.GzipFile("", mode="w", fileobj=file_)) as zipfile: + zipfile.write(data) + data = file_.getvalue() + elif encoding == "deflate": + headers = [("Content-Encoding", "deflate")] + data = zlib.compress(data) + elif encoding == "garbage-gzip": + headers = [("Content-Encoding", "gzip")] + data = b"garbage" + elif encoding == "garbage-deflate": + headers = [("Content-Encoding", "deflate")] + data = b"garbage" + return await make_response(data, 200, headers) + + +@hypercorn_app.route("/redirect", methods=["GET", "POST", "PUT"]) +async def redirect() -> ResponseReturnValue: + "Perform a redirect to ``target``" + values = await request.values + target = values.get("target", "/") + status = values.get("status", "303 See Other") + status_code = status.split(" ")[0] + + headers = [("Location", target)] + return await make_response("", status_code, headers) + + +@hypercorn_app.route("/redirect_after") +async def redirect_after() -> ResponseReturnValue: + "Perform a redirect to ``target``" + params = request.args + date = params.get("date") + if date: + dt = datetime.datetime.fromtimestamp(float(date), tz=datetime.timezone.utc) + http_dt = email.utils.format_datetime(dt, usegmt=True) + retry_after = str(http_dt) + else: + retry_after = "1" + target = params.get("target", "/") + headers = [("Location", target), ("Retry-After", retry_after)] + return await make_response("", 303, headers) + + +@hypercorn_app.route("/retry_after") +async def retry_after() -> ResponseReturnValue: + global LAST_RETRY_AFTER_REQ + params = request.args + if datetime.datetime.now() - LAST_RETRY_AFTER_REQ < datetime.timedelta(seconds=1): + status = params.get("status", "429 Too Many Requests") + status_code = status.split(" ")[0] + + return await make_response("", status_code, [("Retry-After", "1")]) + + LAST_RETRY_AFTER_REQ = datetime.datetime.now() + return await make_response("", 200) + + +@hypercorn_app.route("/status") +@pyodide_testing_app.route("/status") +async def status() -> ResponseReturnValue: + values = await request.values + status = values.get("status", "200 OK") + status_code = status.split(" ")[0] + return await make_response("", status_code) + + +@hypercorn_app.route("/source_address") +async def source_address() -> ResponseReturnValue: + """Return the requester's IP address.""" + return await make_response(request.remote_addr) + + +@hypercorn_app.route("/successful_retry", methods=["GET", "PUT"]) +async def successful_retry() -> ResponseReturnValue: + """First return an error and then success + + It's not currently very flexible as the number of retries is hard-coded. + """ + test_name = request.headers.get("test-name", None) + if not test_name: + return await make_response("test-name header not set", 400) + + RETRY_TEST_NAMES[test_name] += 1 + + if RETRY_TEST_NAMES[test_name] >= 2: + return await make_response("Retry successful!", 200) + else: + return await make_response("need to keep retrying!", 418) + + +@pyodide_testing_app.after_request +def apply_caching(response: Response) -> ResponseReturnValue: + for header, value in DEFAULT_HEADERS: + response.headers[header] = value + return response + + +@pyodide_testing_app.route("/slow") +async def slow() -> ResponseReturnValue: + await trio.sleep(10) + return await make_response("TEN SECONDS LATER", 200) + + +@pyodide_testing_app.route("/dripfeed") +async def dripfeed() -> ResponseReturnValue: + # great big text file which streams half the file + # then pauses for 2 seconds and streams the rest + async def generate() -> AsyncGenerator[bytes]: + for x in range(8): + if x == 4: + await trio.sleep(2) + yield b"WOOO YAY BOOYAKAH" * 131072 + + response = await make_response(generate(), 200) + if hasattr(response, "timeout"): + response.timeout = None + return response + + +@pyodide_testing_app.route("/bigfile") +async def bigfile() -> ResponseReturnValue: + # great big text file, should force streaming + # if supported + bigdata = 1048576 * b"WOOO YAY BOOYAKAH" + return await make_response(bigdata, 200) + + +@pyodide_testing_app.route("/mediumfile") +async def mediumfile() -> ResponseReturnValue: + # quite big file + bigdata = 1024 * b"WOOO YAY BOOYAKAH" + return await make_response(bigdata, 200) + + +@pyodide_testing_app.route("/upload", methods=["POST", "OPTIONS"]) +async def pyodide_upload() -> ResponseReturnValue: + if request.method == "OPTIONS": + return await make_response("", 200) + spare_data = await request.get_data(parse_form_data=True) + if len(spare_data) != 0: + return await make_response("Bad upload data", 404) + files = await request.files + form = await request.form + if form["upload_param"] != "filefield" or form["upload_filename"] != "lolcat.txt": + return await make_response("Bad upload form values", 404) + if len(files) != 1 or files.get("filefield") is None: + return await make_response("Missing file in form", 404) + file = files["filefield"] + if file.filename != "lolcat.txt": + return await make_response(f"File name incorrect {file.name}", 404) + with contextlib.closing(file): + data = file.read().decode("utf-8") + if data != "I'm in ur multipart form-data, hazing a cheezburgr": + return await make_response(f"File data incorrect {data}", 200) + return await make_response("Uploaded file correct", 200) + + +def _find_built_wheel() -> Path | None: + wheel_folder = Path(__file__).parent.parent / "dist" + wheels = list(wheel_folder.glob("*.whl")) + wheels = sorted(wheels, key=lambda w: w.stat().st_mtime) + if len(wheels) > 0: + return wheels[-1] # newest wheel + else: + return None + + +def _get_pyodide_template(py_file: str) -> bytes | None: + # serve code to run pyodide in a webworker, or html template + # these are included in pytest_pyodide, but + # we modify the webworker to automatically load our wheel + # before anything else + if py_file == "webworker_dev.js": + return b""" + importScripts("./pyodide.js"); + + onmessage = async function (e) { + try { + let code = e.data.python; + self.pyodide = await loadPyodide(); + await self.pyodide.loadPackage("/dist/urllib3.whl"); + await self.pyodide.loadPackagesFromImports(code); + let results = await self.pyodide.runPythonAsync(code); + self.postMessage({ results }); + } catch (e) { + self.postMessage({ error: e.message + "\\n" + e.stack }); + } + }; + """ + elif py_file == "test.html": + return b""" + + + + + + + + + """ + else: + return None + + +@pyodide_testing_app.route("/pyodide/") +async def pyodide(py_file: str) -> ResponseReturnValue: + template_data = _get_pyodide_template(py_file) + mime_type: str | None = None + if template_data: + mime_type, _encoding = mimetypes.guess_type(py_file) + if not mime_type: + mime_type = "text/plain" + headers = [("Content-Type", mime_type)] + return await make_response(template_data, 200, headers) + file_path = Path(pyodide_testing_app.config["pyodide_dist_dir"], py_file) + + if file_path is not None and file_path.exists(): + if py_file.endswith(".whl"): + mime_type = "application/x-wheel" + headers = [ + ("Content-Disposition", f"inline; filename='{file_path.name}'"), + ("Content-Type", mime_type), + ] + else: + mime_type, _encoding = mimetypes.guess_type(file_path) + if not mime_type: + mime_type = "text/plain" + headers = [("Content-Type", mime_type)] + return await make_response(file_path.read_bytes(), 200, headers) + else: + return await make_response("", 404) + + +@pyodide_testing_app.route("/dist/urllib3.whl") +async def wheel() -> ResponseReturnValue: + file_path = _find_built_wheel() + if not file_path: + print("NO BUILT WHEEL?") + return await make_response("", 404) + + mime_type = "application/x-wheel" + headers = [ + ("Content-Disposition", f"inline; filename='{file_path.name}'"), + ("Content-Type", mime_type), + ] + return await make_response(file_path.read_bytes(), 200, headers) diff --git a/dummyserver/asgi_proxy.py b/dummyserver/asgi_proxy.py new file mode 100755 index 0000000..107c5e0 --- /dev/null +++ b/dummyserver/asgi_proxy.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import typing + +import httpx +import trio +from hypercorn.typing import ( + ASGIReceiveCallable, + ASGISendCallable, + HTTPResponseBodyEvent, + HTTPResponseStartEvent, + HTTPScope, + Scope, +) + + +async def _read_body(receive: ASGIReceiveCallable) -> bytes: + body = bytearray() + body_consumed = False + while not body_consumed: + event = await receive() + if event["type"] == "http.request": + body.extend(event["body"]) + body_consumed = not event["more_body"] + else: + raise ValueError(event["type"]) + return bytes(body) + + +class ProxyApp: + def __init__(self, upstream_ca_certs: str | None = None): + self.upstream_ca_certs = upstream_ca_certs + + async def __call__( + self, scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable + ) -> None: + assert scope["type"] == "http" + if scope["method"] in ["GET", "POST"]: + await self.absolute_uri(scope, receive, send) + elif scope["method"] == "CONNECT": + await self.connect(scope, send) + else: + raise ValueError(scope["method"]) + + async def absolute_uri( + self, + scope: HTTPScope, + receive: ASGIReceiveCallable, + send: ASGISendCallable, + ) -> None: + async with httpx.AsyncClient(verify=self.upstream_ca_certs or True) as client: + client_response = await client.request( + method=scope["method"], + url=scope["path"], + headers=list(scope["headers"]), + content=await _read_body(receive), + ) + + headers = [] + for header in ( + "Date", + "Cache-Control", + "Server", + "Content-Type", + "Location", + ): + v = client_response.headers.get(header) + if v: + headers.append((header.encode(), v.encode())) + headers.append((b"Content-Length", str(len(client_response.content)).encode())) + + await send( + HTTPResponseStartEvent( + type="http.response.start", + status=client_response.status_code, + headers=headers, + ) + ) + await send( + HTTPResponseBodyEvent( + type="http.response.body", + body=client_response.content, + more_body=False, + ) + ) + + async def connect(self, scope: HTTPScope, send: ASGISendCallable) -> None: + async def start_forward( + reader: trio.SocketStream, writer: trio.SocketStream + ) -> None: + while True: + try: + data = await reader.receive_some(4096) + except trio.ClosedResourceError: + break + if not data: + break + await writer.send_all(data) + await writer.aclose() + + host, port = scope["path"].split(":") + async with await trio.open_tcp_stream(host, int(port)) as upstream: + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": True}) + + client = typing.cast(trio.SocketStream, scope["extensions"]["_transport"]) + + async with trio.open_nursery(strict_exception_groups=True) as nursery: + nursery.start_soon(start_forward, client, upstream) + nursery.start_soon(start_forward, upstream, client) diff --git a/dummyserver/certs/README.rst b/dummyserver/certs/README.rst index 7c712b6..3ee127a 100644 --- a/dummyserver/certs/README.rst +++ b/dummyserver/certs/README.rst @@ -6,7 +6,7 @@ Here's how you can regenerate the certificates:: import trustme ca = trustme.CA() - server_cert = ca.issue_cert(u"localhost") + server_cert = ca.issue_cert("localhost") ca.cert_pem.write_to_path("cacert.pem") ca.private_key_pem.write_to_path("cacert.key") diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py deleted file mode 100644 index acd181d..0000000 --- a/dummyserver/handlers.py +++ /dev/null @@ -1,339 +0,0 @@ -from __future__ import print_function - -import collections -import contextlib -import gzip -import json -import logging -import sys -import time -import zlib -from datetime import datetime, timedelta -from io import BytesIO - -from tornado import httputil -from tornado.web import RequestHandler - -from urllib3.packages.six import binary_type, ensure_str -from urllib3.packages.six.moves.http_client import responses -from urllib3.packages.six.moves.urllib.parse import urlsplit - -log = logging.getLogger(__name__) - - -class Response(object): - def __init__(self, body="", status="200 OK", headers=None): - self.body = body - self.status = status - self.headers = headers or [("Content-type", "text/plain")] - - def __call__(self, request_handler): - status, reason = self.status.split(" ", 1) - request_handler.set_status(int(status), reason) - for header, value in self.headers: - request_handler.add_header(header, value) - - # chunked - if isinstance(self.body, list): - for item in self.body: - if not isinstance(item, bytes): - item = item.encode("utf8") - request_handler.write(item) - request_handler.flush() - else: - body = self.body - if not isinstance(body, bytes): - body = body.encode("utf8") - - request_handler.write(body) - - -RETRY_TEST_NAMES = collections.defaultdict(int) - - -class TestingApp(RequestHandler): - """ - Simple app that performs various operations, useful for testing an HTTP - library. - - Given any path, it will attempt to load a corresponding local method if - it exists. Status code 200 indicates success, 400 indicates failure. Each - method has its own conditions for success/failure. - """ - - def get(self): - """Handle GET requests""" - self._call_method() - - def post(self): - """Handle POST requests""" - self._call_method() - - def put(self): - """Handle PUT requests""" - self._call_method() - - def options(self): - """Handle OPTIONS requests""" - self._call_method() - - def head(self): - """Handle HEAD requests""" - self._call_method() - - def _call_method(self): - """Call the correct method in this class based on the incoming URI""" - req = self.request - req.params = {} - for k, v in req.arguments.items(): - req.params[k] = next(iter(v)) - - path = req.path[:] - if not path.startswith("/"): - path = urlsplit(path).path - - target = path[1:].split("/", 1)[0] - method = getattr(self, target, self.index) - - resp = method(req) - - if dict(resp.headers).get("Connection") == "close": - # FIXME: Can we kill the connection somehow? - pass - - resp(self) - - def index(self, _request): - "Render simple message" - return Response("Dummy server!") - - def certificate(self, request): - """Return the requester's certificate.""" - cert = request.get_ssl_certificate() - subject = dict() - if cert is not None: - subject = dict((k, v) for (k, v) in [y for z in cert["subject"] for y in z]) - return Response(json.dumps(subject)) - - def alpn_protocol(self, request): - """Return the selected ALPN protocol.""" - proto = request.connection.stream.socket.selected_alpn_protocol() - return Response(proto.encode("utf8") if proto is not None else u"") - - def source_address(self, request): - """Return the requester's IP address.""" - return Response(request.remote_ip) - - def set_up(self, request): - test_type = request.params.get("test_type") - test_id = request.params.get("test_id") - if test_id: - print("\nNew test %s: %s" % (test_type, test_id)) - else: - print("\nNew test %s" % test_type) - return Response("Dummy server is ready!") - - def specific_method(self, request): - "Confirm that the request matches the desired method type" - method = request.params.get("method") - if method and not isinstance(method, str): - method = method.decode("utf8") - - if request.method != method: - return Response( - "Wrong method: %s != %s" % (method, request.method), - status="400 Bad Request", - ) - return Response() - - def upload(self, request): - "Confirm that the uploaded file conforms to specification" - # FIXME: This is a huge broken mess - param = request.params.get("upload_param", b"myfile").decode("ascii") - filename = request.params.get("upload_filename", b"").decode("utf-8") - size = int(request.params.get("upload_size", "0")) - files_ = request.files.get(param) - - if len(files_) != 1: - return Response( - "Expected 1 file for '%s', not %d" % (param, len(files_)), - status="400 Bad Request", - ) - file_ = files_[0] - - data = file_["body"] - if int(size) != len(data): - return Response( - "Wrong size: %d != %d" % (size, len(data)), status="400 Bad Request" - ) - - got_filename = file_["filename"] - if isinstance(got_filename, binary_type): - got_filename = got_filename.decode("utf-8") - - # Tornado can leave the trailing \n in place on the filename. - if filename != got_filename: - return Response( - u"Wrong filename: %s != %s" % (filename, file_.filename), - status="400 Bad Request", - ) - - return Response() - - def redirect(self, request): - "Perform a redirect to ``target``" - target = request.params.get("target", "/") - status = request.params.get("status", "303 See Other") - if len(status) == 3: - status = "%s Redirect" % status.decode("latin-1") - elif isinstance(status, bytes): - status = status.decode("latin-1") - - headers = [("Location", target)] - return Response(status=status, headers=headers) - - def not_found(self, request): - return Response("Not found", status="404 Not Found") - - def multi_redirect(self, request): - "Performs a redirect chain based on ``redirect_codes``" - codes = request.params.get("redirect_codes", b"200").decode("utf-8") - head, tail = codes.split(",", 1) if "," in codes else (codes, None) - status = "{0} {1}".format(head, responses[int(head)]) - if not tail: - return Response("Done redirecting", status=status) - - headers = [("Location", "/multi_redirect?redirect_codes=%s" % tail)] - return Response(status=status, headers=headers) - - def keepalive(self, request): - if request.params.get("close", b"0") == b"1": - headers = [("Connection", "close")] - return Response("Closing", headers=headers) - - headers = [("Connection", "keep-alive")] - return Response("Keeping alive", headers=headers) - - def echo_params(self, request): - params = sorted( - [(ensure_str(k), ensure_str(v)) for k, v in request.params.items()] - ) - return Response(repr(params)) - - def sleep(self, request): - "Sleep for a specified amount of ``seconds``" - # DO NOT USE THIS, IT'S DEPRECATED. - # FIXME: Delete this once appengine tests are fixed to not use this handler. - seconds = float(request.params.get("seconds", "1")) - time.sleep(seconds) - return Response() - - def echo(self, request): - "Echo back the params" - if request.method == "GET": - return Response(request.query) - - return Response(request.body) - - def echo_uri(self, request): - "Echo back the requested URI" - return Response(request.uri) - - def encodingrequest(self, request): - "Check for UA accepting gzip/deflate encoding" - data = b"hello, world!" - encoding = request.headers.get("Accept-Encoding", "") - headers = None - if encoding == "gzip": - headers = [("Content-Encoding", "gzip")] - file_ = BytesIO() - with contextlib.closing( - gzip.GzipFile("", mode="w", fileobj=file_) - ) as zipfile: - zipfile.write(data) - data = file_.getvalue() - elif encoding == "deflate": - headers = [("Content-Encoding", "deflate")] - data = zlib.compress(data) - elif encoding == "garbage-gzip": - headers = [("Content-Encoding", "gzip")] - data = "garbage" - elif encoding == "garbage-deflate": - headers = [("Content-Encoding", "deflate")] - data = "garbage" - return Response(data, headers=headers) - - def headers(self, request): - return Response(json.dumps(dict(request.headers))) - - def headers_and_params(self, request): - return Response( - json.dumps({"headers": dict(request.headers), "params": request.params}) - ) - - def successful_retry(self, request): - """Handler which will return an error and then success - - It's not currently very flexible as the number of retries is hard-coded. - """ - test_name = request.headers.get("test-name", None) - if not test_name: - return Response("test-name header not set", status="400 Bad Request") - - RETRY_TEST_NAMES[test_name] += 1 - - if RETRY_TEST_NAMES[test_name] >= 2: - return Response("Retry successful!") - else: - return Response("need to keep retrying!", status="418 I'm A Teapot") - - def chunked(self, request): - return Response(["123"] * 4) - - def chunked_gzip(self, request): - chunks = [] - compressor = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) - - for uncompressed in [b"123"] * 4: - chunks.append(compressor.compress(uncompressed)) - - chunks.append(compressor.flush()) - - return Response(chunks, headers=[("Content-Encoding", "gzip")]) - - def nbytes(self, request): - length = int(request.params.get("length")) - data = b"1" * length - return Response(data, headers=[("Content-Type", "application/octet-stream")]) - - def status(self, request): - status = request.params.get("status", "200 OK") - - return Response(status=status) - - def retry_after(self, request): - if datetime.now() - self.application.last_req < timedelta(seconds=1): - status = request.params.get("status", b"429 Too Many Requests") - return Response( - status=status.decode("utf-8"), headers=[("Retry-After", "1")] - ) - - self.application.last_req = datetime.now() - - return Response(status="200 OK") - - def redirect_after(self, request): - "Perform a redirect to ``target``" - date = request.params.get("date") - if date: - retry_after = str( - httputil.format_timestamp(datetime.utcfromtimestamp(float(date))) - ) - else: - retry_after = "1" - target = request.params.get("target", "/") - headers = [("Location", target), ("Retry-After", retry_after)] - return Response(status="303 See Other", headers=headers) - - def shutdown(self, request): - sys.exit() diff --git a/dummyserver/hypercornserver.py b/dummyserver/hypercornserver.py new file mode 100644 index 0000000..5863033 --- /dev/null +++ b/dummyserver/hypercornserver.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import concurrent.futures +import contextlib +import errno +import functools +import socket +import sys +import threading +import traceback +import typing + +import hypercorn +import hypercorn.config +import hypercorn.trio +import hypercorn.typing +import trio +from quart_trio import QuartTrio + +from urllib3.util.url import parse_url + + +class Config(hypercorn.Config): + def create_sockets(self) -> hypercorn.config.Sockets: + assert len(self.bind) == 1 + secure_sockets, insecure_sockets = [], [] + if self.ssl_enabled: + secure_sockets = self._create_urllib3_sockets(self.bind[0]) + else: + insecure_sockets = self._create_urllib3_sockets(self.bind[0]) + return hypercorn.config.Sockets( + secure_sockets, insecure_sockets, quic_sockets=[] + ) + + def _retry_create_urllib3_sockets(self, bind: str) -> list[socket.socket]: + # When we request a socket with host localhost and port zero, Hypercorn + # only binds to IPv4. But we want to bind to IPv6 too, otherwise we + # waste about 2 second for each test on Windows since urllib3 tries + # IPv6 first as it does not implement Happy Eyeballs. + # We want to use the same port for IPv4 and IPv6, so we first get a + # free port in IPv4 and request the same port in IPv6. But that port + # could easily be taken with IPv6 already, especially on crowded CI + # environments, which would fail the run. For this reason we retry + # _create_urllib3_sockets up to 10 times, which completely eliminates + # this failure mode. + for i in range(10): + try: + return self._create_urllib3_sockets(bind) + except OSError as e: + if e.errno == errno.EADDRINUSE: + print( + f"Retrying binding to {bind} after EADDRINUSE", file=sys.stderr + ) + raise OSError("failed to bind socket") + + def _create_urllib3_sockets(self, bind: str) -> list[socket.socket]: + sockets = [] + + bind = bind.replace("[", "").replace("]", "") + host = bind.rsplit(":", 1)[0] + port = 0 # Get a random port + family = socket.AF_INET6 if ":" in host else socket.AF_UNSPEC + + for res in socket.getaddrinfo( + host, port, family, socket.SOCK_STREAM, 0, socket.AI_PASSIVE + ): + af, socktype, proto, canonname, sockadddr = res + + sock = socket.socket(af, socket.SOCK_STREAM, proto) + + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + sock.setblocking(False) + sock.bind((host, port)) + port = sock.getsockname()[1] + sock.set_inheritable(True) + sockets.append(sock) + + return sockets + + +# https://github.com/pgjones/hypercorn/blob/19dfb96411575a6a647cdea63fa581b48ebb9180/src/hypercorn/utils.py#L172-L178 +async def graceful_shutdown(shutdown_event: threading.Event) -> None: + while True: + if shutdown_event.is_set(): + return + await trio.sleep(0.1) + + +async def _start_server( + config: Config, + app: QuartTrio, + ready_event: threading.Event, + shutdown_event: threading.Event, +) -> None: + async with trio.open_nursery() as nursery: + try: + config.bind = await nursery.start( + functools.partial( + hypercorn.trio.serve, + app, + config, + shutdown_trigger=functools.partial( + graceful_shutdown, shutdown_event + ), + ) + ) + ready_event.set() + except Exception: + print("Starting server failed", file=sys.stderr) + traceback.print_exc() + raise + + +@contextlib.contextmanager +def run_hypercorn_in_thread( + host: str, certs: dict[str, typing.Any] | None, app: hypercorn.typing.ASGIFramework +) -> typing.Iterator[int]: + config = Config() + if certs: + config.certfile = certs["certfile"] + config.keyfile = certs["keyfile"] + if "cert_reqs" in certs: + config.verify_mode = certs["cert_reqs"] + if "ca_certs" in certs: + config.ca_certs = certs["ca_certs"] + if "alpn_protocols" in certs: + config.alpn_protocols = certs["alpn_protocols"] + config.bind = [f"{host}:0"] + + ready_event = threading.Event() + shutdown_event = threading.Event() + + with concurrent.futures.ThreadPoolExecutor( + 1, thread_name_prefix="hypercorn dummyserver" + ) as executor: + future = executor.submit( + trio.run, + _start_server, + config, + app, + ready_event, + shutdown_event, + ) + ready_event.wait(5) + if not ready_event.is_set(): + raise Exception("most likely failed to start server") + + try: + port = parse_url(config.bind[0]).port + assert port is not None + yield port + finally: + shutdown_event.set() + future.result() + + +def main() -> int: + # For debugging dummyserver itself - PYTHONPATH=src python -m dummyserver.hypercornserver + from .app import hypercorn_app + + config = Config() + config.bind = ["localhost:0"] + ready_event = threading.Event() + shutdown_event = threading.Event() + trio.run(_start_server, config, hypercorn_app, ready_event, shutdown_event) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dummyserver/proxy.py b/dummyserver/proxy.py deleted file mode 100755 index 0cd8ded..0000000 --- a/dummyserver/proxy.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python -# -# Simple asynchronous HTTP proxy with tunnelling (CONNECT). -# -# GET/POST proxying based on -# http://groups.google.com/group/python-tornado/msg/7bea08e7a049cf26 -# -# Copyright (C) 2012 Senko Rasic -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -import socket -import ssl -import sys - -import tornado.gen -import tornado.httpclient -import tornado.httpserver -import tornado.ioloop -import tornado.iostream -import tornado.web - -__all__ = ["ProxyHandler", "run_proxy"] - - -class ProxyHandler(tornado.web.RequestHandler): - SUPPORTED_METHODS = ["GET", "POST", "CONNECT"] - - @tornado.gen.coroutine - def get(self): - def handle_response(response): - if response.error and not isinstance( - response.error, tornado.httpclient.HTTPError - ): - self.set_status(500) - self.write("Internal server error:\n" + str(response.error)) - self.finish() - else: - self.set_status(response.code) - for header in ( - "Date", - "Cache-Control", - "Server", - "Content-Type", - "Location", - ): - v = response.headers.get(header) - if v: - self.set_header(header, v) - if response.body: - self.write(response.body) - self.finish() - - upstream_ca_certs = self.application.settings.get("upstream_ca_certs", None) - ssl_options = None - - if upstream_ca_certs: - ssl_options = ssl.create_default_context(cafile=upstream_ca_certs) - - req = tornado.httpclient.HTTPRequest( - url=self.request.uri, - method=self.request.method, - body=self.request.body, - headers=self.request.headers, - follow_redirects=False, - allow_nonstandard_methods=True, - ssl_options=ssl_options, - ) - - client = tornado.httpclient.AsyncHTTPClient() - try: - response = yield client.fetch(req) - yield handle_response(response) - except tornado.httpclient.HTTPError as e: - if hasattr(e, "response") and e.response: - yield handle_response(e.response) - else: - self.set_status(500) - self.write("Internal server error:\n" + str(e)) - self.finish() - - @tornado.gen.coroutine - def post(self): - yield self.get() - - @tornado.gen.coroutine - def connect(self): - host, port = self.request.uri.split(":") - client = self.request.connection.stream - - @tornado.gen.coroutine - def start_forward(reader, writer): - while True: - try: - data = yield reader.read_bytes(4096, partial=True) - except tornado.iostream.StreamClosedError: - break - if not data: - break - writer.write(data) - writer.close() - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - upstream = tornado.iostream.IOStream(s) - yield upstream.connect((host, int(port))) - - client.write(b"HTTP/1.0 200 Connection established\r\n\r\n") - fu1 = start_forward(client, upstream) - fu2 = start_forward(upstream, client) - yield [fu1, fu2] - - -def run_proxy(port, start_ioloop=True): - """ - Run proxy on the specified port. If start_ioloop is True (default), - the tornado IOLoop will be started immediately. - """ - app = tornado.web.Application([(r".*", ProxyHandler)]) - app.listen(port) - ioloop = tornado.ioloop.IOLoop.instance() - if start_ioloop: - ioloop.start() - - -if __name__ == "__main__": - port = 8888 - if len(sys.argv) > 1: - port = int(sys.argv[1]) - - print("Starting HTTP proxy on port %d" % port) - run_proxy(port) diff --git a/dummyserver/server.py b/dummyserver/socketserver.py similarity index 62% rename from dummyserver/server.py rename to dummyserver/socketserver.py index cc1964b..ec7d83e 100755 --- a/dummyserver/server.py +++ b/dummyserver/socketserver.py @@ -3,7 +3,8 @@ """ Dummy server used for unit testing. """ -from __future__ import print_function + +from __future__ import annotations import logging import os @@ -11,35 +12,36 @@ import ssl import sys import threading +import typing import warnings -from datetime import datetime -import tornado.httpserver -import tornado.ioloop -import tornado.netutil -import tornado.web import trustme from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from urllib3.exceptions import HTTPWarning -from urllib3.util import ALPN_PROTOCOLS, resolve_cert_reqs, resolve_ssl_version +from urllib3.util import resolve_cert_reqs, resolve_ssl_version + +if typing.TYPE_CHECKING: + from typing_extensions import ParamSpec + + P = ParamSpec("P") log = logging.getLogger(__name__) CERTS_PATH = os.path.join(os.path.dirname(__file__), "certs") -DEFAULT_CERTS = { +DEFAULT_CERTS: dict[str, typing.Any] = { "certfile": os.path.join(CERTS_PATH, "server.crt"), "keyfile": os.path.join(CERTS_PATH, "server.key"), "cert_reqs": ssl.CERT_OPTIONAL, "ca_certs": os.path.join(CERTS_PATH, "cacert.pem"), - "alpn_protocols": ALPN_PROTOCOLS, + "alpn_protocols": ["h2", "http/1.1"], } DEFAULT_CA = os.path.join(CERTS_PATH, "cacert.pem") DEFAULT_CA_KEY = os.path.join(CERTS_PATH, "cacert.key") -def _resolves_to_ipv6(host): +def _resolves_to_ipv6(host: str) -> bool: """Returns True if the system resolves host to an IPv6 address by default.""" resolves_to_ipv6 = False try: @@ -53,7 +55,7 @@ def _resolves_to_ipv6(host): return resolves_to_ipv6 -def _has_ipv6(host): +def _has_ipv6(host: str) -> bool: """Returns True if the system can bind an IPv6 address.""" sock = None has_ipv6 = False @@ -89,7 +91,6 @@ def _has_ipv6(host): class NoIPv6Warning(HTTPWarning): "IPv6 is not available" - pass class SocketServerThread(threading.Thread): @@ -102,15 +103,22 @@ class SocketServerThread(threading.Thread): USE_IPV6 = HAS_IPV6_AND_DNS - def __init__(self, socket_handler, host="localhost", port=8081, ready_event=None): - threading.Thread.__init__(self) + def __init__( + self, + socket_handler: typing.Callable[[socket.socket], None], + host: str = "localhost", + ready_event: threading.Event | None = None, + quit_event: threading.Event | None = None, + ) -> None: + super().__init__() self.daemon = True self.socket_handler = socket_handler self.host = host self.ready_event = ready_event + self.quit_event = quit_event - def _start_server(self): + def _start_server(self) -> None: if self.USE_IPV6: sock = socket.socket(socket.AF_INET6) else: @@ -118,34 +126,35 @@ def _start_server(self): sock = socket.socket(socket.AF_INET) if sys.platform != "win32": sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((self.host, 0)) - self.port = sock.getsockname()[1] - # Once listen() returns, the server socket is ready - sock.listen(1) + with sock: + sock.bind((self.host, 0)) + self.port = sock.getsockname()[1] - if self.ready_event: - self.ready_event.set() + # Once listen() returns, the server socket is ready + sock.listen(1) - self.socket_handler(sock) - sock.close() + if self.ready_event: + self.ready_event.set() + + self.socket_handler(sock) - def run(self): - self.server = self._start_server() + def run(self) -> None: + self._start_server() -def ssl_options_to_context( +def ssl_options_to_context( # type: ignore[no-untyped-def] keyfile=None, certfile=None, server_side=None, cert_reqs=None, - ssl_version=None, + ssl_version: str | int | None = None, ca_certs=None, do_handshake_on_connect=None, suppress_ragged_eofs=None, ciphers=None, alpn_protocols=None, -): +) -> ssl.SSLContext: """Return an equivalent SSLContext based on ssl.wrap_socket args.""" ssl_version = resolve_ssl_version(ssl_version) cert_none = resolve_cert_reqs("CERT_NONE") @@ -159,62 +168,17 @@ def ssl_options_to_context( ctx.verify_mode = cert_reqs if ctx.verify_mode != cert_none: ctx.load_verify_locations(cafile=ca_certs) - if alpn_protocols and hasattr(ctx, "set_alpn_protocols"): - try: - ctx.set_alpn_protocols(alpn_protocols) - except NotImplementedError: - pass + if alpn_protocols: + ctx.set_alpn_protocols(alpn_protocols) return ctx -def run_tornado_app(app, io_loop, certs, scheme, host): - assert io_loop == tornado.ioloop.IOLoop.current() - - # We can't use fromtimestamp(0) because of CPython issue 29097, so we'll - # just construct the datetime object directly. - app.last_req = datetime(1970, 1, 1) - - if scheme == "https": - if sys.version_info < (2, 7, 9): - ssl_opts = certs - else: - ssl_opts = ssl_options_to_context(**certs) - http_server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_opts) - else: - http_server = tornado.httpserver.HTTPServer(app) - - sockets = tornado.netutil.bind_sockets(None, address=host) - port = sockets[0].getsockname()[1] - http_server.add_sockets(sockets) - return http_server, port - - -def run_loop_in_thread(io_loop): - t = threading.Thread(target=io_loop.start) - t.start() - return t - - -def get_unreachable_address(): +def get_unreachable_address() -> tuple[str, int]: # reserved as per rfc2606 return ("something.invalid", 54321) -if __name__ == "__main__": - # For debugging dummyserver itself - python -m dummyserver.server - from .testcase import TestingApp - - host = "127.0.0.1" - - io_loop = tornado.ioloop.IOLoop.current() - app = tornado.web.Application([(r".*", TestingApp)]) - server, port = run_tornado_app(app, io_loop, None, "http", host) - server_thread = run_loop_in_thread(io_loop) - - print("Listening on http://{host}:{port}".format(host=host, port=port)) - - -def encrypt_key_pem(private_key_pem, password): +def encrypt_key_pem(private_key_pem: trustme.Blob, password: bytes) -> trustme.Blob: private_key = serialization.load_pem_private_key( private_key_pem.bytes(), password=None, backend=default_backend() ) diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 6a49e36..eb4d981 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -1,32 +1,44 @@ +from __future__ import annotations + +import contextlib +import socket +import ssl import threading -from contextlib import contextmanager +import typing +from test import LONG_TIMEOUT import pytest -from tornado import ioloop, web - -from dummyserver.handlers import TestingApp -from dummyserver.proxy import ProxyHandler -from dummyserver.server import ( - DEFAULT_CERTS, - HAS_IPV6, - SocketServerThread, - run_loop_in_thread, - run_tornado_app, -) + +from dummyserver.app import hypercorn_app +from dummyserver.asgi_proxy import ProxyApp +from dummyserver.hypercornserver import run_hypercorn_in_thread +from dummyserver.socketserver import DEFAULT_CERTS, HAS_IPV6, SocketServerThread from urllib3.connection import HTTPConnection +from urllib3.util.ssltransport import SSLTransport -def consume_socket(sock, chunks=65536): +def consume_socket( + sock: SSLTransport | socket.socket, + chunks: int = 65536, + quit_event: threading.Event | None = None, +) -> bytearray: consumed = bytearray() + sock.settimeout(LONG_TIMEOUT) while True: - b = sock.recv(chunks) + if quit_event and quit_event.is_set(): + break + try: + b = sock.recv(chunks) + except (TimeoutError, socket.timeout): + continue + assert isinstance(b, bytes) consumed += b if b.endswith(b"\r\n\r\n"): break return consumed -class SocketDummyServerTestCase(object): +class SocketDummyServerTestCase: """ A simple socket-based server is created for this class that is good for exactly one request. @@ -35,11 +47,33 @@ class SocketDummyServerTestCase(object): scheme = "http" host = "localhost" + server_thread: typing.ClassVar[SocketServerThread] + port: typing.ClassVar[int] + + tmpdir: typing.ClassVar[str] + ca_path: typing.ClassVar[str] + cert_combined_path: typing.ClassVar[str] + cert_path: typing.ClassVar[str] + key_path: typing.ClassVar[str] + password_key_path: typing.ClassVar[str] + + server_context: typing.ClassVar[ssl.SSLContext] + client_context: typing.ClassVar[ssl.SSLContext] + + proxy_server: typing.ClassVar[SocketDummyServerTestCase] + @classmethod - def _start_server(cls, socket_handler): + def _start_server( + cls, + socket_handler: typing.Callable[[socket.socket], None], + quit_event: threading.Event | None = None, + ) -> None: ready_event = threading.Event() cls.server_thread = SocketServerThread( - socket_handler=socket_handler, ready_event=ready_event, host=cls.host + socket_handler=socket_handler, + ready_event=ready_event, + host=cls.host, + quit_event=quit_event, ) cls.server_thread.start() ready_event.wait(5) @@ -48,56 +82,107 @@ def _start_server(cls, socket_handler): cls.port = cls.server_thread.port @classmethod - def start_response_handler(cls, response, num=1, block_send=None): + def start_response_handler( + cls, + response: bytes, + num: int = 1, + block_send: threading.Event | None = None, + ) -> threading.Event: ready_event = threading.Event() + quit_event = threading.Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: for _ in range(num): ready_event.set() - sock = listener.accept()[0] - consume_socket(sock) + listener.settimeout(LONG_TIMEOUT) + while True: + if quit_event.is_set(): + return + try: + sock = listener.accept()[0] + break + except (TimeoutError, socket.timeout): + continue + consume_socket(sock, quit_event=quit_event) + if quit_event.is_set(): + sock.close() + return if block_send: - block_send.wait() + while not block_send.wait(LONG_TIMEOUT): + if quit_event.is_set(): + sock.close() + return block_send.clear() sock.send(response) sock.close() - cls._start_server(socket_handler) + cls._start_server(socket_handler, quit_event=quit_event) return ready_event @classmethod - def start_basic_handler(cls, **kw): + def start_basic_handler( + cls, num: int = 1, block_send: threading.Event | None = None + ) -> threading.Event: return cls.start_response_handler( - b"HTTP/1.1 200 OK\r\n" b"Content-Length: 0\r\n" b"\r\n", **kw + b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", + num, + block_send, ) + @staticmethod + def quit_server_thread(server_thread: SocketServerThread) -> None: + if server_thread.quit_event: + server_thread.quit_event.set() + # in principle the maximum time that the thread can take to notice + # the quit_event is LONG_TIMEOUT and the thread should terminate + # shortly after that, we give 5 seconds leeway just in case + server_thread.join(LONG_TIMEOUT * 2 + 5.0) + if server_thread.is_alive(): + raise Exception("server_thread did not exit") + @classmethod - def teardown_class(cls): + def teardown_class(cls) -> None: if hasattr(cls, "server_thread"): - cls.server_thread.join(0.1) + cls.quit_server_thread(cls.server_thread) + + def teardown_method(self) -> None: + if hasattr(self, "server_thread"): + self.quit_server_thread(self.server_thread) def assert_header_received( - self, received_headers, header_name, expected_value=None - ): - header_name = header_name.encode("ascii") - if expected_value is not None: - expected_value = expected_value.encode("ascii") + self, + received_headers: typing.Iterable[bytes], + header_name: str, + expected_value: str | None = None, + ) -> None: + header_name_bytes = header_name.encode("ascii") + if expected_value is None: + expected_value_bytes = None + else: + expected_value_bytes = expected_value.encode("ascii") header_titles = [] for header in received_headers: key, value = header.split(b": ") header_titles.append(key) - if key == header_name and expected_value is not None: - assert value == expected_value - assert header_name in header_titles + if key == header_name_bytes and expected_value_bytes is not None: + assert value == expected_value_bytes + assert header_name_bytes in header_titles class IPV4SocketDummyServerTestCase(SocketDummyServerTestCase): @classmethod - def _start_server(cls, socket_handler): + def _start_server( + cls, + socket_handler: typing.Callable[[socket.socket], None], + quit_event: threading.Event | None = None, + ) -> None: ready_event = threading.Event() cls.server_thread = SocketServerThread( - socket_handler=socket_handler, ready_event=ready_event, host=cls.host + socket_handler=socket_handler, + ready_event=ready_event, + host=cls.host, + quit_event=quit_event, ) cls.server_thread.USE_IPV6 = False cls.server_thread.start() @@ -107,109 +192,97 @@ def _start_server(cls, socket_handler): cls.port = cls.server_thread.port -class HTTPDummyServerTestCase(object): - """A simple HTTP server that runs when your test class runs - - Have your test class inherit from this one, and then a simple server - will start when your tests run, and automatically shut down when they - complete. For examples of what test requests you can send to the server, - see the TestingApp in dummyserver/handlers.py. - """ - - scheme = "http" +class HypercornDummyServerTestCase: host = "localhost" - host_alt = "127.0.0.1" # Some tests need two hosts - certs = DEFAULT_CERTS - - @classmethod - def _start_server(cls): - cls.io_loop = ioloop.IOLoop.current() - app = web.Application([(r".*", TestingApp)]) - cls.server, cls.port = run_tornado_app( - app, cls.io_loop, cls.certs, cls.scheme, cls.host - ) - cls.server_thread = run_loop_in_thread(cls.io_loop) + host_alt = "127.0.0.1" + port: typing.ClassVar[int] + base_url: typing.ClassVar[str] + base_url_alt: typing.ClassVar[str] + certs: typing.ClassVar[dict[str, typing.Any]] = {} - @classmethod - def _stop_server(cls): - cls.io_loop.add_callback(cls.server.stop) - cls.io_loop.add_callback(cls.io_loop.stop) - cls.server_thread.join() + _stack: typing.ClassVar[contextlib.ExitStack] @classmethod - def setup_class(cls): - cls._start_server() + def setup_class(cls) -> None: + with contextlib.ExitStack() as stack: + cls.port = stack.enter_context( + run_hypercorn_in_thread(cls.host, cls.certs, hypercorn_app) + ) + cls._stack = stack.pop_all() @classmethod - def teardown_class(cls): - cls._stop_server() + def teardown_class(cls) -> None: + cls._stack.close() -class HTTPSDummyServerTestCase(HTTPDummyServerTestCase): +class HTTPSHypercornDummyServerTestCase(HypercornDummyServerTestCase): scheme = "https" host = "localhost" certs = DEFAULT_CERTS - - -class HTTPDummyProxyTestCase(object): - - http_host = "localhost" - http_host_alt = "127.0.0.1" - - https_host = "localhost" - https_host_alt = "127.0.0.1" - https_certs = DEFAULT_CERTS - - proxy_host = "localhost" - proxy_host_alt = "127.0.0.1" + certs_dir = "" + bad_ca_path = "" + + +class HypercornDummyProxyTestCase: + http_host: typing.ClassVar[str] = "localhost" + http_host_alt: typing.ClassVar[str] = "127.0.0.1" + http_port: typing.ClassVar[int] + http_url: typing.ClassVar[str] + http_url_alt: typing.ClassVar[str] + + https_host: typing.ClassVar[str] = "localhost" + https_host_alt: typing.ClassVar[str] = "127.0.0.1" + https_certs: typing.ClassVar[dict[str, typing.Any]] = DEFAULT_CERTS + https_port: typing.ClassVar[int] + https_url: typing.ClassVar[str] + https_url_alt: typing.ClassVar[str] + https_url_fqdn: typing.ClassVar[str] + + proxy_host: typing.ClassVar[str] = "localhost" + proxy_host_alt: typing.ClassVar[str] = "127.0.0.1" + proxy_port: typing.ClassVar[int] + proxy_url: typing.ClassVar[str] + https_proxy_port: typing.ClassVar[int] + https_proxy_url: typing.ClassVar[str] + + certs_dir: typing.ClassVar[str] = "" + bad_ca_path: typing.ClassVar[str] = "" + + server_thread: typing.ClassVar[threading.Thread] + _stack: typing.ClassVar[contextlib.ExitStack] @classmethod - def setup_class(cls): - cls.io_loop = ioloop.IOLoop.current() - - app = web.Application([(r".*", TestingApp)]) - cls.http_server, cls.http_port = run_tornado_app( - app, cls.io_loop, None, "http", cls.http_host - ) - - app = web.Application([(r".*", TestingApp)]) - cls.https_server, cls.https_port = run_tornado_app( - app, cls.io_loop, cls.https_certs, "https", cls.http_host - ) - - app = web.Application([(r".*", ProxyHandler)]) - cls.proxy_server, cls.proxy_port = run_tornado_app( - app, cls.io_loop, None, "http", cls.proxy_host - ) - - upstream_ca_certs = cls.https_certs.get("ca_certs", None) - app = web.Application( - [(r".*", ProxyHandler)], upstream_ca_certs=upstream_ca_certs - ) - cls.https_proxy_server, cls.https_proxy_port = run_tornado_app( - app, cls.io_loop, cls.https_certs, "https", cls.proxy_host - ) - - cls.server_thread = run_loop_in_thread(cls.io_loop) + def setup_class(cls) -> None: + with contextlib.ExitStack() as stack: + cls.http_port = stack.enter_context( + run_hypercorn_in_thread(cls.http_host, None, hypercorn_app) + ) + cls.https_port = stack.enter_context( + run_hypercorn_in_thread(cls.https_host, cls.https_certs, hypercorn_app) + ) + cls.proxy_port = stack.enter_context( + run_hypercorn_in_thread(cls.proxy_host, None, ProxyApp()) + ) + upstream_ca_certs = cls.https_certs.get("ca_certs") + cls.https_proxy_port = stack.enter_context( + run_hypercorn_in_thread( + cls.proxy_host, cls.https_certs, ProxyApp(upstream_ca_certs) + ) + ) + cls._stack = stack.pop_all() @classmethod - def teardown_class(cls): - cls.io_loop.add_callback(cls.http_server.stop) - cls.io_loop.add_callback(cls.https_server.stop) - cls.io_loop.add_callback(cls.proxy_server.stop) - cls.io_loop.add_callback(cls.https_proxy_server.stop) - cls.io_loop.add_callback(cls.io_loop.stop) - cls.server_thread.join() + def teardown_class(cls) -> None: + cls._stack.close() @pytest.mark.skipif(not HAS_IPV6, reason="IPv6 not available") -class IPv6HTTPDummyServerTestCase(HTTPDummyServerTestCase): +class IPv6HypercornDummyServerTestCase(HypercornDummyServerTestCase): host = "::1" @pytest.mark.skipif(not HAS_IPV6, reason="IPv6 not available") -class IPv6HTTPDummyProxyTestCase(HTTPDummyProxyTestCase): - +class IPv6HypercornDummyProxyTestCase(HypercornDummyProxyTestCase): http_host = "localhost" http_host_alt = "127.0.0.1" @@ -221,7 +294,7 @@ class IPv6HTTPDummyProxyTestCase(HTTPDummyProxyTestCase): proxy_host_alt = "127.0.0.1" -class ConnectionMarker(object): +class ConnectionMarker: """ Marks an HTTP(S)Connection's socket after a request was made. @@ -232,32 +305,31 @@ class ConnectionMarker(object): MARK_FORMAT = b"$#MARK%04x*!" @classmethod - @contextmanager - def mark(cls, monkeypatch): + @contextlib.contextmanager + def mark(cls, monkeypatch: pytest.MonkeyPatch) -> typing.Generator[None]: """ Mark connections under in that context. """ orig_request = HTTPConnection.request - orig_request_chunked = HTTPConnection.request_chunked - def call_and_mark(target): - def part(self, *args, **kwargs): - result = target(self, *args, **kwargs) + def call_and_mark( + target: typing.Callable[..., None] + ) -> typing.Callable[..., None]: + def part( + self: HTTPConnection, *args: typing.Any, **kwargs: typing.Any + ) -> None: + target(self, *args, **kwargs) self.sock.sendall(cls._get_socket_mark(self.sock, False)) - return result return part with monkeypatch.context() as m: m.setattr(HTTPConnection, "request", call_and_mark(orig_request)) - m.setattr( - HTTPConnection, "request_chunked", call_and_mark(orig_request_chunked) - ) yield @classmethod - def consume_request(cls, sock, chunks=65536): + def consume_request(cls, sock: socket.socket, chunks: int = 65536) -> bytearray: """ Consume a socket until after the HTTP request is sent. """ @@ -273,7 +345,7 @@ def consume_request(cls, sock, chunks=65536): return consumed @classmethod - def _get_socket_mark(cls, sock, server): + def _get_socket_mark(cls, sock: socket.socket, server: bool) -> bytes: if server: port = sock.getpeername()[1] else: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b40f6cf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,136 @@ +# This file is protected via CODEOWNERS + +[build-system] +requires = ["hatchling>=1.6.0,<2", "hatch-vcs==0.4.0"] +build-backend = "hatchling.build" + +[project] +name = "urllib3" +description = "HTTP library with thread-safe connection pooling, file post, and more." +readme = "README.md" +keywords = ["urllib", "httplib", "threadsafe", "filepost", "http", "https", "ssl", "pooling"] +authors = [ + {name = "Andrey Petrov", email = "andrey.petrov@shazow.net"} +] +maintainers = [ + {name = "Seth Michael Larson", email="sethmichaellarson@gmail.com"}, + {name = "Quentin Pradet", email="quentin@pradet.me"}, + {name = "Illia Volochii", email = "illia.volochii@gmail.com"}, +] +classifiers = [ + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries", +] +requires-python = ">=3.9" +dynamic = ["version"] + +[project.optional-dependencies] +brotli = [ + "brotli>=1.0.9; platform_python_implementation == 'CPython'", + "brotlicffi>=0.8.0; platform_python_implementation != 'CPython'" +] +zstd = [ + "zstandard>=0.18.0", +] +socks = [ + "PySocks>=1.5.6,<2.0,!=1.5.7", +] +h2 = [ + "h2>=4,<5" +] + +[project.urls] +"Changelog" = "https://github.com/urllib3/urllib3/blob/main/CHANGES.rst" +"Documentation" = "https://urllib3.readthedocs.io" +"Code" = "https://github.com/urllib3/urllib3" +"Issue tracker" = "https://github.com/urllib3/urllib3/issues" + +[tool.hatch.version] +source = "vcs" +[tool.hatch.version.raw-options] +local_scheme = "no-local-version" +[tool.hatch.build.hooks.vcs] +version-file = "src/urllib3/_version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/docs", + "/dummyserver", + "/src", + "/test", + "/dev-requirements.txt", + "/CHANGES.rst", + "/README.md", + "/LICENSE.txt", +] + +[tool.pytest.ini_options] +xfail_strict = true +python_classes = ["Test", "*TestCase"] +markers = [ + "limit_memory: Limit memory with memray", + "requires_network: This test needs access to the Internet", + "integration: Slow integrations tests not run by default", + "in_webbrowser: Emscripten - run only in browser (not in Node.js)", + "with_jspi: Emscripten - run only if WebAssembly JavaScript Promise Integration is supported", + "without_jspi: Emscripten - run only if this platform works without WebAssembly JavaScript Promise Integration", + "webworkers: Emscripten - run only if this platform can test web workers", + "node_without_jspi: Emscripten - run in node with jspi enabled (for failure testing only)", +] +log_level = "DEBUG" +filterwarnings = [ + "error", + '''default:urllib3 v2 only supports OpenSSL 1.1.1+.*''', + '''default:No IPv6 support. Falling back to IPv4:urllib3.exceptions.HTTPWarning''', + '''default:No IPv6 support. skipping:urllib3.exceptions.HTTPWarning''', + '''default:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning''', + '''default:ssl\.PROTOCOL_TLS is deprecated:DeprecationWarning''', + '''default:ssl\.PROTOCOL_TLSv1 is deprecated:DeprecationWarning''', + '''default:ssl\.TLSVersion\.TLSv1_1 is deprecated:DeprecationWarning''', + '''default:ssl\.PROTOCOL_TLSv1_1 is deprecated:DeprecationWarning''', + '''default:ssl\.PROTOCOL_TLSv1_2 is deprecated:DeprecationWarning''', + '''default:ssl NPN is deprecated, use ALPN instead:DeprecationWarning''', + # https://github.com/SeleniumHQ/selenium/issues/13328 + '''default:unclosed file <_io\.BufferedWriter name='/dev/null'>:ResourceWarning''', + # https://github.com/SeleniumHQ/selenium/issues/14686 + '''default:setting remote_server_addr in RemoteConnection\(\) is deprecated, set in ClientConfig instance instead:DeprecationWarning''' +] + +[tool.isort] +profile = "black" +add_imports = "from __future__ import annotations" + +[tool.mypy] +mypy_path = "src" +check_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_decorators = true +disallow_untyped_defs = true +no_implicit_optional = true +no_implicit_reexport = true +show_error_codes = true +strict_equality = true +warn_redundant_casts = true +warn_return_any = true +warn_unused_configs = true +warn_unused_ignores = true +enable_error_code = [ + "ignore-without-code", +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 61b540c..0000000 --- a/setup.cfg +++ /dev/null @@ -1,37 +0,0 @@ -[flake8] -ignore = E501, E203, W503, W504 -exclude = ./docs/conf.py,./src/urllib3/packages/* -max-line-length = 99 - -[bdist_wheel] -universal = 1 - -[metadata] -license_file = LICENSE.txt -provides-extra = - secure - socks - brotli -requires-dist = - pyOpenSSL>=0.14; extra == 'secure' - cryptography>=1.3.4; extra == 'secure' - idna>=2.0.0; extra == 'secure' - certifi; extra == 'secure' - ipaddress; python_version=="2.7" and extra == 'secure' - urllib3-secure-extra; extra == 'secure' - PySocks>=1.5.6,<2.0,!=1.5.7; extra == 'socks' - brotli>=1.0.9; (os_name != 'nt' or python_version >= '3') and platform_python_implementation == 'CPython' and extra == 'brotli' - brotlicffi>=0.8.0; (os_name != 'nt' or python_version >= '3') and platform_python_implementation != 'CPython' and extra == 'brotli' - brotlipy>=0.6.0; os_name == 'nt' and python_version < '3' and extra == 'brotli' - -[tool:pytest] -xfail_strict = true -python_classes = Test *TestCase - -[isort] -profile = black - -[egg_info] -tag_build = -tag_date = 0 - diff --git a/setup.py b/setup.py deleted file mode 100755 index fb0bed7..0000000 --- a/setup.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python -# This file is protected via CODEOWNERS - -import codecs -import os -import re - -from setuptools import setup - -base_path = os.path.dirname(__file__) - -# Get the version (borrowed from SQLAlchemy) -with open(os.path.join(base_path, "src", "urllib3", "_version.py")) as fp: - VERSION = ( - re.compile(r""".*__version__ = ["'](.*?)['"]""", re.S).match(fp.read()).group(1) - ) - - -with codecs.open("README.rst", encoding="utf-8") as fp: - # Remove reST raw directive from README as they're not allowed on PyPI - # Those blocks start with a newline and continue until the next newline - mode = None - lines = [] - for line in fp: - if line.startswith(".. raw::"): - mode = "ignore_nl" - elif line == "\n": - mode = "wait_nl" if mode == "ignore_nl" else None - - if mode is None: - lines.append(line) - readme = "".join(lines) - -with codecs.open("CHANGES.rst", encoding="utf-8") as fp: - changes = fp.read() - -version = VERSION - -setup( - name="urllib3", - version=version, - description="HTTP library with thread-safe connection pooling, file post, and more.", - long_description=u"\n\n".join([readme, changes]), - long_description_content_type="text/x-rst", - classifiers=[ - "Environment :: Web Environment", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development :: Libraries", - ], - keywords="urllib httplib threadsafe filepost http https ssl pooling", - author="Andrey Petrov", - author_email="andrey.petrov@shazow.net", - url="https://urllib3.readthedocs.io/", - project_urls={ - "Documentation": "https://urllib3.readthedocs.io/", - "Code": "https://github.com/urllib3/urllib3", - "Issue tracker": "https://github.com/urllib3/urllib3/issues", - }, - license="MIT", - packages=[ - "urllib3", - "urllib3.packages", - "urllib3.packages.backports", - "urllib3.contrib", - "urllib3.contrib._securetransport", - "urllib3.util", - ], - package_dir={"": "src"}, - requires=[], - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", - extras_require={ - "brotli": [ - # https://github.com/google/brotli/issues/1074 - "brotli==1.0.9; os_name != 'nt' and python_version < '3' and platform_python_implementation == 'CPython'", - "brotli>=1.0.9; python_version >= '3' and platform_python_implementation == 'CPython'", - "brotlicffi>=0.8.0; (os_name != 'nt' or python_version >= '3') and platform_python_implementation != 'CPython'", - "brotlipy>=0.6.0; os_name == 'nt' and python_version < '3'", - ], - "secure": [ - "pyOpenSSL>=0.14", - "cryptography>=1.3.4", - "idna>=2.0.0", - "certifi", - "ipaddress; python_version=='2.7'", - "urllib3-secure-extra", - ], - "socks": ["PySocks>=1.5.6,<2.0,!=1.5.7"], - }, -) diff --git a/src/urllib3.egg-info/PKG-INFO b/src/urllib3.egg-info/PKG-INFO deleted file mode 100644 index 4336677..0000000 --- a/src/urllib3.egg-info/PKG-INFO +++ /dev/null @@ -1,1492 +0,0 @@ -Metadata-Version: 2.1 -Name: urllib3 -Version: 1.26.18 -Summary: HTTP library with thread-safe connection pooling, file post, and more. -Home-page: https://urllib3.readthedocs.io/ -Author: Andrey Petrov -Author-email: andrey.petrov@shazow.net -License: MIT -Project-URL: Documentation, https://urllib3.readthedocs.io/ -Project-URL: Code, https://github.com/urllib3/urllib3 -Project-URL: Issue tracker, https://github.com/urllib3/urllib3/issues -Keywords: urllib httplib threadsafe filepost http https ssl pooling -Classifier: Environment :: Web Environment -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Internet :: WWW/HTTP -Classifier: Topic :: Software Development :: Libraries -Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* -Description-Content-Type: text/x-rst -License-File: LICENSE.txt -Provides-Extra: brotli -Requires-Dist: brotli==1.0.9; (os_name != "nt" and python_version < "3" and platform_python_implementation == "CPython") and extra == "brotli" -Requires-Dist: brotli>=1.0.9; (python_version >= "3" and platform_python_implementation == "CPython") and extra == "brotli" -Requires-Dist: brotlicffi>=0.8.0; ((os_name != "nt" or python_version >= "3") and platform_python_implementation != "CPython") and extra == "brotli" -Requires-Dist: brotlipy>=0.6.0; (os_name == "nt" and python_version < "3") and extra == "brotli" -Provides-Extra: secure -Requires-Dist: pyOpenSSL>=0.14; extra == "secure" -Requires-Dist: cryptography>=1.3.4; extra == "secure" -Requires-Dist: idna>=2.0.0; extra == "secure" -Requires-Dist: certifi; extra == "secure" -Requires-Dist: ipaddress; python_version == "2.7" and extra == "secure" -Requires-Dist: urllib3-secure-extra; extra == "secure" -Provides-Extra: socks -Requires-Dist: PySocks!=1.5.7,<2.0,>=1.5.6; extra == "socks" - - -urllib3 is a powerful, *user-friendly* HTTP client for Python. Much of the -Python ecosystem already uses urllib3 and you should too. -urllib3 brings many critical features that are missing from the Python -standard libraries: - -- Thread safety. -- Connection pooling. -- Client-side SSL/TLS verification. -- File uploads with multipart encoding. -- Helpers for retrying requests and dealing with HTTP redirects. -- Support for gzip, deflate, and brotli encoding. -- Proxy support for HTTP and SOCKS. -- 100% test coverage. - -urllib3 is powerful and easy to use: - -.. code-block:: python - - >>> import urllib3 - >>> http = urllib3.PoolManager() - >>> r = http.request('GET', 'http://httpbin.org/robots.txt') - >>> r.status - 200 - >>> r.data - 'User-agent: *\nDisallow: /deny\n' - - -Installing ----------- - -urllib3 can be installed with `pip `_:: - - $ python -m pip install urllib3 - -Alternatively, you can grab the latest source code from `GitHub `_:: - - $ git clone https://github.com/urllib3/urllib3.git - $ cd urllib3 - $ git checkout 1.26.x - $ pip install . - - -Documentation -------------- - -urllib3 has usage and reference documentation at `urllib3.readthedocs.io `_. - - -Contributing ------------- - -urllib3 happily accepts contributions. Please see our -`contributing documentation `_ -for some tips on getting started. - - -Security Disclosures --------------------- - -To report a security vulnerability, please use the -`Tidelift security contact `_. -Tidelift will coordinate the fix and disclosure with maintainers. - - -Maintainers ------------ - -- `@sethmlarson `__ (Seth M. Larson) -- `@pquentin `__ (Quentin Pradet) -- `@theacodes `__ (Thea Flowers) -- `@haikuginger `__ (Jess Shapiro) -- `@lukasa `__ (Cory Benfield) -- `@sigmavirus24 `__ (Ian Stapleton Cordasco) -- `@shazow `__ (Andrey Petrov) - -👋 - - -Sponsorship ------------ - -If your company benefits from this library, please consider `sponsoring its -development `_. - - -For Enterprise --------------- - -.. |tideliftlogo| image:: https://nedbatchelder.com/pix/Tidelift_Logos_RGB_Tidelift_Shorthand_On-White_small.png - :width: 75 - :alt: Tidelift - -.. list-table:: - :widths: 10 100 - - * - |tideliftlogo| - - Professional support for urllib3 is available as part of the `Tidelift - Subscription`_. Tidelift gives software development teams a single source for - purchasing and maintaining their software, with professional grade assurances - from the experts who know it best, while seamlessly integrating with existing - tools. - -.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-urllib3?utm_source=pypi-urllib3&utm_medium=referral&utm_campaign=readme - - -Changes -======= - -1.26.18 (2023-10-17) --------------------- - -* Made body stripped from HTTP requests changing the request method to GET after HTTP 303 "See Other" redirect responses. - - -1.26.17 (2023-10-02) --------------------- - -* Added the ``Cookie`` header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via ``Retry.remove_headers_on_redirect``. - - -1.26.16 (2023-05-23) --------------------- - -* Fixed thread-safety issue where accessing a ``PoolManager`` with many distinct origins - would cause connection pools to be closed while requests are in progress (`#2954 `_) - - -1.26.15 (2023-03-10) --------------------- - -* Fix socket timeout value when ``HTTPConnection`` is reused (`#2645 `__) -* Remove "!" character from the unreserved characters in IPv6 Zone ID parsing - (`#2899 `__) -* Fix IDNA handling of '\x80' byte (`#2901 `__) - -1.26.14 (2023-01-11) --------------------- - -* Fixed parsing of port 0 (zero) returning None, instead of 0. (`#2850 `__) -* Removed deprecated getheaders() calls in contrib module. - -1.26.13 (2022-11-23) --------------------- - -* Deprecated the ``HTTPResponse.getheaders()`` and ``HTTPResponse.getheader()`` methods. -* Fixed an issue where parsing a URL with leading zeroes in the port would be rejected - even when the port number after removing the zeroes was valid. -* Fixed a deprecation warning when using cryptography v39.0.0. -* Removed the ``<4`` in the ``Requires-Python`` packaging metadata field. - - -1.26.12 (2022-08-22) --------------------- - -* Deprecated the `urllib3[secure]` extra and the `urllib3.contrib.pyopenssl` module. - Both will be removed in v2.x. See this `GitHub issue `_ - for justification and info on how to migrate. - - -1.26.11 (2022-07-25) --------------------- - -* Fixed an issue where reading more than 2 GiB in a call to ``HTTPResponse.read`` would - raise an ``OverflowError`` on Python 3.9 and earlier. - - -1.26.10 (2022-07-07) --------------------- - -* Removed support for Python 3.5 -* Fixed an issue where a ``ProxyError`` recommending configuring the proxy as HTTP - instead of HTTPS could appear even when an HTTPS proxy wasn't configured. - - -1.26.9 (2022-03-16) -------------------- - -* Changed ``urllib3[brotli]`` extra to favor installing Brotli libraries that are still - receiving updates like ``brotli`` and ``brotlicffi`` instead of ``brotlipy``. - This change does not impact behavior of urllib3, only which dependencies are installed. -* Fixed a socket leaking when ``HTTPSConnection.connect()`` raises an exception. -* Fixed ``server_hostname`` being forwarded from ``PoolManager`` to ``HTTPConnectionPool`` - when requesting an HTTP URL. Should only be forwarded when requesting an HTTPS URL. - - -1.26.8 (2022-01-07) -------------------- - -* Added extra message to ``urllib3.exceptions.ProxyError`` when urllib3 detects that - a proxy is configured to use HTTPS but the proxy itself appears to only use HTTP. -* Added a mention of the size of the connection pool when discarding a connection due to the pool being full. -* Added explicit support for Python 3.11. -* Deprecated the ``Retry.MAX_BACKOFF`` class property in favor of ``Retry.DEFAULT_MAX_BACKOFF`` - to better match the rest of the default parameter names. ``Retry.MAX_BACKOFF`` is removed in v2.0. -* Changed location of the vendored ``ssl.match_hostname`` function from ``urllib3.packages.ssl_match_hostname`` - to ``urllib3.util.ssl_match_hostname`` to ensure Python 3.10+ compatibility after being repackaged - by downstream distributors. -* Fixed absolute imports, all imports are now relative. - - -1.26.7 (2021-09-22) -------------------- - -* Fixed a bug with HTTPS hostname verification involving IP addresses and lack - of SNI. (Issue #2400) -* Fixed a bug where IPv6 braces weren't stripped during certificate hostname - matching. (Issue #2240) - - -1.26.6 (2021-06-25) -------------------- - -* Deprecated the ``urllib3.contrib.ntlmpool`` module. urllib3 is not able to support - it properly due to `reasons listed in this issue `_. - If you are a user of this module please leave a comment. -* Changed ``HTTPConnection.request_chunked()`` to not erroneously emit multiple - ``Transfer-Encoding`` headers in the case that one is already specified. -* Fixed typo in deprecation message to recommend ``Retry.DEFAULT_ALLOWED_METHODS``. - - -1.26.5 (2021-05-26) -------------------- - -* Fixed deprecation warnings emitted in Python 3.10. -* Updated vendored ``six`` library to 1.16.0. -* Improved performance of URL parser when splitting - the authority component. - - -1.26.4 (2021-03-15) -------------------- - -* Changed behavior of the default ``SSLContext`` when connecting to HTTPS proxy - during HTTPS requests. The default ``SSLContext`` now sets ``check_hostname=True``. - - -1.26.3 (2021-01-26) -------------------- - -* Fixed bytes and string comparison issue with headers (Pull #2141) - -* Changed ``ProxySchemeUnknown`` error message to be - more actionable if the user supplies a proxy URL without - a scheme. (Pull #2107) - - -1.26.2 (2020-11-12) -------------------- - -* Fixed an issue where ``wrap_socket`` and ``CERT_REQUIRED`` wouldn't - be imported properly on Python 2.7.8 and earlier (Pull #2052) - - -1.26.1 (2020-11-11) -------------------- - -* Fixed an issue where two ``User-Agent`` headers would be sent if a - ``User-Agent`` header key is passed as ``bytes`` (Pull #2047) - - -1.26.0 (2020-11-10) -------------------- - -* **NOTE: urllib3 v2.0 will drop support for Python 2**. - `Read more in the v2.0 Roadmap `_. - -* Added support for HTTPS proxies contacting HTTPS servers (Pull #1923, Pull #1806) - -* Deprecated negotiating TLSv1 and TLSv1.1 by default. Users that - still wish to use TLS earlier than 1.2 without a deprecation warning - should opt-in explicitly by setting ``ssl_version=ssl.PROTOCOL_TLSv1_1`` (Pull #2002) - **Starting in urllib3 v2.0: Connections that receive a ``DeprecationWarning`` will fail** - -* Deprecated ``Retry`` options ``Retry.DEFAULT_METHOD_WHITELIST``, ``Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST`` - and ``Retry(method_whitelist=...)`` in favor of ``Retry.DEFAULT_ALLOWED_METHODS``, - ``Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT``, and ``Retry(allowed_methods=...)`` - (Pull #2000) **Starting in urllib3 v2.0: Deprecated options will be removed** - -* Added default ``User-Agent`` header to every request (Pull #1750) - -* Added ``urllib3.util.SKIP_HEADER`` for skipping ``User-Agent``, ``Accept-Encoding``, - and ``Host`` headers from being automatically emitted with requests (Pull #2018) - -* Collapse ``transfer-encoding: chunked`` request data and framing into - the same ``socket.send()`` call (Pull #1906) - -* Send ``http/1.1`` ALPN identifier with every TLS handshake by default (Pull #1894) - -* Properly terminate SecureTransport connections when CA verification fails (Pull #1977) - -* Don't emit an ``SNIMissingWarning`` when passing ``server_hostname=None`` - to SecureTransport (Pull #1903) - -* Disabled requesting TLSv1.2 session tickets as they weren't being used by urllib3 (Pull #1970) - -* Suppress ``BrokenPipeError`` when writing request body after the server - has closed the socket (Pull #1524) - -* Wrap ``ssl.SSLError`` that can be raised from reading a socket (e.g. "bad MAC") - into an ``urllib3.exceptions.SSLError`` (Pull #1939) - - -1.25.11 (2020-10-19) --------------------- - -* Fix retry backoff time parsed from ``Retry-After`` header when given - in the HTTP date format. The HTTP date was parsed as the local timezone - rather than accounting for the timezone in the HTTP date (typically - UTC) (Pull #1932, Pull #1935, Pull #1938, Pull #1949) - -* Fix issue where an error would be raised when the ``SSLKEYLOGFILE`` - environment variable was set to the empty string. Now ``SSLContext.keylog_file`` - is not set in this situation (Pull #2016) - - -1.25.10 (2020-07-22) --------------------- - -* Added support for ``SSLKEYLOGFILE`` environment variable for - logging TLS session keys with use with programs like - Wireshark for decrypting captured web traffic (Pull #1867) - -* Fixed loading of SecureTransport libraries on macOS Big Sur - due to the new dynamic linker cache (Pull #1905) - -* Collapse chunked request bodies data and framing into one - call to ``send()`` to reduce the number of TCP packets by 2-4x (Pull #1906) - -* Don't insert ``None`` into ``ConnectionPool`` if the pool - was empty when requesting a connection (Pull #1866) - -* Avoid ``hasattr`` call in ``BrotliDecoder.decompress()`` (Pull #1858) - - -1.25.9 (2020-04-16) -------------------- - -* Added ``InvalidProxyConfigurationWarning`` which is raised when - erroneously specifying an HTTPS proxy URL. urllib3 doesn't currently - support connecting to HTTPS proxies but will soon be able to - and we would like users to migrate properly without much breakage. - - See `this GitHub issue `_ - for more information on how to fix your proxy config. (Pull #1851) - -* Drain connection after ``PoolManager`` redirect (Pull #1817) - -* Ensure ``load_verify_locations`` raises ``SSLError`` for all backends (Pull #1812) - -* Rename ``VerifiedHTTPSConnection`` to ``HTTPSConnection`` (Pull #1805) - -* Allow the CA certificate data to be passed as a string (Pull #1804) - -* Raise ``ValueError`` if method contains control characters (Pull #1800) - -* Add ``__repr__`` to ``Timeout`` (Pull #1795) - - -1.25.8 (2020-01-20) -------------------- - -* Drop support for EOL Python 3.4 (Pull #1774) - -* Optimize _encode_invalid_chars (Pull #1787) - - -1.25.7 (2019-11-11) -------------------- - -* Preserve ``chunked`` parameter on retries (Pull #1715, Pull #1734) - -* Allow unset ``SERVER_SOFTWARE`` in App Engine (Pull #1704, Issue #1470) - -* Fix issue where URL fragment was sent within the request target. (Pull #1732) - -* Fix issue where an empty query section in a URL would fail to parse. (Pull #1732) - -* Remove TLS 1.3 support in SecureTransport due to Apple removing support (Pull #1703) - - -1.25.6 (2019-09-24) -------------------- - -* Fix issue where tilde (``~``) characters were incorrectly - percent-encoded in the path. (Pull #1692) - - -1.25.5 (2019-09-19) -------------------- - -* Add mitigation for BPO-37428 affecting Python <3.7.4 and OpenSSL 1.1.1+ which - caused certificate verification to be enabled when using ``cert_reqs=CERT_NONE``. - (Issue #1682) - - -1.25.4 (2019-09-19) -------------------- - -* Propagate Retry-After header settings to subsequent retries. (Pull #1607) - -* Fix edge case where Retry-After header was still respected even when - explicitly opted out of. (Pull #1607) - -* Remove dependency on ``rfc3986`` for URL parsing. - -* Fix issue where URLs containing invalid characters within ``Url.auth`` would - raise an exception instead of percent-encoding those characters. - -* Add support for ``HTTPResponse.auto_close = False`` which makes HTTP responses - work well with BufferedReaders and other ``io`` module features. (Pull #1652) - -* Percent-encode invalid characters in URL for ``HTTPConnectionPool.request()`` (Pull #1673) - - -1.25.3 (2019-05-23) -------------------- - -* Change ``HTTPSConnection`` to load system CA certificates - when ``ca_certs``, ``ca_cert_dir``, and ``ssl_context`` are - unspecified. (Pull #1608, Issue #1603) - -* Upgrade bundled rfc3986 to v1.3.2. (Pull #1609, Issue #1605) - - -1.25.2 (2019-04-28) -------------------- - -* Change ``is_ipaddress`` to not detect IPvFuture addresses. (Pull #1583) - -* Change ``parse_url`` to percent-encode invalid characters within the - path, query, and target components. (Pull #1586) - - -1.25.1 (2019-04-24) -------------------- - -* Add support for Google's ``Brotli`` package. (Pull #1572, Pull #1579) - -* Upgrade bundled rfc3986 to v1.3.1 (Pull #1578) - - -1.25 (2019-04-22) ------------------ - -* Require and validate certificates by default when using HTTPS (Pull #1507) - -* Upgraded ``urllib3.utils.parse_url()`` to be RFC 3986 compliant. (Pull #1487) - -* Added support for ``key_password`` for ``HTTPSConnectionPool`` to use - encrypted ``key_file`` without creating your own ``SSLContext`` object. (Pull #1489) - -* Add TLSv1.3 support to CPython, pyOpenSSL, and SecureTransport ``SSLContext`` - implementations. (Pull #1496) - -* Switched the default multipart header encoder from RFC 2231 to HTML 5 working draft. (Issue #303, Pull #1492) - -* Fixed issue where OpenSSL would block if an encrypted client private key was - given and no password was given. Instead an ``SSLError`` is raised. (Pull #1489) - -* Added support for Brotli content encoding. It is enabled automatically if - ``brotlipy`` package is installed which can be requested with - ``urllib3[brotli]`` extra. (Pull #1532) - -* Drop ciphers using DSS key exchange from default TLS cipher suites. - Improve default ciphers when using SecureTransport. (Pull #1496) - -* Implemented a more efficient ``HTTPResponse.__iter__()`` method. (Issue #1483) - -1.24.3 (2019-05-01) -------------------- - -* Apply fix for CVE-2019-9740. (Pull #1591) - -1.24.2 (2019-04-17) -------------------- - -* Don't load system certificates by default when any other ``ca_certs``, ``ca_certs_dir`` or - ``ssl_context`` parameters are specified. - -* Remove Authorization header regardless of case when redirecting to cross-site. (Issue #1510) - -* Add support for IPv6 addresses in subjectAltName section of certificates. (Issue #1269) - - -1.24.1 (2018-11-02) -------------------- - -* Remove quadratic behavior within ``GzipDecoder.decompress()`` (Issue #1467) - -* Restored functionality of ``ciphers`` parameter for ``create_urllib3_context()``. (Issue #1462) - - -1.24 (2018-10-16) ------------------ - -* Allow key_server_hostname to be specified when initializing a PoolManager to allow custom SNI to be overridden. (Pull #1449) - -* Test against Python 3.7 on AppVeyor. (Pull #1453) - -* Early-out ipv6 checks when running on App Engine. (Pull #1450) - -* Change ambiguous description of backoff_factor (Pull #1436) - -* Add ability to handle multiple Content-Encodings (Issue #1441 and Pull #1442) - -* Skip DNS names that can't be idna-decoded when using pyOpenSSL (Issue #1405). - -* Add a server_hostname parameter to HTTPSConnection which allows for - overriding the SNI hostname sent in the handshake. (Pull #1397) - -* Drop support for EOL Python 2.6 (Pull #1429 and Pull #1430) - -* Fixed bug where responses with header Content-Type: message/* erroneously - raised HeaderParsingError, resulting in a warning being logged. (Pull #1439) - -* Move urllib3 to src/urllib3 (Pull #1409) - - -1.23 (2018-06-04) ------------------ - -* Allow providing a list of headers to strip from requests when redirecting - to a different host. Defaults to the ``Authorization`` header. Different - headers can be set via ``Retry.remove_headers_on_redirect``. (Issue #1316) - -* Fix ``util.selectors._fileobj_to_fd`` to accept ``long`` (Issue #1247). - -* Dropped Python 3.3 support. (Pull #1242) - -* Put the connection back in the pool when calling stream() or read_chunked() on - a chunked HEAD response. (Issue #1234) - -* Fixed pyOpenSSL-specific ssl client authentication issue when clients - attempted to auth via certificate + chain (Issue #1060) - -* Add the port to the connectionpool connect print (Pull #1251) - -* Don't use the ``uuid`` module to create multipart data boundaries. (Pull #1380) - -* ``read_chunked()`` on a closed response returns no chunks. (Issue #1088) - -* Add Python 2.6 support to ``contrib.securetransport`` (Pull #1359) - -* Added support for auth info in url for SOCKS proxy (Pull #1363) - - -1.22 (2017-07-20) ------------------ - -* Fixed missing brackets in ``HTTP CONNECT`` when connecting to IPv6 address via - IPv6 proxy. (Issue #1222) - -* Made the connection pool retry on ``SSLError``. The original ``SSLError`` - is available on ``MaxRetryError.reason``. (Issue #1112) - -* Drain and release connection before recursing on retry/redirect. Fixes - deadlocks with a blocking connectionpool. (Issue #1167) - -* Fixed compatibility for cookiejar. (Issue #1229) - -* pyopenssl: Use vendored version of ``six``. (Issue #1231) - - -1.21.1 (2017-05-02) -------------------- - -* Fixed SecureTransport issue that would cause long delays in response body - delivery. (Pull #1154) - -* Fixed regression in 1.21 that threw exceptions when users passed the - ``socket_options`` flag to the ``PoolManager``. (Issue #1165) - -* Fixed regression in 1.21 that threw exceptions when users passed the - ``assert_hostname`` or ``assert_fingerprint`` flag to the ``PoolManager``. - (Pull #1157) - - -1.21 (2017-04-25) ------------------ - -* Improved performance of certain selector system calls on Python 3.5 and - later. (Pull #1095) - -* Resolved issue where the PyOpenSSL backend would not wrap SysCallError - exceptions appropriately when sending data. (Pull #1125) - -* Selectors now detects a monkey-patched select module after import for modules - that patch the select module like eventlet, greenlet. (Pull #1128) - -* Reduced memory consumption when streaming zlib-compressed responses - (as opposed to raw deflate streams). (Pull #1129) - -* Connection pools now use the entire request context when constructing the - pool key. (Pull #1016) - -* ``PoolManager.connection_from_*`` methods now accept a new keyword argument, - ``pool_kwargs``, which are merged with the existing ``connection_pool_kw``. - (Pull #1016) - -* Add retry counter for ``status_forcelist``. (Issue #1147) - -* Added ``contrib`` module for using SecureTransport on macOS: - ``urllib3.contrib.securetransport``. (Pull #1122) - -* urllib3 now only normalizes the case of ``http://`` and ``https://`` schemes: - for schemes it does not recognise, it assumes they are case-sensitive and - leaves them unchanged. - (Issue #1080) - - -1.20 (2017-01-19) ------------------ - -* Added support for waiting for I/O using selectors other than select, - improving urllib3's behaviour with large numbers of concurrent connections. - (Pull #1001) - -* Updated the date for the system clock check. (Issue #1005) - -* ConnectionPools now correctly consider hostnames to be case-insensitive. - (Issue #1032) - -* Outdated versions of PyOpenSSL now cause the PyOpenSSL contrib module - to fail when it is injected, rather than at first use. (Pull #1063) - -* Outdated versions of cryptography now cause the PyOpenSSL contrib module - to fail when it is injected, rather than at first use. (Issue #1044) - -* Automatically attempt to rewind a file-like body object when a request is - retried or redirected. (Pull #1039) - -* Fix some bugs that occur when modules incautiously patch the queue module. - (Pull #1061) - -* Prevent retries from occurring on read timeouts for which the request method - was not in the method whitelist. (Issue #1059) - -* Changed the PyOpenSSL contrib module to lazily load idna to avoid - unnecessarily bloating the memory of programs that don't need it. (Pull - #1076) - -* Add support for IPv6 literals with zone identifiers. (Pull #1013) - -* Added support for socks5h:// and socks4a:// schemes when working with SOCKS - proxies, and controlled remote DNS appropriately. (Issue #1035) - - -1.19.1 (2016-11-16) -------------------- - -* Fixed AppEngine import that didn't function on Python 3.5. (Pull #1025) - - -1.19 (2016-11-03) ------------------ - -* urllib3 now respects Retry-After headers on 413, 429, and 503 responses when - using the default retry logic. (Pull #955) - -* Remove markers from setup.py to assist ancient setuptools versions. (Issue - #986) - -* Disallow superscripts and other integerish things in URL ports. (Issue #989) - -* Allow urllib3's HTTPResponse.stream() method to continue to work with - non-httplib underlying FPs. (Pull #990) - -* Empty filenames in multipart headers are now emitted as such, rather than - being suppressed. (Issue #1015) - -* Prefer user-supplied Host headers on chunked uploads. (Issue #1009) - - -1.18.1 (2016-10-27) -------------------- - -* CVE-2016-9015. Users who are using urllib3 version 1.17 or 1.18 along with - PyOpenSSL injection and OpenSSL 1.1.0 *must* upgrade to this version. This - release fixes a vulnerability whereby urllib3 in the above configuration - would silently fail to validate TLS certificates due to erroneously setting - invalid flags in OpenSSL's ``SSL_CTX_set_verify`` function. These erroneous - flags do not cause a problem in OpenSSL versions before 1.1.0, which - interprets the presence of any flag as requesting certificate validation. - - There is no PR for this patch, as it was prepared for simultaneous disclosure - and release. The master branch received the same fix in Pull #1010. - - -1.18 (2016-09-26) ------------------ - -* Fixed incorrect message for IncompleteRead exception. (Pull #973) - -* Accept ``iPAddress`` subject alternative name fields in TLS certificates. - (Issue #258) - -* Fixed consistency of ``HTTPResponse.closed`` between Python 2 and 3. - (Issue #977) - -* Fixed handling of wildcard certificates when using PyOpenSSL. (Issue #979) - - -1.17 (2016-09-06) ------------------ - -* Accept ``SSLContext`` objects for use in SSL/TLS negotiation. (Issue #835) - -* ConnectionPool debug log now includes scheme, host, and port. (Issue #897) - -* Substantially refactored documentation. (Issue #887) - -* Used URLFetch default timeout on AppEngine, rather than hardcoding our own. - (Issue #858) - -* Normalize the scheme and host in the URL parser (Issue #833) - -* ``HTTPResponse`` contains the last ``Retry`` object, which now also - contains retries history. (Issue #848) - -* Timeout can no longer be set as boolean, and must be greater than zero. - (Pull #924) - -* Removed pyasn1 and ndg-httpsclient from dependencies used for PyOpenSSL. We - now use cryptography and idna, both of which are already dependencies of - PyOpenSSL. (Pull #930) - -* Fixed infinite loop in ``stream`` when amt=None. (Issue #928) - -* Try to use the operating system's certificates when we are using an - ``SSLContext``. (Pull #941) - -* Updated cipher suite list to allow ChaCha20+Poly1305. AES-GCM is preferred to - ChaCha20, but ChaCha20 is then preferred to everything else. (Pull #947) - -* Updated cipher suite list to remove 3DES-based cipher suites. (Pull #958) - -* Removed the cipher suite fallback to allow HIGH ciphers. (Pull #958) - -* Implemented ``length_remaining`` to determine remaining content - to be read. (Pull #949) - -* Implemented ``enforce_content_length`` to enable exceptions when - incomplete data chunks are received. (Pull #949) - -* Dropped connection start, dropped connection reset, redirect, forced retry, - and new HTTPS connection log levels to DEBUG, from INFO. (Pull #967) - - -1.16 (2016-06-11) ------------------ - -* Disable IPv6 DNS when IPv6 connections are not possible. (Issue #840) - -* Provide ``key_fn_by_scheme`` pool keying mechanism that can be - overridden. (Issue #830) - -* Normalize scheme and host to lowercase for pool keys, and include - ``source_address``. (Issue #830) - -* Cleaner exception chain in Python 3 for ``_make_request``. - (Issue #861) - -* Fixed installing ``urllib3[socks]`` extra. (Issue #864) - -* Fixed signature of ``ConnectionPool.close`` so it can actually safely be - called by subclasses. (Issue #873) - -* Retain ``release_conn`` state across retries. (Issues #651, #866) - -* Add customizable ``HTTPConnectionPool.ResponseCls``, which defaults to - ``HTTPResponse`` but can be replaced with a subclass. (Issue #879) - - -1.15.1 (2016-04-11) -------------------- - -* Fix packaging to include backports module. (Issue #841) - - -1.15 (2016-04-06) ------------------ - -* Added Retry(raise_on_status=False). (Issue #720) - -* Always use setuptools, no more distutils fallback. (Issue #785) - -* Dropped support for Python 3.2. (Issue #786) - -* Chunked transfer encoding when requesting with ``chunked=True``. - (Issue #790) - -* Fixed regression with IPv6 port parsing. (Issue #801) - -* Append SNIMissingWarning messages to allow users to specify it in - the PYTHONWARNINGS environment variable. (Issue #816) - -* Handle unicode headers in Py2. (Issue #818) - -* Log certificate when there is a hostname mismatch. (Issue #820) - -* Preserve order of request/response headers. (Issue #821) - - -1.14 (2015-12-29) ------------------ - -* contrib: SOCKS proxy support! (Issue #762) - -* Fixed AppEngine handling of transfer-encoding header and bug - in Timeout defaults checking. (Issue #763) - - -1.13.1 (2015-12-18) -------------------- - -* Fixed regression in IPv6 + SSL for match_hostname. (Issue #761) - - -1.13 (2015-12-14) ------------------ - -* Fixed ``pip install urllib3[secure]`` on modern pip. (Issue #706) - -* pyopenssl: Fixed SSL3_WRITE_PENDING error. (Issue #717) - -* pyopenssl: Support for TLSv1.1 and TLSv1.2. (Issue #696) - -* Close connections more defensively on exception. (Issue #734) - -* Adjusted ``read_chunked`` to handle gzipped, chunk-encoded bodies without - repeatedly flushing the decoder, to function better on Jython. (Issue #743) - -* Accept ``ca_cert_dir`` for SSL-related PoolManager configuration. (Issue #758) - - -1.12 (2015-09-03) ------------------ - -* Rely on ``six`` for importing ``httplib`` to work around - conflicts with other Python 3 shims. (Issue #688) - -* Add support for directories of certificate authorities, as supported by - OpenSSL. (Issue #701) - -* New exception: ``NewConnectionError``, raised when we fail to establish - a new connection, usually ``ECONNREFUSED`` socket error. - - -1.11 (2015-07-21) ------------------ - -* When ``ca_certs`` is given, ``cert_reqs`` defaults to - ``'CERT_REQUIRED'``. (Issue #650) - -* ``pip install urllib3[secure]`` will install Certifi and - PyOpenSSL as dependencies. (Issue #678) - -* Made ``HTTPHeaderDict`` usable as a ``headers`` input value - (Issues #632, #679) - -* Added `urllib3.contrib.appengine `_ - which has an ``AppEngineManager`` for using ``URLFetch`` in a - Google AppEngine environment. (Issue #664) - -* Dev: Added test suite for AppEngine. (Issue #631) - -* Fix performance regression when using PyOpenSSL. (Issue #626) - -* Passing incorrect scheme (e.g. ``foo://``) will raise - ``ValueError`` instead of ``AssertionError`` (backwards - compatible for now, but please migrate). (Issue #640) - -* Fix pools not getting replenished when an error occurs during a - request using ``release_conn=False``. (Issue #644) - -* Fix pool-default headers not applying for url-encoded requests - like GET. (Issue #657) - -* log.warning in Python 3 when headers are skipped due to parsing - errors. (Issue #642) - -* Close and discard connections if an error occurs during read. - (Issue #660) - -* Fix host parsing for IPv6 proxies. (Issue #668) - -* Separate warning type SubjectAltNameWarning, now issued once - per host. (Issue #671) - -* Fix ``httplib.IncompleteRead`` not getting converted to - ``ProtocolError`` when using ``HTTPResponse.stream()`` - (Issue #674) - -1.10.4 (2015-05-03) -------------------- - -* Migrate tests to Tornado 4. (Issue #594) - -* Append default warning configuration rather than overwrite. - (Issue #603) - -* Fix streaming decoding regression. (Issue #595) - -* Fix chunked requests losing state across keep-alive connections. - (Issue #599) - -* Fix hanging when chunked HEAD response has no body. (Issue #605) - - -1.10.3 (2015-04-21) -------------------- - -* Emit ``InsecurePlatformWarning`` when SSLContext object is missing. - (Issue #558) - -* Fix regression of duplicate header keys being discarded. - (Issue #563) - -* ``Response.stream()`` returns a generator for chunked responses. - (Issue #560) - -* Set upper-bound timeout when waiting for a socket in PyOpenSSL. - (Issue #585) - -* Work on platforms without `ssl` module for plain HTTP requests. - (Issue #587) - -* Stop relying on the stdlib's default cipher list. (Issue #588) - - -1.10.2 (2015-02-25) -------------------- - -* Fix file descriptor leakage on retries. (Issue #548) - -* Removed RC4 from default cipher list. (Issue #551) - -* Header performance improvements. (Issue #544) - -* Fix PoolManager not obeying redirect retry settings. (Issue #553) - - -1.10.1 (2015-02-10) -------------------- - -* Pools can be used as context managers. (Issue #545) - -* Don't re-use connections which experienced an SSLError. (Issue #529) - -* Don't fail when gzip decoding an empty stream. (Issue #535) - -* Add sha256 support for fingerprint verification. (Issue #540) - -* Fixed handling of header values containing commas. (Issue #533) - - -1.10 (2014-12-14) ------------------ - -* Disabled SSLv3. (Issue #473) - -* Add ``Url.url`` property to return the composed url string. (Issue #394) - -* Fixed PyOpenSSL + gevent ``WantWriteError``. (Issue #412) - -* ``MaxRetryError.reason`` will always be an exception, not string. - (Issue #481) - -* Fixed SSL-related timeouts not being detected as timeouts. (Issue #492) - -* Py3: Use ``ssl.create_default_context()`` when available. (Issue #473) - -* Emit ``InsecureRequestWarning`` for *every* insecure HTTPS request. - (Issue #496) - -* Emit ``SecurityWarning`` when certificate has no ``subjectAltName``. - (Issue #499) - -* Close and discard sockets which experienced SSL-related errors. - (Issue #501) - -* Handle ``body`` param in ``.request(...)``. (Issue #513) - -* Respect timeout with HTTPS proxy. (Issue #505) - -* PyOpenSSL: Handle ZeroReturnError exception. (Issue #520) - - -1.9.1 (2014-09-13) ------------------- - -* Apply socket arguments before binding. (Issue #427) - -* More careful checks if fp-like object is closed. (Issue #435) - -* Fixed packaging issues of some development-related files not - getting included. (Issue #440) - -* Allow performing *only* fingerprint verification. (Issue #444) - -* Emit ``SecurityWarning`` if system clock is waaay off. (Issue #445) - -* Fixed PyOpenSSL compatibility with PyPy. (Issue #450) - -* Fixed ``BrokenPipeError`` and ``ConnectionError`` handling in Py3. - (Issue #443) - - - -1.9 (2014-07-04) ----------------- - -* Shuffled around development-related files. If you're maintaining a distro - package of urllib3, you may need to tweak things. (Issue #415) - -* Unverified HTTPS requests will trigger a warning on the first request. See - our new `security documentation - `_ for details. - (Issue #426) - -* New retry logic and ``urllib3.util.retry.Retry`` configuration object. - (Issue #326) - -* All raised exceptions should now wrapped in a - ``urllib3.exceptions.HTTPException``-extending exception. (Issue #326) - -* All errors during a retry-enabled request should be wrapped in - ``urllib3.exceptions.MaxRetryError``, including timeout-related exceptions - which were previously exempt. Underlying error is accessible from the - ``.reason`` property. (Issue #326) - -* ``urllib3.exceptions.ConnectionError`` renamed to - ``urllib3.exceptions.ProtocolError``. (Issue #326) - -* Errors during response read (such as IncompleteRead) are now wrapped in - ``urllib3.exceptions.ProtocolError``. (Issue #418) - -* Requesting an empty host will raise ``urllib3.exceptions.LocationValueError``. - (Issue #417) - -* Catch read timeouts over SSL connections as - ``urllib3.exceptions.ReadTimeoutError``. (Issue #419) - -* Apply socket arguments before connecting. (Issue #427) - - -1.8.3 (2014-06-23) ------------------- - -* Fix TLS verification when using a proxy in Python 3.4.1. (Issue #385) - -* Add ``disable_cache`` option to ``urllib3.util.make_headers``. (Issue #393) - -* Wrap ``socket.timeout`` exception with - ``urllib3.exceptions.ReadTimeoutError``. (Issue #399) - -* Fixed proxy-related bug where connections were being reused incorrectly. - (Issues #366, #369) - -* Added ``socket_options`` keyword parameter which allows to define - ``setsockopt`` configuration of new sockets. (Issue #397) - -* Removed ``HTTPConnection.tcp_nodelay`` in favor of - ``HTTPConnection.default_socket_options``. (Issue #397) - -* Fixed ``TypeError`` bug in Python 2.6.4. (Issue #411) - - -1.8.2 (2014-04-17) ------------------- - -* Fix ``urllib3.util`` not being included in the package. - - -1.8.1 (2014-04-17) ------------------- - -* Fix AppEngine bug of HTTPS requests going out as HTTP. (Issue #356) - -* Don't install ``dummyserver`` into ``site-packages`` as it's only needed - for the test suite. (Issue #362) - -* Added support for specifying ``source_address``. (Issue #352) - - -1.8 (2014-03-04) ----------------- - -* Improved url parsing in ``urllib3.util.parse_url`` (properly parse '@' in - username, and blank ports like 'hostname:'). - -* New ``urllib3.connection`` module which contains all the HTTPConnection - objects. - -* Several ``urllib3.util.Timeout``-related fixes. Also changed constructor - signature to a more sensible order. [Backwards incompatible] - (Issues #252, #262, #263) - -* Use ``backports.ssl_match_hostname`` if it's installed. (Issue #274) - -* Added ``.tell()`` method to ``urllib3.response.HTTPResponse`` which - returns the number of bytes read so far. (Issue #277) - -* Support for platforms without threading. (Issue #289) - -* Expand default-port comparison in ``HTTPConnectionPool.is_same_host`` - to allow a pool with no specified port to be considered equal to to an - HTTP/HTTPS url with port 80/443 explicitly provided. (Issue #305) - -* Improved default SSL/TLS settings to avoid vulnerabilities. - (Issue #309) - -* Fixed ``urllib3.poolmanager.ProxyManager`` not retrying on connect errors. - (Issue #310) - -* Disable Nagle's Algorithm on the socket for non-proxies. A subset of requests - will send the entire HTTP request ~200 milliseconds faster; however, some of - the resulting TCP packets will be smaller. (Issue #254) - -* Increased maximum number of SubjectAltNames in ``urllib3.contrib.pyopenssl`` - from the default 64 to 1024 in a single certificate. (Issue #318) - -* Headers are now passed and stored as a custom - ``urllib3.collections_.HTTPHeaderDict`` object rather than a plain ``dict``. - (Issue #329, #333) - -* Headers no longer lose their case on Python 3. (Issue #236) - -* ``urllib3.contrib.pyopenssl`` now uses the operating system's default CA - certificates on inject. (Issue #332) - -* Requests with ``retries=False`` will immediately raise any exceptions without - wrapping them in ``MaxRetryError``. (Issue #348) - -* Fixed open socket leak with SSL-related failures. (Issue #344, #348) - - -1.7.1 (2013-09-25) ------------------- - -* Added granular timeout support with new ``urllib3.util.Timeout`` class. - (Issue #231) - -* Fixed Python 3.4 support. (Issue #238) - - -1.7 (2013-08-14) ----------------- - -* More exceptions are now pickle-able, with tests. (Issue #174) - -* Fixed redirecting with relative URLs in Location header. (Issue #178) - -* Support for relative urls in ``Location: ...`` header. (Issue #179) - -* ``urllib3.response.HTTPResponse`` now inherits from ``io.IOBase`` for bonus - file-like functionality. (Issue #187) - -* Passing ``assert_hostname=False`` when creating a HTTPSConnectionPool will - skip hostname verification for SSL connections. (Issue #194) - -* New method ``urllib3.response.HTTPResponse.stream(...)`` which acts as a - generator wrapped around ``.read(...)``. (Issue #198) - -* IPv6 url parsing enforces brackets around the hostname. (Issue #199) - -* Fixed thread race condition in - ``urllib3.poolmanager.PoolManager.connection_from_host(...)`` (Issue #204) - -* ``ProxyManager`` requests now include non-default port in ``Host: ...`` - header. (Issue #217) - -* Added HTTPS proxy support in ``ProxyManager``. (Issue #170 #139) - -* New ``RequestField`` object can be passed to the ``fields=...`` param which - can specify headers. (Issue #220) - -* Raise ``urllib3.exceptions.ProxyError`` when connecting to proxy fails. - (Issue #221) - -* Use international headers when posting file names. (Issue #119) - -* Improved IPv6 support. (Issue #203) - - -1.6 (2013-04-25) ----------------- - -* Contrib: Optional SNI support for Py2 using PyOpenSSL. (Issue #156) - -* ``ProxyManager`` automatically adds ``Host: ...`` header if not given. - -* Improved SSL-related code. ``cert_req`` now optionally takes a string like - "REQUIRED" or "NONE". Same with ``ssl_version`` takes strings like "SSLv23" - The string values reflect the suffix of the respective constant variable. - (Issue #130) - -* Vendored ``socksipy`` now based on Anorov's fork which handles unexpectedly - closed proxy connections and larger read buffers. (Issue #135) - -* Ensure the connection is closed if no data is received, fixes connection leak - on some platforms. (Issue #133) - -* Added SNI support for SSL/TLS connections on Py32+. (Issue #89) - -* Tests fixed to be compatible with Py26 again. (Issue #125) - -* Added ability to choose SSL version by passing an ``ssl.PROTOCOL_*`` constant - to the ``ssl_version`` parameter of ``HTTPSConnectionPool``. (Issue #109) - -* Allow an explicit content type to be specified when encoding file fields. - (Issue #126) - -* Exceptions are now pickleable, with tests. (Issue #101) - -* Fixed default headers not getting passed in some cases. (Issue #99) - -* Treat "content-encoding" header value as case-insensitive, per RFC 2616 - Section 3.5. (Issue #110) - -* "Connection Refused" SocketErrors will get retried rather than raised. - (Issue #92) - -* Updated vendored ``six``, no longer overrides the global ``six`` module - namespace. (Issue #113) - -* ``urllib3.exceptions.MaxRetryError`` contains a ``reason`` property holding - the exception that prompted the final retry. If ``reason is None`` then it - was due to a redirect. (Issue #92, #114) - -* Fixed ``PoolManager.urlopen()`` from not redirecting more than once. - (Issue #149) - -* Don't assume ``Content-Type: text/plain`` for multi-part encoding parameters - that are not files. (Issue #111) - -* Pass `strict` param down to ``httplib.HTTPConnection``. (Issue #122) - -* Added mechanism to verify SSL certificates by fingerprint (md5, sha1) or - against an arbitrary hostname (when connecting by IP or for misconfigured - servers). (Issue #140) - -* Streaming decompression support. (Issue #159) - - -1.5 (2012-08-02) ----------------- - -* Added ``urllib3.add_stderr_logger()`` for quickly enabling STDERR debug - logging in urllib3. - -* Native full URL parsing (including auth, path, query, fragment) available in - ``urllib3.util.parse_url(url)``. - -* Built-in redirect will switch method to 'GET' if status code is 303. - (Issue #11) - -* ``urllib3.PoolManager`` strips the scheme and host before sending the request - uri. (Issue #8) - -* New ``urllib3.exceptions.DecodeError`` exception for when automatic decoding, - based on the Content-Type header, fails. - -* Fixed bug with pool depletion and leaking connections (Issue #76). Added - explicit connection closing on pool eviction. Added - ``urllib3.PoolManager.clear()``. - -* 99% -> 100% unit test coverage. - - -1.4 (2012-06-16) ----------------- - -* Minor AppEngine-related fixes. - -* Switched from ``mimetools.choose_boundary`` to ``uuid.uuid4()``. - -* Improved url parsing. (Issue #73) - -* IPv6 url support. (Issue #72) - - -1.3 (2012-03-25) ----------------- - -* Removed pre-1.0 deprecated API. - -* Refactored helpers into a ``urllib3.util`` submodule. - -* Fixed multipart encoding to support list-of-tuples for keys with multiple - values. (Issue #48) - -* Fixed multiple Set-Cookie headers in response not getting merged properly in - Python 3. (Issue #53) - -* AppEngine support with Py27. (Issue #61) - -* Minor ``encode_multipart_formdata`` fixes related to Python 3 strings vs - bytes. - - -1.2.2 (2012-02-06) ------------------- - -* Fixed packaging bug of not shipping ``test-requirements.txt``. (Issue #47) - - -1.2.1 (2012-02-05) ------------------- - -* Fixed another bug related to when ``ssl`` module is not available. (Issue #41) - -* Location parsing errors now raise ``urllib3.exceptions.LocationParseError`` - which inherits from ``ValueError``. - - -1.2 (2012-01-29) ----------------- - -* Added Python 3 support (tested on 3.2.2) - -* Dropped Python 2.5 support (tested on 2.6.7, 2.7.2) - -* Use ``select.poll`` instead of ``select.select`` for platforms that support - it. - -* Use ``Queue.LifoQueue`` instead of ``Queue.Queue`` for more aggressive - connection reusing. Configurable by overriding ``ConnectionPool.QueueCls``. - -* Fixed ``ImportError`` during install when ``ssl`` module is not available. - (Issue #41) - -* Fixed ``PoolManager`` redirects between schemes (such as HTTP -> HTTPS) not - completing properly. (Issue #28, uncovered by Issue #10 in v1.1) - -* Ported ``dummyserver`` to use ``tornado`` instead of ``webob`` + - ``eventlet``. Removed extraneous unsupported dummyserver testing backends. - Added socket-level tests. - -* More tests. Achievement Unlocked: 99% Coverage. - - -1.1 (2012-01-07) ----------------- - -* Refactored ``dummyserver`` to its own root namespace module (used for - testing). - -* Added hostname verification for ``VerifiedHTTPSConnection`` by vendoring in - Py32's ``ssl_match_hostname``. (Issue #25) - -* Fixed cross-host HTTP redirects when using ``PoolManager``. (Issue #10) - -* Fixed ``decode_content`` being ignored when set through ``urlopen``. (Issue - #27) - -* Fixed timeout-related bugs. (Issues #17, #23) - - -1.0.2 (2011-11-04) ------------------- - -* Fixed typo in ``VerifiedHTTPSConnection`` which would only present as a bug if - you're using the object manually. (Thanks pyos) - -* Made RecentlyUsedContainer (and consequently PoolManager) more thread-safe by - wrapping the access log in a mutex. (Thanks @christer) - -* Made RecentlyUsedContainer more dict-like (corrected ``__delitem__`` and - ``__getitem__`` behaviour), with tests. Shouldn't affect core urllib3 code. - - -1.0.1 (2011-10-10) ------------------- - -* Fixed a bug where the same connection would get returned into the pool twice, - causing extraneous "HttpConnectionPool is full" log warnings. - - -1.0 (2011-10-08) ----------------- - -* Added ``PoolManager`` with LRU expiration of connections (tested and - documented). -* Added ``ProxyManager`` (needs tests, docs, and confirmation that it works - with HTTPS proxies). -* Added optional partial-read support for responses when - ``preload_content=False``. You can now make requests and just read the headers - without loading the content. -* Made response decoding optional (default on, same as before). -* Added optional explicit boundary string for ``encode_multipart_formdata``. -* Convenience request methods are now inherited from ``RequestMethods``. Old - helpers like ``get_url`` and ``post_url`` should be abandoned in favour of - the new ``request(method, url, ...)``. -* Refactored code to be even more decoupled, reusable, and extendable. -* License header added to ``.py`` files. -* Embiggened the documentation: Lots of Sphinx-friendly docstrings in the code - and docs in ``docs/`` and on https://urllib3.readthedocs.io/. -* Embettered all the things! -* Started writing this file. - - -0.4.1 (2011-07-17) ------------------- - -* Minor bug fixes, code cleanup. - - -0.4 (2011-03-01) ----------------- - -* Better unicode support. -* Added ``VerifiedHTTPSConnection``. -* Added ``NTLMConnectionPool`` in contrib. -* Minor improvements. - - -0.3.1 (2010-07-13) ------------------- - -* Added ``assert_host_name`` optional parameter. Now compatible with proxies. - - -0.3 (2009-12-10) ----------------- - -* Added HTTPS support. -* Minor bug fixes. -* Refactored, broken backwards compatibility with 0.2. -* API to be treated as stable from this version forward. - - -0.2 (2008-11-17) ----------------- - -* Added unit tests. -* Bug fixes. - - -0.1 (2008-11-16) ----------------- - -* First release. diff --git a/src/urllib3.egg-info/SOURCES.txt b/src/urllib3.egg-info/SOURCES.txt deleted file mode 100644 index d924e26..0000000 --- a/src/urllib3.egg-info/SOURCES.txt +++ /dev/null @@ -1,136 +0,0 @@ -CHANGES.rst -LICENSE.txt -MANIFEST.in -README.rst -dev-requirements.txt -setup.cfg -setup.py -docs/Makefile -docs/advanced-usage.rst -docs/conf.py -docs/contributing.rst -docs/index.rst -docs/make.bat -docs/requirements.txt -docs/sponsors.rst -docs/user-guide.rst -docs/v2-roadmap.rst -docs/_static/banner.svg -docs/_static/dark-logo.svg -docs/images/demo-button.png -docs/images/favicon.png -docs/images/learn-more-button.png -docs/images/logo.png -docs/images/logo.svg -docs/reference/index.rst -docs/reference/urllib3.connection.rst -docs/reference/urllib3.connectionpool.rst -docs/reference/urllib3.exceptions.rst -docs/reference/urllib3.fields.rst -docs/reference/urllib3.poolmanager.rst -docs/reference/urllib3.request.rst -docs/reference/urllib3.response.rst -docs/reference/urllib3.util.rst -docs/reference/contrib/appengine.rst -docs/reference/contrib/index.rst -docs/reference/contrib/ntlmpool.rst -docs/reference/contrib/pyopenssl.rst -docs/reference/contrib/securetransport.rst -docs/reference/contrib/socks.rst -dummyserver/__init__.py -dummyserver/handlers.py -dummyserver/proxy.py -dummyserver/server.py -dummyserver/testcase.py -dummyserver/certs/README.rst -dummyserver/certs/cacert.key -dummyserver/certs/cacert.pem -dummyserver/certs/server.crt -dummyserver/certs/server.key -src/urllib3/__init__.py -src/urllib3/_collections.py -src/urllib3/_version.py -src/urllib3/connection.py -src/urllib3/connectionpool.py -src/urllib3/exceptions.py -src/urllib3/fields.py -src/urllib3/filepost.py -src/urllib3/poolmanager.py -src/urllib3/request.py -src/urllib3/response.py -src/urllib3.egg-info/PKG-INFO -src/urllib3.egg-info/SOURCES.txt -src/urllib3.egg-info/dependency_links.txt -src/urllib3.egg-info/requires.txt -src/urllib3.egg-info/top_level.txt -src/urllib3/contrib/__init__.py -src/urllib3/contrib/_appengine_environ.py -src/urllib3/contrib/appengine.py -src/urllib3/contrib/ntlmpool.py -src/urllib3/contrib/pyopenssl.py -src/urllib3/contrib/securetransport.py -src/urllib3/contrib/socks.py -src/urllib3/contrib/_securetransport/__init__.py -src/urllib3/contrib/_securetransport/bindings.py -src/urllib3/contrib/_securetransport/low_level.py -src/urllib3/packages/__init__.py -src/urllib3/packages/six.py -src/urllib3/packages/backports/__init__.py -src/urllib3/packages/backports/makefile.py -src/urllib3/packages/backports/weakref_finalize.py -src/urllib3/util/__init__.py -src/urllib3/util/connection.py -src/urllib3/util/proxy.py -src/urllib3/util/queue.py -src/urllib3/util/request.py -src/urllib3/util/response.py -src/urllib3/util/retry.py -src/urllib3/util/ssl_.py -src/urllib3/util/ssl_match_hostname.py -src/urllib3/util/ssltransport.py -src/urllib3/util/timeout.py -src/urllib3/util/url.py -src/urllib3/util/wait.py -test/__init__.py -test/benchmark.py -test/conftest.py -test/port_helpers.py -test/socketpair_helper.py -test/test_collections.py -test/test_compatibility.py -test/test_connection.py -test/test_connectionpool.py -test/test_exceptions.py -test/test_fields.py -test/test_filepost.py -test/test_no_ssl.py -test/test_poolmanager.py -test/test_proxymanager.py -test/test_queue_monkeypatch.py -test/test_request.py -test/test_response.py -test/test_retry.py -test/test_retry_deprecated.py -test/test_ssl.py -test/test_ssltransport.py -test/test_util.py -test/test_wait.py -test/tz_stub.py -test/appengine/__init__.py -test/appengine/conftest.py -test/appengine/test_gae_manager.py -test/appengine/test_urlfetch.py -test/contrib/__init__.py -test/contrib/duplicate_san.pem -test/contrib/test_pyopenssl.py -test/contrib/test_pyopenssl_dependencies.py -test/contrib/test_securetransport.py -test/contrib/test_socks.py -test/with_dummyserver/__init__.py -test/with_dummyserver/test_chunked_transfer.py -test/with_dummyserver/test_connectionpool.py -test/with_dummyserver/test_https.py -test/with_dummyserver/test_no_ssl.py -test/with_dummyserver/test_poolmanager.py -test/with_dummyserver/test_proxy_poolmanager.py -test/with_dummyserver/test_socketlevel.py \ No newline at end of file diff --git a/src/urllib3.egg-info/dependency_links.txt b/src/urllib3.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/src/urllib3.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/urllib3.egg-info/requires.txt b/src/urllib3.egg-info/requires.txt deleted file mode 100644 index 1a69380..0000000 --- a/src/urllib3.egg-info/requires.txt +++ /dev/null @@ -1,27 +0,0 @@ - -[brotli] - -[brotli:(os_name != "nt" or python_version >= "3") and platform_python_implementation != "CPython"] -brotlicffi>=0.8.0 - -[brotli:os_name != "nt" and python_version < "3" and platform_python_implementation == "CPython"] -brotli==1.0.9 - -[brotli:os_name == "nt" and python_version < "3"] -brotlipy>=0.6.0 - -[brotli:python_version >= "3" and platform_python_implementation == "CPython"] -brotli>=1.0.9 - -[secure] -pyOpenSSL>=0.14 -cryptography>=1.3.4 -idna>=2.0.0 -certifi -urllib3-secure-extra - -[secure:python_version == "2.7"] -ipaddress - -[socks] -PySocks!=1.5.7,<2.0,>=1.5.6 diff --git a/src/urllib3.egg-info/top_level.txt b/src/urllib3.egg-info/top_level.txt deleted file mode 100644 index a42590b..0000000 --- a/src/urllib3.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -urllib3 diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index c6fa382..3fe782c 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -1,40 +1,49 @@ """ Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more """ -from __future__ import absolute_import + +from __future__ import annotations # Set default logging handler to avoid "No handler found" warnings. import logging +import sys +import typing import warnings from logging import NullHandler from . import exceptions +from ._base_connection import _TYPE_BODY +from ._collections import HTTPHeaderDict from ._version import __version__ from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url -from .filepost import encode_multipart_formdata +from .filepost import _TYPE_FIELDS, encode_multipart_formdata from .poolmanager import PoolManager, ProxyManager, proxy_from_url -from .response import HTTPResponse +from .response import BaseHTTPResponse, HTTPResponse from .util.request import make_headers from .util.retry import Retry from .util.timeout import Timeout -from .util.url import get_host -# === NOTE TO REPACKAGERS AND VENDORS === -# Please delete this block, this logic is only -# for urllib3 being distributed via PyPI. -# See: https://github.com/urllib3/urllib3/issues/2680 +# Ensure that Python is compiled with OpenSSL 1.1.1+ +# If the 'ssl' module isn't available at all that's +# fine, we only care if the module is available. try: - import urllib3_secure_extra # type: ignore # noqa: F401 + import ssl except ImportError: pass else: - warnings.warn( - "'urllib3[secure]' extra is deprecated and will be removed " - "in a future release of urllib3 2.x. Read more in this issue: " - "https://github.com/urllib3/urllib3/issues/2680", - category=DeprecationWarning, - stacklevel=2, - ) + if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): # Defensive: + warnings.warn( + "urllib3 v2 only supports OpenSSL 1.1.1+, currently " + f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. " + "See: https://github.com/urllib3/urllib3/issues/3020", + exceptions.NotOpenSSLWarning, + ) + elif ssl.OPENSSL_VERSION_INFO < (1, 1, 1): # Defensive: + raise ImportError( + "urllib3 v2 only supports OpenSSL 1.1.1+, currently " + f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. " + "See: https://github.com/urllib3/urllib3/issues/2168" + ) __author__ = "Andrey Petrov (andrey.petrov@shazow.net)" __license__ = "MIT" @@ -42,6 +51,7 @@ __all__ = ( "HTTPConnectionPool", + "HTTPHeaderDict", "HTTPSConnectionPool", "PoolManager", "ProxyManager", @@ -52,15 +62,18 @@ "connection_from_url", "disable_warnings", "encode_multipart_formdata", - "get_host", "make_headers", "proxy_from_url", + "request", + "BaseHTTPResponse", ) logging.getLogger(__name__).addHandler(NullHandler()) -def add_stderr_logger(level=logging.DEBUG): +def add_stderr_logger( + level: int = logging.DEBUG, +) -> logging.StreamHandler[typing.TextIO]: """ Helper for quickly adding a StreamHandler to the logger. Useful for debugging. @@ -87,16 +100,112 @@ def add_stderr_logger(level=logging.DEBUG): # mechanisms to silence them. # SecurityWarning's always go off by default. warnings.simplefilter("always", exceptions.SecurityWarning, append=True) -# SubjectAltNameWarning's should go off once per host -warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True) -# SNIMissingWarnings should go off only once. -warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True) -def disable_warnings(category=exceptions.HTTPWarning): +def disable_warnings(category: type[Warning] = exceptions.HTTPWarning) -> None: """ Helper for quickly disabling all urllib3 warnings. """ warnings.simplefilter("ignore", category) + + +_DEFAULT_POOL = PoolManager() + + +def request( + method: str, + url: str, + *, + body: _TYPE_BODY | None = None, + fields: _TYPE_FIELDS | None = None, + headers: typing.Mapping[str, str] | None = None, + preload_content: bool | None = True, + decode_content: bool | None = True, + redirect: bool | None = True, + retries: Retry | bool | int | None = None, + timeout: Timeout | float | int | None = 3, + json: typing.Any | None = None, +) -> BaseHTTPResponse: + """ + A convenience, top-level request method. It uses a module-global ``PoolManager`` instance. + Therefore, its side effects could be shared across dependencies relying on it. + To avoid side effects create a new ``PoolManager`` instance and use it instead. + The method does not accept low-level ``**urlopen_kw`` keyword arguments. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param body: + Data to send in the request body, either :class:`str`, :class:`bytes`, + an iterable of :class:`str`/:class:`bytes`, or a file-like object. + + :param fields: + Data to encode and send in the request body. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. + + :param bool preload_content: + If True, the response's body will be preloaded into memory. + + :param bool decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + + :param redirect: + If True, automatically handle redirects (status codes 301, 302, + 303, 307, 308). Each redirect counts as a retry. Disabling retries + will disable redirect, too. + + :param retries: + Configure the number of retries to allow before raising a + :class:`~urllib3.exceptions.MaxRetryError` exception. + + If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a + :class:`~urllib3.util.retry.Retry` object for fine-grained control + over different types of retries. + Pass an integer number to retry connection errors that many times, + but no other types of errors. Pass zero to never retry. + + If ``False``, then retries are disabled and any exception is raised + immediately. Also, instead of raising a MaxRetryError on redirects, + the redirect response will be returned. + + :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. + + :param timeout: + If specified, overrides the default timeout for this one + request. It may be a float (in seconds) or an instance of + :class:`urllib3.util.Timeout`. + + :param json: + Data to encode and send as JSON with UTF-encoded in the request body. + The ``"Content-Type"`` header will be set to ``"application/json"`` + unless specified otherwise. + """ + + return _DEFAULT_POOL.request( + method, + url, + body=body, + fields=fields, + headers=headers, + preload_content=preload_content, + decode_content=decode_content, + redirect=redirect, + retries=retries, + timeout=timeout, + json=json, + ) + + +if sys.platform == "emscripten": + from .contrib.emscripten import inject_into_urllib3 # noqa: 401 + + inject_into_urllib3() diff --git a/src/urllib3/_base_connection.py b/src/urllib3/_base_connection.py new file mode 100644 index 0000000..dc0f318 --- /dev/null +++ b/src/urllib3/_base_connection.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import typing + +from .util.connection import _TYPE_SOCKET_OPTIONS +from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT +from .util.url import Url + +_TYPE_BODY = typing.Union[bytes, typing.IO[typing.Any], typing.Iterable[bytes], str] + + +class ProxyConfig(typing.NamedTuple): + ssl_context: ssl.SSLContext | None + use_forwarding_for_https: bool + assert_hostname: None | str | typing.Literal[False] + assert_fingerprint: str | None + + +class _ResponseOptions(typing.NamedTuple): + # TODO: Remove this in favor of a better + # HTTP request/response lifecycle tracking. + request_method: str + request_url: str + preload_content: bool + decode_content: bool + enforce_content_length: bool + + +if typing.TYPE_CHECKING: + import ssl + from typing import Protocol + + from .response import BaseHTTPResponse + + class BaseHTTPConnection(Protocol): + default_port: typing.ClassVar[int] + default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS] + + host: str + port: int + timeout: None | ( + float + ) # Instance doesn't store _DEFAULT_TIMEOUT, must be resolved. + blocksize: int + source_address: tuple[str, int] | None + socket_options: _TYPE_SOCKET_OPTIONS | None + + proxy: Url | None + proxy_config: ProxyConfig | None + + is_verified: bool + proxy_is_verified: bool | None + + def __init__( + self, + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 8192, + socket_options: _TYPE_SOCKET_OPTIONS | None = ..., + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + ) -> None: ... + + def set_tunnel( + self, + host: str, + port: int | None = None, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: ... + + def connect(self) -> None: ... + + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + # We know *at least* botocore is depending on the order of the + # first 3 parameters so to be safe we only mark the later ones + # as keyword-only to ensure we have space to extend. + *, + chunked: bool = False, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> None: ... + + def getresponse(self) -> BaseHTTPResponse: ... + + def close(self) -> None: ... + + @property + def is_closed(self) -> bool: + """Whether the connection either is brand new or has been previously closed. + If this property is True then both ``is_connected`` and ``has_connected_to_proxy`` + properties must be False. + """ + + @property + def is_connected(self) -> bool: + """Whether the connection is actively connected to any origin (proxy or target)""" + + @property + def has_connected_to_proxy(self) -> bool: + """Whether the connection has successfully connected to its proxy. + This returns False if no proxy is in use. Used to determine whether + errors are coming from the proxy layer or from tunnelling to the target origin. + """ + + class BaseHTTPSConnection(BaseHTTPConnection, Protocol): + default_port: typing.ClassVar[int] + default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS] + + # Certificate verification methods + cert_reqs: int | str | None + assert_hostname: None | str | typing.Literal[False] + assert_fingerprint: str | None + ssl_context: ssl.SSLContext | None + + # Trusted CAs + ca_certs: str | None + ca_cert_dir: str | None + ca_cert_data: None | str | bytes + + # TLS version + ssl_minimum_version: int | None + ssl_maximum_version: int | None + ssl_version: int | str | None # Deprecated + + # Client certificates + cert_file: str | None + key_file: str | None + key_password: str | None + + def __init__( + self, + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: _TYPE_SOCKET_OPTIONS | None = ..., + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + cert_reqs: int | str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + server_hostname: str | None = None, + ssl_context: ssl.SSLContext | None = None, + ca_certs: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, + ssl_version: int | str | None = None, # Deprecated + cert_file: str | None = None, + key_file: str | None = None, + key_password: str | None = None, + ) -> None: ... diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py index bceb845..1b6c136 100644 --- a/src/urllib3/_collections.py +++ b/src/urllib3/_collections.py @@ -1,34 +1,66 @@ -from __future__ import absolute_import - -try: - from collections.abc import Mapping, MutableMapping -except ImportError: - from collections import Mapping, MutableMapping -try: - from threading import RLock -except ImportError: # Platform-specific: No threads available - - class RLock: - def __enter__(self): - pass +from __future__ import annotations - def __exit__(self, exc_type, exc_value, traceback): - pass +import typing +from collections import OrderedDict +from enum import Enum, auto +from threading import RLock +if typing.TYPE_CHECKING: + # We can only import Protocol if TYPE_CHECKING because it's a development + # dependency, and is not available at runtime. + from typing import Protocol -from collections import OrderedDict + from typing_extensions import Self -from .exceptions import InvalidHeader -from .packages import six -from .packages.six import iterkeys, itervalues + class HasGettableStringKeys(Protocol): + def keys(self) -> typing.Iterator[str]: ... -__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"] + def __getitem__(self, key: str) -> str: ... -_Null = object() +__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"] -class RecentlyUsedContainer(MutableMapping): +# Key type +_KT = typing.TypeVar("_KT") +# Value type +_VT = typing.TypeVar("_VT") +# Default type +_DT = typing.TypeVar("_DT") + +ValidHTTPHeaderSource = typing.Union[ + "HTTPHeaderDict", + typing.Mapping[str, str], + typing.Iterable[tuple[str, str]], + "HasGettableStringKeys", +] + + +class _Sentinel(Enum): + not_passed = auto() + + +def ensure_can_construct_http_header_dict( + potential: object, +) -> ValidHTTPHeaderSource | None: + if isinstance(potential, HTTPHeaderDict): + return potential + elif isinstance(potential, typing.Mapping): + # Full runtime checking of the contents of a Mapping is expensive, so for the + # purposes of typechecking, we assume that any Mapping is the right shape. + return typing.cast(typing.Mapping[str, str], potential) + elif isinstance(potential, typing.Iterable): + # Similarly to Mapping, full runtime checking of the contents of an Iterable is + # expensive, so for the purposes of typechecking, we assume that any Iterable + # is the right shape. + return typing.cast(typing.Iterable[tuple[str, str]], potential) + elif hasattr(potential, "keys") and hasattr(potential, "__getitem__"): + return typing.cast("HasGettableStringKeys", potential) + else: + return None + + +class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT, _VT]): """ Provides a thread-safe dict-like container which maintains up to ``maxsize`` keys while throwing away the least-recently-used keys beyond @@ -42,69 +74,134 @@ class RecentlyUsedContainer(MutableMapping): ``dispose_func(value)`` is called. Callback which will get called """ - ContainerCls = OrderedDict - - def __init__(self, maxsize=10, dispose_func=None): + _container: typing.OrderedDict[_KT, _VT] + _maxsize: int + dispose_func: typing.Callable[[_VT], None] | None + lock: RLock + + def __init__( + self, + maxsize: int = 10, + dispose_func: typing.Callable[[_VT], None] | None = None, + ) -> None: + super().__init__() self._maxsize = maxsize self.dispose_func = dispose_func - - self._container = self.ContainerCls() + self._container = OrderedDict() self.lock = RLock() - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> _VT: # Re-insert the item, moving it to the end of the eviction line. with self.lock: item = self._container.pop(key) self._container[key] = item return item - def __setitem__(self, key, value): - evicted_value = _Null + def __setitem__(self, key: _KT, value: _VT) -> None: + evicted_item = None with self.lock: # Possibly evict the existing value of 'key' - evicted_value = self._container.get(key, _Null) - self._container[key] = value - - # If we didn't evict an existing value, we might have to evict the - # least recently used item from the beginning of the container. - if len(self._container) > self._maxsize: - _key, evicted_value = self._container.popitem(last=False) - - if self.dispose_func and evicted_value is not _Null: + try: + # If the key exists, we'll overwrite it, which won't change the + # size of the pool. Because accessing a key should move it to + # the end of the eviction line, we pop it out first. + evicted_item = key, self._container.pop(key) + self._container[key] = value + except KeyError: + # When the key does not exist, we insert the value first so that + # evicting works in all cases, including when self._maxsize is 0 + self._container[key] = value + if len(self._container) > self._maxsize: + # If we didn't evict an existing value, and we've hit our maximum + # size, then we have to evict the least recently used item from + # the beginning of the container. + evicted_item = self._container.popitem(last=False) + + # After releasing the lock on the pool, dispose of any evicted value. + if evicted_item is not None and self.dispose_func: + _, evicted_value = evicted_item self.dispose_func(evicted_value) - def __delitem__(self, key): + def __delitem__(self, key: _KT) -> None: with self.lock: value = self._container.pop(key) if self.dispose_func: self.dispose_func(value) - def __len__(self): + def __len__(self) -> int: with self.lock: return len(self._container) - def __iter__(self): + def __iter__(self) -> typing.NoReturn: raise NotImplementedError( "Iteration over this class is unlikely to be threadsafe." ) - def clear(self): + def clear(self) -> None: with self.lock: # Copy pointers to all values, then wipe the mapping - values = list(itervalues(self._container)) + values = list(self._container.values()) self._container.clear() if self.dispose_func: for value in values: self.dispose_func(value) - def keys(self): + def keys(self) -> set[_KT]: # type: ignore[override] with self.lock: - return list(iterkeys(self._container)) + return set(self._container.keys()) + +class HTTPHeaderDictItemView(set[tuple[str, str]]): + """ + HTTPHeaderDict is unusual for a Mapping[str, str] in that it has two modes of + address. + + If we directly try to get an item with a particular name, we will get a string + back that is the concatenated version of all the values: + + >>> d['X-Header-Name'] + 'Value1, Value2, Value3' + + However, if we iterate over an HTTPHeaderDict's items, we will optionally combine + these values based on whether combine=True was called when building up the dictionary + + >>> d = HTTPHeaderDict({"A": "1", "B": "foo"}) + >>> d.add("A", "2", combine=True) + >>> d.add("B", "bar") + >>> list(d.items()) + [ + ('A', '1, 2'), + ('B', 'foo'), + ('B', 'bar'), + ] + + This class conforms to the interface required by the MutableMapping ABC while + also giving us the nonstandard iteration behavior we want; items with duplicate + keys, ordered by time of first insertion. + """ + + _headers: HTTPHeaderDict + + def __init__(self, headers: HTTPHeaderDict) -> None: + self._headers = headers + + def __len__(self) -> int: + return len(list(self._headers.iteritems())) + + def __iter__(self) -> typing.Iterator[tuple[str, str]]: + return self._headers.iteritems() -class HTTPHeaderDict(MutableMapping): + def __contains__(self, item: object) -> bool: + if isinstance(item, tuple) and len(item) == 2: + passed_key, passed_val = item + if isinstance(passed_key, str) and isinstance(passed_val, str): + return self._headers._has_value_for_header(passed_key, passed_val) + return False + + +class HTTPHeaderDict(typing.MutableMapping[str, str]): """ :param headers: An iterable of field-value pairs. Must not contain multiple field names @@ -138,9 +235,11 @@ class HTTPHeaderDict(MutableMapping): '7' """ - def __init__(self, headers=None, **kwargs): - super(HTTPHeaderDict, self).__init__() - self._container = OrderedDict() + _container: typing.MutableMapping[str, list[str]] + + def __init__(self, headers: ValidHTTPHeaderSource | None = None, **kwargs: str): + super().__init__() + self._container = {} # 'dict' is insert-ordered if headers is not None: if isinstance(headers, HTTPHeaderDict): self._copy_from(headers) @@ -149,126 +248,148 @@ def __init__(self, headers=None, **kwargs): if kwargs: self.extend(kwargs) - def __setitem__(self, key, val): + def __setitem__(self, key: str, val: str) -> None: + # avoid a bytes/str comparison by decoding before httplib + if isinstance(key, bytes): + key = key.decode("latin-1") self._container[key.lower()] = [key, val] - return self._container[key.lower()] - def __getitem__(self, key): + def __getitem__(self, key: str) -> str: val = self._container[key.lower()] return ", ".join(val[1:]) - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._container[key.lower()] - def __contains__(self, key): - return key.lower() in self._container + def __contains__(self, key: object) -> bool: + if isinstance(key, str): + return key.lower() in self._container + return False - def __eq__(self, other): - if not isinstance(other, Mapping) and not hasattr(other, "keys"): - return False - if not isinstance(other, type(self)): - other = type(self)(other) - return dict((k.lower(), v) for k, v in self.itermerged()) == dict( - (k.lower(), v) for k, v in other.itermerged() - ) + def setdefault(self, key: str, default: str = "") -> str: + return super().setdefault(key, default) - def __ne__(self, other): - return not self.__eq__(other) + def __eq__(self, other: object) -> bool: + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return False + else: + other_as_http_header_dict = type(self)(maybe_constructable) - if six.PY2: # Python 2 - iterkeys = MutableMapping.iterkeys - itervalues = MutableMapping.itervalues + return {k.lower(): v for k, v in self.itermerged()} == { + k.lower(): v for k, v in other_as_http_header_dict.itermerged() + } - __marker = object() + def __ne__(self, other: object) -> bool: + return not self.__eq__(other) - def __len__(self): + def __len__(self) -> int: return len(self._container) - def __iter__(self): + def __iter__(self) -> typing.Iterator[str]: # Only provide the originally cased names for vals in self._container.values(): yield vals[0] - def pop(self, key, default=__marker): - """D.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - """ - # Using the MutableMapping function directly fails due to the private marker. - # Using ordinary dict.pop would expose the internal structures. - # So let's reinvent the wheel. - try: - value = self[key] - except KeyError: - if default is self.__marker: - raise - return default - else: - del self[key] - return value - - def discard(self, key): + def discard(self, key: str) -> None: try: del self[key] except KeyError: pass - def add(self, key, val): + def add(self, key: str, val: str, *, combine: bool = False) -> None: """Adds a (name, value) pair, doesn't overwrite the value if it already exists. + If this is called with combine=True, instead of adding a new header value + as a distinct item during iteration, this will instead append the value to + any existing header value with a comma. If no existing header value exists + for the key, then the value will simply be added, ignoring the combine parameter. + >>> headers = HTTPHeaderDict(foo='bar') >>> headers.add('Foo', 'baz') >>> headers['foo'] 'bar, baz' + >>> list(headers.items()) + [('foo', 'bar'), ('foo', 'baz')] + >>> headers.add('foo', 'quz', combine=True) + >>> list(headers.items()) + [('foo', 'bar, baz, quz')] """ + # avoid a bytes/str comparison by decoding before httplib + if isinstance(key, bytes): + key = key.decode("latin-1") key_lower = key.lower() new_vals = [key, val] # Keep the common case aka no item present as fast as possible vals = self._container.setdefault(key_lower, new_vals) if new_vals is not vals: - vals.append(val) + # if there are values here, then there is at least the initial + # key/value pair + assert len(vals) >= 2 + if combine: + vals[-1] = vals[-1] + ", " + val + else: + vals.append(val) - def extend(self, *args, **kwargs): + def extend(self, *args: ValidHTTPHeaderSource, **kwargs: str) -> None: """Generic import function for any type of header-like object. Adapted version of MutableMapping.update in order to insert items with self.add instead of self.__setitem__ """ if len(args) > 1: raise TypeError( - "extend() takes at most 1 positional " - "arguments ({0} given)".format(len(args)) + f"extend() takes at most 1 positional arguments ({len(args)} given)" ) other = args[0] if len(args) >= 1 else () if isinstance(other, HTTPHeaderDict): for key, val in other.iteritems(): self.add(key, val) - elif isinstance(other, Mapping): - for key in other: - self.add(key, other[key]) - elif hasattr(other, "keys"): - for key in other.keys(): - self.add(key, other[key]) - else: + elif isinstance(other, typing.Mapping): + for key, val in other.items(): + self.add(key, val) + elif isinstance(other, typing.Iterable): + other = typing.cast(typing.Iterable[tuple[str, str]], other) for key, value in other: self.add(key, value) + elif hasattr(other, "keys") and hasattr(other, "__getitem__"): + # THIS IS NOT A TYPESAFE BRANCH + # In this branch, the object has a `keys` attr but is not a Mapping or any of + # the other types indicated in the method signature. We do some stuff with + # it as though it partially implements the Mapping interface, but we're not + # doing that stuff safely AT ALL. + for key in other.keys(): + self.add(key, other[key]) for key, value in kwargs.items(): self.add(key, value) - def getlist(self, key, default=__marker): + @typing.overload + def getlist(self, key: str) -> list[str]: ... + + @typing.overload + def getlist(self, key: str, default: _DT) -> list[str] | _DT: ... + + def getlist( + self, key: str, default: _Sentinel | _DT = _Sentinel.not_passed + ) -> list[str] | _DT: """Returns a list of all the values for the named field. Returns an empty list if the key doesn't exist.""" try: vals = self._container[key.lower()] except KeyError: - if default is self.__marker: + if default is _Sentinel.not_passed: + # _DT is unbound; empty list is instance of List[str] return [] + # _DT is bound; default is instance of _DT return default else: + # _DT may or may not be bound; vals[1:] is instance of List[str], which + # meets our external interface requirement of `Union[List[str], _DT]`. return vals[1:] - def _prepare_for_method_change(self): + def _prepare_for_method_change(self) -> Self: """ Remove content-specific header fields before changing the request method to GET or HEAD according to RFC 9110, Section 15.4. @@ -294,62 +415,65 @@ def _prepare_for_method_change(self): # Backwards compatibility for http.cookiejar get_all = getlist - def __repr__(self): - return "%s(%s)" % (type(self).__name__, dict(self.itermerged())) + def __repr__(self) -> str: + return f"{type(self).__name__}({dict(self.itermerged())})" - def _copy_from(self, other): + def _copy_from(self, other: HTTPHeaderDict) -> None: for key in other: val = other.getlist(key) - if isinstance(val, list): - # Don't need to convert tuples - val = list(val) - self._container[key.lower()] = [key] + val + self._container[key.lower()] = [key, *val] - def copy(self): + def copy(self) -> Self: clone = type(self)() clone._copy_from(self) return clone - def iteritems(self): + def iteritems(self) -> typing.Iterator[tuple[str, str]]: """Iterate over all header lines, including duplicate ones.""" for key in self: vals = self._container[key.lower()] for val in vals[1:]: yield vals[0], val - def itermerged(self): + def itermerged(self) -> typing.Iterator[tuple[str, str]]: """Iterate over all headers, merging duplicate ones together.""" for key in self: val = self._container[key.lower()] yield val[0], ", ".join(val[1:]) - def items(self): - return list(self.iteritems()) - - @classmethod - def from_httplib(cls, message): # Python 2 - """Read headers from a Python 2 httplib message object.""" - # python2.7 does not expose a proper API for exporting multiheaders - # efficiently. This function re-reads raw lines from the message - # object and extracts the multiheaders properly. - obs_fold_continued_leaders = (" ", "\t") - headers = [] - - for line in message.headers: - if line.startswith(obs_fold_continued_leaders): - if not headers: - # We received a header line that starts with OWS as described - # in RFC-7230 S3.2.4. This indicates a multiline header, but - # there exists no previous header to which we can attach it. - raise InvalidHeader( - "Header continuation with no previous header: %s" % line - ) - else: - key, value = headers[-1] - headers[-1] = (key, value + " " + line.strip()) - continue - - key, value = line.split(":", 1) - headers.append((key, value.strip())) - - return cls(headers) + def items(self) -> HTTPHeaderDictItemView: # type: ignore[override] + return HTTPHeaderDictItemView(self) + + def _has_value_for_header(self, header_name: str, potential_value: str) -> bool: + if header_name in self: + return potential_value in self._container[header_name.lower()][1:] + return False + + def __ior__(self, other: object) -> HTTPHeaderDict: + # Supports extending a header dict in-place using operator |= + # combining items with add instead of __setitem__ + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return NotImplemented + self.extend(maybe_constructable) + return self + + def __or__(self, other: object) -> Self: + # Supports merging header dicts using operator | + # combining items with add instead of __setitem__ + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return NotImplemented + result = self.copy() + result.extend(maybe_constructable) + return result + + def __ror__(self, other: object) -> Self: + # Supports merging header dicts using operator | when other is on left side + # combining items with add instead of __setitem__ + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return NotImplemented + result = type(self)(maybe_constructable) + result.extend(self) + return result diff --git a/src/urllib3/request.py b/src/urllib3/_request_methods.py similarity index 50% rename from src/urllib3/request.py rename to src/urllib3/_request_methods.py index 3b4cf99..297c271 100644 --- a/src/urllib3/request.py +++ b/src/urllib3/_request_methods.py @@ -1,15 +1,23 @@ -from __future__ import absolute_import +from __future__ import annotations -import sys +import json as _json +import typing +from urllib.parse import urlencode -from .filepost import encode_multipart_formdata -from .packages import six -from .packages.six.moves.urllib.parse import urlencode +from ._base_connection import _TYPE_BODY +from ._collections import HTTPHeaderDict +from .filepost import _TYPE_FIELDS, encode_multipart_formdata +from .response import BaseHTTPResponse __all__ = ["RequestMethods"] +_TYPE_ENCODE_URL_FIELDS = typing.Union[ + typing.Sequence[tuple[str, typing.Union[str, bytes]]], + typing.Mapping[str, typing.Union[str, bytes]], +] -class RequestMethods(object): + +class RequestMethods: """ Convenience mixin for classes who implement a :meth:`urlopen` method, such as :class:`urllib3.HTTPConnectionPool` and @@ -40,25 +48,34 @@ class RequestMethods(object): _encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"} - def __init__(self, headers=None): + def __init__(self, headers: typing.Mapping[str, str] | None = None) -> None: self.headers = headers or {} def urlopen( self, - method, - url, - body=None, - headers=None, - encode_multipart=True, - multipart_boundary=None, - **kw - ): # Abstract + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + encode_multipart: bool = True, + multipart_boundary: str | None = None, + **kw: typing.Any, + ) -> BaseHTTPResponse: # Abstract raise NotImplementedError( "Classes extending RequestMethods must implement " "their own ``urlopen`` method." ) - def request(self, method, url, fields=None, headers=None, **urlopen_kw): + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + fields: _TYPE_FIELDS | None = None, + headers: typing.Mapping[str, str] | None = None, + json: typing.Any | None = None, + **urlopen_kw: typing.Any, + ) -> BaseHTTPResponse: """ Make a request using :meth:`urlopen` with the appropriate encoding of ``fields`` based on the ``method`` used. @@ -68,29 +85,95 @@ def request(self, method, url, fields=None, headers=None, **urlopen_kw): option to drop down to more specific methods when necessary, such as :meth:`request_encode_url`, :meth:`request_encode_body`, or even the lowest level :meth:`urlopen`. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param body: + Data to send in the request body, either :class:`str`, :class:`bytes`, + an iterable of :class:`str`/:class:`bytes`, or a file-like object. + + :param fields: + Data to encode and send in the URL or request body, depending on ``method``. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param json: + Data to encode and send as JSON with UTF-encoded in the request body. + The ``"Content-Type"`` header will be set to ``"application/json"`` + unless specified otherwise. """ method = method.upper() - urlopen_kw["request_url"] = url + if json is not None and body is not None: + raise TypeError( + "request got values for both 'body' and 'json' parameters which are mutually exclusive" + ) + + if json is not None: + if headers is None: + headers = self.headers + + if not ("content-type" in map(str.lower, headers.keys())): + headers = HTTPHeaderDict(headers) + headers["Content-Type"] = "application/json" + + body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode( + "utf-8" + ) + + if body is not None: + urlopen_kw["body"] = body if method in self._encode_url_methods: return self.request_encode_url( - method, url, fields=fields, headers=headers, **urlopen_kw + method, + url, + fields=fields, # type: ignore[arg-type] + headers=headers, + **urlopen_kw, ) else: return self.request_encode_body( method, url, fields=fields, headers=headers, **urlopen_kw ) - def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw): + def request_encode_url( + self, + method: str, + url: str, + fields: _TYPE_ENCODE_URL_FIELDS | None = None, + headers: typing.Mapping[str, str] | None = None, + **urlopen_kw: str, + ) -> BaseHTTPResponse: """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the url. This is useful for request methods like GET, HEAD, DELETE, etc. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param fields: + Data to encode and send in the URL. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. """ if headers is None: headers = self.headers - extra_kw = {"headers": headers} + extra_kw: dict[str, typing.Any] = {"headers": headers} extra_kw.update(urlopen_kw) if fields: @@ -100,14 +183,14 @@ def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_k def request_encode_body( self, - method, - url, - fields=None, - headers=None, - encode_multipart=True, - multipart_boundary=None, - **urlopen_kw - ): + method: str, + url: str, + fields: _TYPE_FIELDS | None = None, + headers: typing.Mapping[str, str] | None = None, + encode_multipart: bool = True, + multipart_boundary: str | None = None, + **urlopen_kw: str, + ) -> BaseHTTPResponse: """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the body. This is useful for request methods like POST, PUT, PATCH, etc. @@ -142,11 +225,34 @@ def request_encode_body( be overwritten because it depends on the dynamic random boundary string which is used to compose the body of the request. The random boundary string can be explicitly set with the ``multipart_boundary`` parameter. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param fields: + Data to encode and send in the request body. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param encode_multipart: + If True, encode the ``fields`` using the multipart/form-data MIME + format. + + :param multipart_boundary: + If not specified, then a random boundary will be generated using + :func:`urllib3.filepost.choose_boundary`. """ if headers is None: headers = self.headers - extra_kw = {"headers": {}} + extra_kw: dict[str, typing.Any] = {"headers": HTTPHeaderDict(headers)} + body: bytes | str if fields: if "body" in urlopen_kw: @@ -160,32 +266,13 @@ def request_encode_body( ) else: body, content_type = ( - urlencode(fields), + urlencode(fields), # type: ignore[arg-type] "application/x-www-form-urlencoded", ) extra_kw["body"] = body - extra_kw["headers"] = {"Content-Type": content_type} + extra_kw["headers"].setdefault("Content-Type", content_type) - extra_kw["headers"].update(headers) extra_kw.update(urlopen_kw) return self.urlopen(method, url, **extra_kw) - - -if not six.PY2: - - class RequestModule(sys.modules[__name__].__class__): - def __call__(self, *args, **kwargs): - """ - If user tries to call this module directly urllib3 v2.x style raise an error to the user - suggesting they may need urllib3 v2 - """ - raise TypeError( - "'module' object is not callable\n" - "urllib3.request() method is not supported in this release, " - "upgrade to urllib3 v2 to use it\n" - "see https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html" - ) - - sys.modules[__name__].__class__ = RequestModule diff --git a/src/urllib3/_version.py b/src/urllib3/_version.py index 85e725e..af6144b 100644 --- a/src/urllib3/_version.py +++ b/src/urllib3/_version.py @@ -1,2 +1,16 @@ -# This file is protected via CODEOWNERS -__version__ = "1.26.18" +# file generated by setuptools_scm +# don't change, don't track in version control +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple, Union + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '2.3.0' +__version_tuple__ = version_tuple = (2, 3, 0) diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 54b96b1..591ac40 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -1,59 +1,59 @@ -from __future__ import absolute_import +from __future__ import annotations import datetime +import http.client import logging import os import re import socket +import sys +import threading +import typing import warnings -from socket import error as SocketError +from http.client import HTTPConnection as _HTTPConnection +from http.client import HTTPException as HTTPException # noqa: F401 +from http.client import ResponseNotReady from socket import timeout as SocketTimeout -from .packages import six -from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection -from .packages.six.moves.http_client import HTTPException # noqa: F401 -from .util.proxy import create_proxy_ssl_context +if typing.TYPE_CHECKING: + from .response import HTTPResponse + from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT + from .util.ssltransport import SSLTransport + +from ._collections import HTTPHeaderDict +from .http2 import probe as http2_probe +from .util.response import assert_header_parsing +from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout +from .util.util import to_str +from .util.wait import wait_for_read try: # Compiled with SSL? import ssl BaseSSLError = ssl.SSLError -except (ImportError, AttributeError): # Platform-specific: No SSL. - ssl = None - - class BaseSSLError(BaseException): - pass - - -try: - # Python 3: not a no-op, we're adding this to the namespace so it can be imported. - ConnectionError = ConnectionError -except NameError: - # Python 2 - class ConnectionError(Exception): - pass - - -try: # Python 3: - # Not a no-op, we're adding this to the namespace so it can be imported. - BrokenPipeError = BrokenPipeError -except NameError: # Python 2: +except (ImportError, AttributeError): + ssl = None # type: ignore[assignment] - class BrokenPipeError(Exception): + class BaseSSLError(BaseException): # type: ignore[no-redef] pass -from ._collections import HTTPHeaderDict # noqa (historical, removed in v2) +from ._base_connection import _TYPE_BODY +from ._base_connection import ProxyConfig as ProxyConfig +from ._base_connection import _ResponseOptions as _ResponseOptions from ._version import __version__ from .exceptions import ( ConnectTimeoutError, + HeaderParsingError, + NameResolutionError, NewConnectionError, - SubjectAltNameWarning, + ProxyError, SystemTimeWarning, ) -from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection +from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_ +from .util.request import body_to_chunks +from .util.ssl_ import assert_fingerprint as _assert_fingerprint from .util.ssl_ import ( - assert_fingerprint, create_urllib3_context, is_ipaddress, resolve_cert_reqs, @@ -61,6 +61,12 @@ class BrokenPipeError(Exception): ssl_wrap_socket, ) from .util.ssl_match_hostname import CertificateError, match_hostname +from .util.url import Url + +# Not a no-op, we're adding this to the namespace so it can be imported. +ConnectionError = ConnectionError +BrokenPipeError = BrokenPipeError + log = logging.getLogger(__name__) @@ -68,12 +74,12 @@ class BrokenPipeError(Exception): # When it comes time to update this value as a part of regular maintenance # (ie test_recent_date is failing) update it to ~6 months before the current date. -RECENT_DATE = datetime.date(2022, 1, 1) +RECENT_DATE = datetime.date(2023, 6, 1) _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") -class HTTPConnection(_HTTPConnection, object): +class HTTPConnection(_HTTPConnection): """ Based on :class:`http.client.HTTPConnection` but provides an extra constructor backwards-compatibility layer between older and newer Pythons. @@ -81,7 +87,6 @@ class HTTPConnection(_HTTPConnection, object): Additional keyword parameters are used to configure attributes of the connection. Accepted parameters include: - - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` - ``source_address``: Set the source address for the current connection. - ``socket_options``: Set specific options on the underlying socket. If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling @@ -99,38 +104,64 @@ class HTTPConnection(_HTTPConnection, object): Or you may want to disable the defaults by passing an empty list (e.g., ``[]``). """ - default_port = port_by_scheme["http"] + default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc] #: Disable Nagle's algorithm by default. #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` - default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + default_socket_options: typing.ClassVar[connection._TYPE_SOCKET_OPTIONS] = [ + (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + ] #: Whether this connection verifies the host's certificate. - is_verified = False + is_verified: bool = False - #: Whether this proxy connection (if used) verifies the proxy host's - #: certificate. - proxy_is_verified = None + #: Whether this proxy connection verified the proxy host's certificate. + # If no proxy is currently connected to the value will be ``None``. + proxy_is_verified: bool | None = None - def __init__(self, *args, **kw): - if not six.PY2: - kw.pop("strict", None) + blocksize: int + source_address: tuple[str, int] | None + socket_options: connection._TYPE_SOCKET_OPTIONS | None - # Pre-set source_address. - self.source_address = kw.get("source_address") + _has_connected_to_proxy: bool + _response_options: _ResponseOptions | None + _tunnel_host: str | None + _tunnel_port: int | None + _tunnel_scheme: str | None - #: The socket options provided by the user. If no options are - #: provided, we use the default options. - self.socket_options = kw.pop("socket_options", self.default_socket_options) - - # Proxy options provided by the user. - self.proxy = kw.pop("proxy", None) - self.proxy_config = kw.pop("proxy_config", None) + def __init__( + self, + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: None | ( + connection._TYPE_SOCKET_OPTIONS + ) = default_socket_options, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + ) -> None: + super().__init__( + host=host, + port=port, + timeout=Timeout.resolve_default_timeout(timeout), + source_address=source_address, + blocksize=blocksize, + ) + self.socket_options = socket_options + self.proxy = proxy + self.proxy_config = proxy_config - _HTTPConnection.__init__(self, *args, **kw) + self._has_connected_to_proxy = False + self._response_options = None + self._tunnel_host: str | None = None + self._tunnel_port: int | None = None + self._tunnel_scheme: str | None = None @property - def host(self): + def host(self) -> str: """ Getter method to remove any trailing dots that indicate the hostname is an FQDN. @@ -149,7 +180,7 @@ def host(self): return self._dns_host.rstrip(".") @host.setter - def host(self, value): + def host(self, value: str) -> None: """ Setter for the `host` property. @@ -158,129 +189,360 @@ def host(self, value): """ self._dns_host = value - def _new_conn(self): + def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ - extra_kw = {} - if self.source_address: - extra_kw["source_address"] = self.source_address - - if self.socket_options: - extra_kw["socket_options"] = self.socket_options - try: - conn = connection.create_connection( - (self._dns_host, self.port), self.timeout, **extra_kw + sock = connection.create_connection( + (self._dns_host, self.port), + self.timeout, + source_address=self.source_address, + socket_options=self.socket_options, ) - - except SocketTimeout: + except socket.gaierror as e: + raise NameResolutionError(self.host, self, e) from e + except SocketTimeout as e: raise ConnectTimeoutError( self, - "Connection to %s timed out. (connect timeout=%s)" - % (self.host, self.timeout), - ) + f"Connection to {self.host} timed out. (connect timeout={self.timeout})", + ) from e - except SocketError as e: + except OSError as e: raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e + self, f"Failed to establish a new connection: {e}" + ) from e + + sys.audit("http.client.connect", self, self.host, self.port) + + return sock + + def set_tunnel( + self, + host: str, + port: int | None = None, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: + if scheme not in ("http", "https"): + raise ValueError( + f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'" ) + super().set_tunnel(host, port=port, headers=headers) + self._tunnel_scheme = scheme - return conn + if sys.version_info < (3, 11, 4): - def _is_using_tunnel(self): - # Google App Engine's httplib does not define _tunnel_host - return getattr(self, "_tunnel_host", None) + def _tunnel(self) -> None: + _MAXLINE = http.client._MAXLINE # type: ignore[attr-defined] + connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( # type: ignore[str-format] + self._tunnel_host.encode("ascii"), # type: ignore[union-attr] + self._tunnel_port, + ) + headers = [connect] + for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined] + headers.append(f"{header}: {value}\r\n".encode("latin-1")) + headers.append(b"\r\n") + # Making a single send() call instead of one per line encourages + # the host OS to use a more optimal packet size instead of + # potentially emitting a series of small packets. + self.send(b"".join(headers)) + del headers + + response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined] + try: + (version, code, message) = response._read_status() # type: ignore[attr-defined] + + if code != http.HTTPStatus.OK: + self.close() + raise OSError(f"Tunnel connection failed: {code} {message.strip()}") + while True: + line = response.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise http.client.LineTooLong("header line") + if not line: + # for sites which EOF without sending a trailer + break + if line in (b"\r\n", b"\n", b""): + break + + if self.debuglevel > 0: + print("header:", line.decode()) + finally: + response.close() + + def connect(self) -> None: + self.sock = self._new_conn() + if self._tunnel_host: + # If we're tunneling it means we're connected to our proxy. + self._has_connected_to_proxy = True - def _prepare_conn(self, conn): - self.sock = conn - if self._is_using_tunnel(): # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() - # Mark this connection as not reusable - self.auto_open = 0 - def connect(self): - conn = self._new_conn() - self._prepare_conn(conn) + # If there's a proxy to be connected to we are fully connected. + # This is set twice (once above and here) due to forwarding proxies + # not using tunnelling. + self._has_connected_to_proxy = bool(self.proxy) + + if self._has_connected_to_proxy: + self.proxy_is_verified = False + + @property + def is_closed(self) -> bool: + return self.sock is None + + @property + def is_connected(self) -> bool: + if self.sock is None: + return False + return not wait_for_read(self.sock, timeout=0.0) + + @property + def has_connected_to_proxy(self) -> bool: + return self._has_connected_to_proxy + + @property + def proxy_is_forwarding(self) -> bool: + """ + Return True if a forwarding proxy is configured, else return False + """ + return bool(self.proxy) and self._tunnel_host is None - def putrequest(self, method, url, *args, **kwargs): - """ """ + @property + def proxy_is_tunneling(self) -> bool: + """ + Return True if a tunneling proxy is configured, else return False + """ + return self._tunnel_host is not None + + def close(self) -> None: + try: + super().close() + finally: + # Reset all stateful properties so connection + # can be re-used without leaking prior configs. + self.sock = None + self.is_verified = False + self.proxy_is_verified = None + self._has_connected_to_proxy = False + self._response_options = None + self._tunnel_host = None + self._tunnel_port = None + self._tunnel_scheme = None + + def putrequest( + self, + method: str, + url: str, + skip_host: bool = False, + skip_accept_encoding: bool = False, + ) -> None: + """""" # Empty docstring because the indentation of CPython's implementation # is broken but we don't want this method in our documentation. match = _CONTAINS_CONTROL_CHAR_RE.search(method) if match: raise ValueError( - "Method cannot contain non-token characters %r (found at least %r)" - % (method, match.group()) + f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})" ) - return _HTTPConnection.putrequest(self, method, url, *args, **kwargs) + return super().putrequest( + method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding + ) - def putheader(self, header, *values): - """ """ + def putheader(self, header: str, *values: str) -> None: # type: ignore[override] + """""" if not any(isinstance(v, str) and v == SKIP_HEADER for v in values): - _HTTPConnection.putheader(self, header, *values) - elif six.ensure_str(header.lower()) not in SKIPPABLE_HEADERS: + super().putheader(header, *values) + elif to_str(header.lower()) not in SKIPPABLE_HEADERS: + skippable_headers = "', '".join( + [str.title(header) for header in sorted(SKIPPABLE_HEADERS)] + ) raise ValueError( - "urllib3.util.SKIP_HEADER only supports '%s'" - % ("', '".join(map(str.title, sorted(SKIPPABLE_HEADERS))),) + f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'" ) - def request(self, method, url, body=None, headers=None): + # `request` method's signature intentionally violates LSP. + # urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental. + def request( # type: ignore[override] + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + *, + chunked: bool = False, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> None: # Update the inner socket's timeout value to send the request. # This only triggers if the connection is re-used. - if getattr(self, "sock", None) is not None: + if self.sock is not None: self.sock.settimeout(self.timeout) + # Store these values to be fed into the HTTPResponse + # object later. TODO: Remove this in favor of a real + # HTTP lifecycle mechanism. + + # We have to store these before we call .request() + # because sometimes we can still salvage a response + # off the wire even if we aren't able to completely + # send the request body. + self._response_options = _ResponseOptions( + request_method=method, + request_url=url, + preload_content=preload_content, + decode_content=decode_content, + enforce_content_length=enforce_content_length, + ) + if headers is None: headers = {} - else: - # Avoid modifying the headers passed into .request() - headers = headers.copy() - if "user-agent" not in (six.ensure_str(k.lower()) for k in headers): - headers["User-Agent"] = _get_default_user_agent() - super(HTTPConnection, self).request(method, url, body=body, headers=headers) - - def request_chunked(self, method, url, body=None, headers=None): - """ - Alternative to the common request method, which sends the - body with chunked encoding and not as one block - """ - headers = headers or {} - header_keys = set([six.ensure_str(k.lower()) for k in headers]) + header_keys = frozenset(to_str(k.lower()) for k in headers) skip_accept_encoding = "accept-encoding" in header_keys skip_host = "host" in header_keys self.putrequest( method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host ) + + # Transform the body into an iterable of sendall()-able chunks + # and detect if an explicit Content-Length is doable. + chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize) + chunks = chunks_and_cl.chunks + content_length = chunks_and_cl.content_length + + # When chunked is explicit set to 'True' we respect that. + if chunked: + if "transfer-encoding" not in header_keys: + self.putheader("Transfer-Encoding", "chunked") + else: + # Detect whether a framing mechanism is already in use. If so + # we respect that value, otherwise we pick chunked vs content-length + # depending on the type of 'body'. + if "content-length" in header_keys: + chunked = False + elif "transfer-encoding" in header_keys: + chunked = True + + # Otherwise we go off the recommendation of 'body_to_chunks()'. + else: + chunked = False + if content_length is None: + if chunks is not None: + chunked = True + self.putheader("Transfer-Encoding", "chunked") + else: + self.putheader("Content-Length", str(content_length)) + + # Now that framing headers are out of the way we send all the other headers. if "user-agent" not in header_keys: self.putheader("User-Agent", _get_default_user_agent()) for header, value in headers.items(): self.putheader(header, value) - if "transfer-encoding" not in header_keys: - self.putheader("Transfer-Encoding", "chunked") self.endheaders() - if body is not None: - stringish_types = six.string_types + (bytes,) - if isinstance(body, stringish_types): - body = (body,) - for chunk in body: + # If we're given a body we start sending that in chunks. + if chunks is not None: + for chunk in chunks: + # Sending empty chunks isn't allowed for TE: chunked + # as it indicates the end of the body. if not chunk: continue - if not isinstance(chunk, bytes): - chunk = chunk.encode("utf8") - len_str = hex(len(chunk))[2:] - to_send = bytearray(len_str.encode()) - to_send += b"\r\n" - to_send += chunk - to_send += b"\r\n" - self.send(to_send) + if isinstance(chunk, str): + chunk = chunk.encode("utf-8") + if chunked: + self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk)) + else: + self.send(chunk) + + # Regardless of whether we have a body or not, if we're in + # chunked mode we want to send an explicit empty chunk. + if chunked: + self.send(b"0\r\n\r\n") + + def request_chunked( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + ) -> None: + """ + Alternative to the common request method, which sends the + body with chunked encoding and not as one block + """ + warnings.warn( + "HTTPConnection.request_chunked() is deprecated and will be removed " + "in urllib3 v2.1.0. Instead use HTTPConnection.request(..., chunked=True).", + category=DeprecationWarning, + stacklevel=2, + ) + self.request(method, url, body=body, headers=headers, chunked=True) + + def getresponse( # type: ignore[override] + self, + ) -> HTTPResponse: + """ + Get the response from the server. + + If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable. + + If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed. + """ + # Raise the same error as http.client.HTTPConnection + if self._response_options is None: + raise ResponseNotReady() + + # Reset this attribute for being used again. + resp_options = self._response_options + self._response_options = None + + # Since the connection's timeout value may have been updated + # we need to set the timeout on the socket. + self.sock.settimeout(self.timeout) - # After the if clause, to always have a closed body - self.send(b"0\r\n\r\n") + # This is needed here to avoid circular import errors + from .response import HTTPResponse + + # Save a reference to the shutdown function before ownership is passed + # to httplib_response + # TODO should we implement it everywhere? + _shutdown = getattr(self.sock, "shutdown", None) + + # Get the response from http.client.HTTPConnection + httplib_response = super().getresponse() + + try: + assert_header_parsing(httplib_response.msg) + except (HeaderParsingError, TypeError) as hpe: + log.warning( + "Failed to parse headers (url=%s): %s", + _url_from_connection(self, resp_options.request_url), + hpe, + exc_info=True, + ) + + headers = HTTPHeaderDict(httplib_response.msg.items()) + + response = HTTPResponse( + body=httplib_response, + headers=headers, + status=httplib_response.status, + version=httplib_response.version, + version_string=getattr(self, "_http_vsn_str", "HTTP/?"), + reason=httplib_response.reason, + preload_content=resp_options.preload_content, + decode_content=resp_options.decode_content, + original_response=httplib_response, + enforce_content_length=resp_options.enforce_content_length, + request_method=resp_options.request_method, + request_url=resp_options.request_url, + sock_shutdown=_shutdown, + ) + return response class HTTPSConnection(HTTPConnection): @@ -289,57 +551,103 @@ class HTTPSConnection(HTTPConnection): socket by means of :py:func:`urllib3.util.ssl_wrap_socket`. """ - default_port = port_by_scheme["https"] + default_port = port_by_scheme["https"] # type: ignore[misc] - cert_reqs = None - ca_certs = None - ca_cert_dir = None - ca_cert_data = None - ssl_version = None - assert_fingerprint = None - tls_in_tls_required = False + cert_reqs: int | str | None = None + ca_certs: str | None = None + ca_cert_dir: str | None = None + ca_cert_data: None | str | bytes = None + ssl_version: int | str | None = None + ssl_minimum_version: int | None = None + ssl_maximum_version: int | None = None + assert_fingerprint: str | None = None + _connect_callback: typing.Callable[..., None] | None = None def __init__( self, - host, - port=None, - key_file=None, - cert_file=None, - key_password=None, - strict=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, - server_hostname=None, - **kw - ): - - HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) + host: str, + port: int | None = None, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: None | ( + connection._TYPE_SOCKET_OPTIONS + ) = HTTPConnection.default_socket_options, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + cert_reqs: int | str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + server_hostname: str | None = None, + ssl_context: ssl.SSLContext | None = None, + ca_certs: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, + ssl_version: int | str | None = None, # Deprecated + cert_file: str | None = None, + key_file: str | None = None, + key_password: str | None = None, + ) -> None: + super().__init__( + host, + port=port, + timeout=timeout, + source_address=source_address, + blocksize=blocksize, + socket_options=socket_options, + proxy=proxy, + proxy_config=proxy_config, + ) self.key_file = key_file self.cert_file = cert_file self.key_password = key_password self.ssl_context = ssl_context self.server_hostname = server_hostname + self.assert_hostname = assert_hostname + self.assert_fingerprint = assert_fingerprint + self.ssl_version = ssl_version + self.ssl_minimum_version = ssl_minimum_version + self.ssl_maximum_version = ssl_maximum_version + self.ca_certs = ca_certs and os.path.expanduser(ca_certs) + self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) + self.ca_cert_data = ca_cert_data - # Required property for Google AppEngine 1.9.0 which otherwise causes - # HTTPS requests to go out as HTTP. (See Issue #356) - self._protocol = "https" + # cert_reqs depends on ssl_context so calculate last. + if cert_reqs is None: + if self.ssl_context is not None: + cert_reqs = self.ssl_context.verify_mode + else: + cert_reqs = resolve_cert_reqs(None) + self.cert_reqs = cert_reqs + self._connect_callback = None def set_cert( self, - key_file=None, - cert_file=None, - cert_reqs=None, - key_password=None, - ca_certs=None, - assert_hostname=None, - assert_fingerprint=None, - ca_cert_dir=None, - ca_cert_data=None, - ): + key_file: str | None = None, + cert_file: str | None = None, + cert_reqs: int | str | None = None, + key_password: str | None = None, + ca_certs: str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ) -> None: """ This method should only be called once, before the connection is used. """ + warnings.warn( + "HTTPSConnection.set_cert() is deprecated and will be removed " + "in urllib3 v2.1.0. Instead provide the parameters to the " + "HTTPSConnection constructor.", + category=DeprecationWarning, + stacklevel=2, + ) + # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also # have an SSLContext object in which case we'll use its verify_mode. if cert_reqs is None: @@ -358,191 +666,322 @@ def set_cert( self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) self.ca_cert_data = ca_cert_data - def connect(self): - # Add certificate verification - self.sock = conn = self._new_conn() - hostname = self.host - tls_in_tls = False - - if self._is_using_tunnel(): - if self.tls_in_tls_required: - self.sock = conn = self._connect_tls_proxy(hostname, conn) - tls_in_tls = True - - # Calls self._set_hostport(), so self.host is - # self._tunnel_host below. - self._tunnel() - # Mark this connection as not reusable - self.auto_open = 0 - - # Override the host with the one we're requesting data from. - hostname = self._tunnel_host - - server_hostname = hostname - if self.server_hostname is not None: - server_hostname = self.server_hostname - - is_time_off = datetime.date.today() < RECENT_DATE - if is_time_off: - warnings.warn( - ( - "System time is way off (before {0}). This will probably " - "lead to SSL verification errors" - ).format(RECENT_DATE), - SystemTimeWarning, - ) - - # Wrap socket using verification with the root certs in - # trusted_root_certs - default_ssl_context = False - if self.ssl_context is None: - default_ssl_context = True - self.ssl_context = create_urllib3_context( - ssl_version=resolve_ssl_version(self.ssl_version), - cert_reqs=resolve_cert_reqs(self.cert_reqs), + def connect(self) -> None: + # Today we don't need to be doing this step before the /actual/ socket + # connection, however in the future we'll need to decide whether to + # create a new socket or re-use an existing "shared" socket as a part + # of the HTTP/2 handshake dance. + if self._tunnel_host is not None and self._tunnel_port is not None: + probe_http2_host = self._tunnel_host + probe_http2_port = self._tunnel_port + else: + probe_http2_host = self.host + probe_http2_port = self.port + + # Check if the target origin supports HTTP/2. + # If the value comes back as 'None' it means that the current thread + # is probing for HTTP/2 support. Otherwise, we're waiting for another + # probe to complete, or we get a value right away. + target_supports_http2: bool | None + if "h2" in ssl_.ALPN_PROTOCOLS: + target_supports_http2 = http2_probe.acquire_and_get( + host=probe_http2_host, port=probe_http2_port ) - - context = self.ssl_context - context.verify_mode = resolve_cert_reqs(self.cert_reqs) - - # Try to load OS default certs if none are given. - # Works well on Windows (requires Python3.4+) - if ( - not self.ca_certs - and not self.ca_cert_dir - and not self.ca_cert_data - and default_ssl_context - and hasattr(context, "load_default_certs") - ): - context.load_default_certs() - - self.sock = ssl_wrap_socket( - sock=conn, - keyfile=self.key_file, - certfile=self.cert_file, - key_password=self.key_password, - ca_certs=self.ca_certs, - ca_cert_dir=self.ca_cert_dir, - ca_cert_data=self.ca_cert_data, - server_hostname=server_hostname, - ssl_context=context, - tls_in_tls=tls_in_tls, - ) - - # If we're using all defaults and the connection - # is TLSv1 or TLSv1.1 we throw a DeprecationWarning - # for the host. - if ( - default_ssl_context - and self.ssl_version is None - and hasattr(self.sock, "version") - and self.sock.version() in {"TLSv1", "TLSv1.1"} - ): - warnings.warn( - "Negotiating TLSv1/TLSv1.1 by default is deprecated " - "and will be disabled in urllib3 v2.0.0. Connecting to " - "'%s' with '%s' can be enabled by explicitly opting-in " - "with 'ssl_version'" % (self.host, self.sock.version()), - DeprecationWarning, + else: + # If HTTP/2 isn't going to be offered it doesn't matter if + # the target supports HTTP/2. Don't want to make a probe. + target_supports_http2 = False + + if self._connect_callback is not None: + self._connect_callback( + "before connect", + thread_id=threading.get_ident(), + target_supports_http2=target_supports_http2, ) - if self.assert_fingerprint: - assert_fingerprint( - self.sock.getpeercert(binary_form=True), self.assert_fingerprint - ) - elif ( - context.verify_mode != ssl.CERT_NONE - and not getattr(context, "check_hostname", False) - and self.assert_hostname is not False - ): - # While urllib3 attempts to always turn off hostname matching from - # the TLS library, this cannot always be done. So we check whether - # the TLS Library still thinks it's matching hostnames. - cert = self.sock.getpeercert() - if not cert.get("subjectAltName", ()): + try: + sock: socket.socket | ssl.SSLSocket + self.sock = sock = self._new_conn() + server_hostname: str = self.host + tls_in_tls = False + + # Do we need to establish a tunnel? + if self.proxy_is_tunneling: + # We're tunneling to an HTTPS origin so need to do TLS-in-TLS. + if self._tunnel_scheme == "https": + # _connect_tls_proxy will verify and assign proxy_is_verified + self.sock = sock = self._connect_tls_proxy(self.host, sock) + tls_in_tls = True + elif self._tunnel_scheme == "http": + self.proxy_is_verified = False + + # If we're tunneling it means we're connected to our proxy. + self._has_connected_to_proxy = True + + self._tunnel() + # Override the host with the one we're requesting data from. + server_hostname = typing.cast(str, self._tunnel_host) + + if self.server_hostname is not None: + server_hostname = self.server_hostname + + is_time_off = datetime.date.today() < RECENT_DATE + if is_time_off: warnings.warn( ( - "Certificate for {0} has no `subjectAltName`, falling back to check for a " - "`commonName` for now. This feature is being removed by major browsers and " - "deprecated by RFC 2818. (See https://github.com/urllib3/urllib3/issues/497 " - "for details.)".format(hostname) + f"System time is way off (before {RECENT_DATE}). This will probably " + "lead to SSL verification errors" ), - SubjectAltNameWarning, + SystemTimeWarning, ) - _match_hostname(cert, self.assert_hostname or server_hostname) - self.is_verified = ( - context.verify_mode == ssl.CERT_REQUIRED - or self.assert_fingerprint is not None - ) + # Remove trailing '.' from fqdn hostnames to allow certificate validation + server_hostname_rm_dot = server_hostname.rstrip(".") + + sock_and_verified = _ssl_wrap_socket_and_match_hostname( + sock=sock, + cert_reqs=self.cert_reqs, + ssl_version=self.ssl_version, + ssl_minimum_version=self.ssl_minimum_version, + ssl_maximum_version=self.ssl_maximum_version, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + ca_cert_data=self.ca_cert_data, + cert_file=self.cert_file, + key_file=self.key_file, + key_password=self.key_password, + server_hostname=server_hostname_rm_dot, + ssl_context=self.ssl_context, + tls_in_tls=tls_in_tls, + assert_hostname=self.assert_hostname, + assert_fingerprint=self.assert_fingerprint, + ) + self.sock = sock_and_verified.socket + + # If an error occurs during connection/handshake we may need to release + # our lock so another connection can probe the origin. + except BaseException: + if self._connect_callback is not None: + self._connect_callback( + "after connect failure", + thread_id=threading.get_ident(), + target_supports_http2=target_supports_http2, + ) - def _connect_tls_proxy(self, hostname, conn): + if target_supports_http2 is None: + http2_probe.set_and_release( + host=probe_http2_host, port=probe_http2_port, supports_http2=None + ) + raise + + # If this connection doesn't know if the origin supports HTTP/2 + # we report back to the HTTP/2 probe our result. + if target_supports_http2 is None: + supports_http2 = sock_and_verified.socket.selected_alpn_protocol() == "h2" + http2_probe.set_and_release( + host=probe_http2_host, + port=probe_http2_port, + supports_http2=supports_http2, + ) + + # Forwarding proxies can never have a verified target since + # the proxy is the one doing the verification. Should instead + # use a CONNECT tunnel in order to verify the target. + # See: https://github.com/urllib3/urllib3/issues/3267. + if self.proxy_is_forwarding: + self.is_verified = False + else: + self.is_verified = sock_and_verified.is_verified + + # If there's a proxy to be connected to we are fully connected. + # This is set twice (once above and here) due to forwarding proxies + # not using tunnelling. + self._has_connected_to_proxy = bool(self.proxy) + + # Set `self.proxy_is_verified` unless it's already set while + # establishing a tunnel. + if self._has_connected_to_proxy and self.proxy_is_verified is None: + self.proxy_is_verified = sock_and_verified.is_verified + + def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket: """ Establish a TLS connection to the proxy using the provided SSL context. """ - proxy_config = self.proxy_config + # `_connect_tls_proxy` is called when self._tunnel_host is truthy. + proxy_config = typing.cast(ProxyConfig, self.proxy_config) ssl_context = proxy_config.ssl_context - if ssl_context: - # If the user provided a proxy context, we assume CA and client - # certificates have already been set - return ssl_wrap_socket( - sock=conn, - server_hostname=hostname, - ssl_context=ssl_context, - ) - - ssl_context = create_proxy_ssl_context( - self.ssl_version, - self.cert_reqs, - self.ca_certs, - self.ca_cert_dir, - self.ca_cert_data, - ) - - # If no cert was provided, use only the default options for server - # certificate validation - socket = ssl_wrap_socket( - sock=conn, + sock_and_verified = _ssl_wrap_socket_and_match_hostname( + sock, + cert_reqs=self.cert_reqs, + ssl_version=self.ssl_version, + ssl_minimum_version=self.ssl_minimum_version, + ssl_maximum_version=self.ssl_maximum_version, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, ca_cert_data=self.ca_cert_data, server_hostname=hostname, ssl_context=ssl_context, + assert_hostname=proxy_config.assert_hostname, + assert_fingerprint=proxy_config.assert_fingerprint, + # Features that aren't implemented for proxies yet: + cert_file=None, + key_file=None, + key_password=None, + tls_in_tls=False, ) + self.proxy_is_verified = sock_and_verified.is_verified + return sock_and_verified.socket # type: ignore[return-value] - if ssl_context.verify_mode != ssl.CERT_NONE and not getattr( - ssl_context, "check_hostname", False + +class _WrappedAndVerifiedSocket(typing.NamedTuple): + """ + Wrapped socket and whether the connection is + verified after the TLS handshake + """ + + socket: ssl.SSLSocket | SSLTransport + is_verified: bool + + +def _ssl_wrap_socket_and_match_hostname( + sock: socket.socket, + *, + cert_reqs: None | str | int, + ssl_version: None | str | int, + ssl_minimum_version: int | None, + ssl_maximum_version: int | None, + cert_file: str | None, + key_file: str | None, + key_password: str | None, + ca_certs: str | None, + ca_cert_dir: str | None, + ca_cert_data: None | str | bytes, + assert_hostname: None | str | typing.Literal[False], + assert_fingerprint: str | None, + server_hostname: str | None, + ssl_context: ssl.SSLContext | None, + tls_in_tls: bool = False, +) -> _WrappedAndVerifiedSocket: + """Logic for constructing an SSLContext from all TLS parameters, passing + that down into ssl_wrap_socket, and then doing certificate verification + either via hostname or fingerprint. This function exists to guarantee + that both proxies and targets have the same behavior when connecting via TLS. + """ + default_ssl_context = False + if ssl_context is None: + default_ssl_context = True + context = create_urllib3_context( + ssl_version=resolve_ssl_version(ssl_version), + ssl_minimum_version=ssl_minimum_version, + ssl_maximum_version=ssl_maximum_version, + cert_reqs=resolve_cert_reqs(cert_reqs), + ) + else: + context = ssl_context + + context.verify_mode = resolve_cert_reqs(cert_reqs) + + # In some cases, we want to verify hostnames ourselves + if ( + # `ssl` can't verify fingerprints or alternate hostnames + assert_fingerprint + or assert_hostname + # assert_hostname can be set to False to disable hostname checking + or assert_hostname is False + # We still support OpenSSL 1.0.2, which prevents us from verifying + # hostnames easily: https://github.com/pyca/pyopenssl/pull/933 + or ssl_.IS_PYOPENSSL + or not ssl_.HAS_NEVER_CHECK_COMMON_NAME + ): + context.check_hostname = False + + # Try to load OS default certs if none are given. We need to do the hasattr() check + # for custom pyOpenSSL SSLContext objects because they don't support + # load_default_certs(). + if ( + not ca_certs + and not ca_cert_dir + and not ca_cert_data + and default_ssl_context + and hasattr(context, "load_default_certs") + ): + context.load_default_certs() + + # Ensure that IPv6 addresses are in the proper format and don't have a + # scope ID. Python's SSL module fails to recognize scoped IPv6 addresses + # and interprets them as DNS hostnames. + if server_hostname is not None: + normalized = server_hostname.strip("[]") + if "%" in normalized: + normalized = normalized[: normalized.rfind("%")] + if is_ipaddress(normalized): + server_hostname = normalized + + ssl_sock = ssl_wrap_socket( + sock=sock, + keyfile=key_file, + certfile=cert_file, + key_password=key_password, + ca_certs=ca_certs, + ca_cert_dir=ca_cert_dir, + ca_cert_data=ca_cert_data, + server_hostname=server_hostname, + ssl_context=context, + tls_in_tls=tls_in_tls, + ) + + try: + if assert_fingerprint: + _assert_fingerprint( + ssl_sock.getpeercert(binary_form=True), assert_fingerprint + ) + elif ( + context.verify_mode != ssl.CERT_NONE + and not context.check_hostname + and assert_hostname is not False ): - # While urllib3 attempts to always turn off hostname matching from - # the TLS library, this cannot always be done. So we check whether - # the TLS Library still thinks it's matching hostnames. - cert = socket.getpeercert() - if not cert.get("subjectAltName", ()): - warnings.warn( - ( - "Certificate for {0} has no `subjectAltName`, falling back to check for a " - "`commonName` for now. This feature is being removed by major browsers and " - "deprecated by RFC 2818. (See https://github.com/urllib3/urllib3/issues/497 " - "for details.)".format(hostname) - ), - SubjectAltNameWarning, + cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment] + + # Need to signal to our match_hostname whether to use 'commonName' or not. + # If we're using our own constructed SSLContext we explicitly set 'False' + # because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name. + if default_ssl_context: + hostname_checks_common_name = False + else: + hostname_checks_common_name = ( + getattr(context, "hostname_checks_common_name", False) or False ) - _match_hostname(cert, hostname) - self.proxy_is_verified = ssl_context.verify_mode == ssl.CERT_REQUIRED - return socket + _match_hostname( + cert, + assert_hostname or server_hostname, # type: ignore[arg-type] + hostname_checks_common_name, + ) + + return _WrappedAndVerifiedSocket( + socket=ssl_sock, + is_verified=context.verify_mode == ssl.CERT_REQUIRED + or bool(assert_fingerprint), + ) + except BaseException: + ssl_sock.close() + raise -def _match_hostname(cert, asserted_hostname): +def _match_hostname( + cert: _TYPE_PEER_CERT_RET_DICT | None, + asserted_hostname: str, + hostname_checks_common_name: bool = False, +) -> None: # Our upstream implementation of ssl.match_hostname() # only applies this normalization to IP addresses so it doesn't # match DNS SANs so we do the same thing! - stripped_hostname = asserted_hostname.strip("u[]") + stripped_hostname = asserted_hostname.strip("[]") if is_ipaddress(stripped_hostname): asserted_hostname = stripped_hostname try: - match_hostname(cert, asserted_hostname) + match_hostname(cert, asserted_hostname, hostname_checks_common_name) except CertificateError as e: log.warning( "Certificate did not match expected hostname: %s. Certificate: %s", @@ -551,22 +990,55 @@ def _match_hostname(cert, asserted_hostname): ) # Add cert to exception and reraise so client code can inspect # the cert when catching the exception, if they want to - e._peer_cert = cert + e._peer_cert = cert # type: ignore[attr-defined] raise -def _get_default_user_agent(): - return "python-urllib3/%s" % __version__ - - -class DummyConnection(object): +def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError: + # Look for the phrase 'wrong version number', if found + # then we should warn the user that we're very sure that + # this proxy is HTTP-only and they have a configuration issue. + error_normalized = " ".join(re.split("[^a-z]", str(err).lower())) + is_likely_http_proxy = ( + "wrong version number" in error_normalized + or "unknown protocol" in error_normalized + or "record layer failure" in error_normalized + ) + http_proxy_warning = ( + ". Your proxy appears to only use HTTP and not HTTPS, " + "try changing your proxy URL to be HTTP. See: " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#https-proxy-error-http-proxy" + ) + new_err = ProxyError( + f"Unable to connect to proxy" + f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}", + err, + ) + new_err.__cause__ = err + return new_err + + +def _get_default_user_agent() -> str: + return f"python-urllib3/{__version__}" + + +class DummyConnection: """Used to detect a failed ConnectionCls import.""" - pass - if not ssl: - HTTPSConnection = DummyConnection # noqa: F811 + HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811 VerifiedHTTPSConnection = HTTPSConnection + + +def _url_from_connection( + conn: HTTPConnection | HTTPSConnection, path: str | None = None +) -> str: + """Returns the URL from a given connection. This is mainly used for testing and logging.""" + + scheme = "https" if isinstance(conn, HTTPSConnection) else "http" + + return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 5a6adcb..3a0685b 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -1,15 +1,18 @@ -from __future__ import absolute_import +from __future__ import annotations import errno import logging -import re -import socket +import queue import sys +import typing import warnings -from socket import error as SocketError +import weakref from socket import timeout as SocketTimeout +from types import TracebackType +from ._base_connection import _TYPE_BODY from ._collections import HTTPHeaderDict +from ._request_methods import RequestMethods from .connection import ( BaseSSLError, BrokenPipeError, @@ -17,13 +20,14 @@ HTTPConnection, HTTPException, HTTPSConnection, - VerifiedHTTPSConnection, - port_by_scheme, + ProxyConfig, + _wrap_proxy_error, ) +from .connection import port_by_scheme as port_by_scheme from .exceptions import ( ClosedPoolError, EmptyPoolError, - HeaderParsingError, + FullPoolError, HostChangedError, InsecureRequestWarning, LocationValueError, @@ -35,38 +39,32 @@ SSLError, TimeoutError, ) -from .packages import six -from .packages.six.moves import queue -from .request import RequestMethods -from .response import HTTPResponse +from .response import BaseHTTPResponse from .util.connection import is_connection_dropped from .util.proxy import connection_requires_http_tunnel -from .util.queue import LifoQueue -from .util.request import set_file_position -from .util.response import assert_header_parsing +from .util.request import _TYPE_BODY_POSITION, set_file_position from .util.retry import Retry from .util.ssl_match_hostname import CertificateError -from .util.timeout import Timeout +from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_DEFAULT, Timeout from .util.url import Url, _encode_target from .util.url import _normalize_host as normalize_host -from .util.url import get_host, parse_url +from .util.url import parse_url +from .util.util import to_str -try: # Platform-specific: Python 3 - import weakref +if typing.TYPE_CHECKING: + import ssl - weakref_finalize = weakref.finalize -except AttributeError: # Platform-specific: Python 2 - from .packages.backports.weakref_finalize import weakref_finalize + from typing_extensions import Self -xrange = six.moves.xrange + from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection log = logging.getLogger(__name__) -_Default = object() +_TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None] # Pool objects -class ConnectionPool(object): +class ConnectionPool: """ Base class for all connection pools, such as :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. @@ -77,33 +75,42 @@ class ConnectionPool(object): target URIs. """ - scheme = None - QueueCls = LifoQueue + scheme: str | None = None + QueueCls = queue.LifoQueue - def __init__(self, host, port=None): + def __init__(self, host: str, port: int | None = None) -> None: if not host: raise LocationValueError("No host specified.") self.host = _normalize_host(host, scheme=self.scheme) - self._proxy_host = host.lower() self.port = port - def __str__(self): - return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port) + # This property uses 'normalize_host()' (not '_normalize_host()') + # to avoid removing square braces around IPv6 addresses. + # This value is sent to `HTTPConnection.set_tunnel()` if called + # because square braces are required for HTTP CONNECT tunneling. + self._tunnel_host = normalize_host(host, scheme=self.scheme).lower() - def __enter__(self): + def __str__(self) -> str: + return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})" + + def __enter__(self) -> Self: return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> typing.Literal[False]: self.close() # Return False to re-raise any potential exceptions return False - def close(self): + def close(self) -> None: """ Close all pooled connections and disable the pool. """ - pass # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 @@ -122,14 +129,6 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Port used for this HTTP Connection (None is equivalent to 80), passed into :class:`http.client.HTTPConnection`. - :param strict: - Causes BadStatusLine to be raised if the status line can't be parsed - as a valid HTTP/1.0 or 1.1 status line, passed into - :class:`http.client.HTTPConnection`. - - .. note:: - Only works in Python 2. This parameter is ignored in Python 3. - :param timeout: Socket timeout in seconds for each individual connection. This can be a float or integer, which sets the timeout for the HTTP request, @@ -171,29 +170,25 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """ scheme = "http" - ConnectionCls = HTTPConnection - ResponseCls = HTTPResponse + ConnectionCls: type[BaseHTTPConnection] | type[BaseHTTPSConnection] = HTTPConnection def __init__( self, - host, - port=None, - strict=False, - timeout=Timeout.DEFAULT_TIMEOUT, - maxsize=1, - block=False, - headers=None, - retries=None, - _proxy=None, - _proxy_headers=None, - _proxy_config=None, - **conn_kw + host: str, + port: int | None = None, + timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT, + maxsize: int = 1, + block: bool = False, + headers: typing.Mapping[str, str] | None = None, + retries: Retry | bool | int | None = None, + _proxy: Url | None = None, + _proxy_headers: typing.Mapping[str, str] | None = None, + _proxy_config: ProxyConfig | None = None, + **conn_kw: typing.Any, ): ConnectionPool.__init__(self, host, port) RequestMethods.__init__(self, headers) - self.strict = strict - if not isinstance(timeout, Timeout): timeout = Timeout.from_float(timeout) @@ -203,7 +198,7 @@ def __init__( self.timeout = timeout self.retries = retries - self.pool = self.QueueCls(maxsize) + self.pool: queue.LifoQueue[typing.Any] | None = self.QueueCls(maxsize) self.block = block self.proxy = _proxy @@ -211,7 +206,7 @@ def __init__( self.proxy_config = _proxy_config # Fill the queue up so that doing get() on it will block properly - for _ in xrange(maxsize): + for _ in range(maxsize): self.pool.put(None) # These are mostly for testing and debugging purposes. @@ -236,9 +231,9 @@ def __init__( # Close all the HTTPConnections in the pool before the # HTTPConnectionPool object is garbage collected. - weakref_finalize(self, _close_pool_connections, pool) + weakref.finalize(self, _close_pool_connections, pool) - def _new_conn(self): + def _new_conn(self) -> BaseHTTPConnection: """ Return a fresh :class:`HTTPConnection`. """ @@ -254,12 +249,11 @@ def _new_conn(self): host=self.host, port=self.port, timeout=self.timeout.connect_timeout, - strict=self.strict, - **self.conn_kw + **self.conn_kw, ) return conn - def _get_conn(self, timeout=None): + def _get_conn(self, timeout: float | None = None) -> BaseHTTPConnection: """ Get a connection. Will return a pooled connection if one is available. @@ -272,33 +266,32 @@ def _get_conn(self, timeout=None): :prop:`.block` is ``True``. """ conn = None + + if self.pool is None: + raise ClosedPoolError(self, "Pool is closed.") + try: conn = self.pool.get(block=self.block, timeout=timeout) except AttributeError: # self.pool is None - raise ClosedPoolError(self, "Pool is closed.") + raise ClosedPoolError(self, "Pool is closed.") from None # Defensive: except queue.Empty: if self.block: raise EmptyPoolError( self, - "Pool reached maximum size and no more connections are allowed.", - ) + "Pool is empty and a new connection can't be opened due to blocking mode.", + ) from None pass # Oh well, we'll create a new connection then # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): log.debug("Resetting dropped connection: %s", self.host) conn.close() - if getattr(conn, "auto_open", 1) == 0: - # This is a proxied connection that has been mutated by - # http.client._tunnel() and cannot be reused (since it would - # attempt to bypass the proxy) - conn = None return conn or self._new_conn() - def _put_conn(self, conn): + def _put_conn(self, conn: BaseHTTPConnection | None) -> None: """ Put a connection back into the pool. @@ -312,36 +305,47 @@ def _put_conn(self, conn): If the pool is closed, then the connection will be closed and discarded. """ - try: - self.pool.put(conn, block=False) - return # Everything is dandy, done. - except AttributeError: - # self.pool is None. - pass - except queue.Full: - # This should never happen if self.block == True - log.warning( - "Connection pool is full, discarding connection: %s. Connection pool size: %s", - self.host, - self.pool.qsize(), - ) + if self.pool is not None: + try: + self.pool.put(conn, block=False) + return # Everything is dandy, done. + except AttributeError: + # self.pool is None. + pass + except queue.Full: + # Connection never got put back into the pool, close it. + if conn: + conn.close() + + if self.block: + # This should never happen if you got the conn from self._get_conn + raise FullPoolError( + self, + "Pool reached maximum size and no more connections are allowed.", + ) from None + + log.warning( + "Connection pool is full, discarding connection: %s. Connection pool size: %s", + self.host, + self.pool.qsize(), + ) + # Connection never got put back into the pool, close it. if conn: conn.close() - def _validate_conn(self, conn): + def _validate_conn(self, conn: BaseHTTPConnection) -> None: """ Called right before a request is made, after the socket is created. """ - pass - def _prepare_proxy(self, conn): + def _prepare_proxy(self, conn: BaseHTTPConnection) -> None: # Nothing to do for HTTP connections. pass - def _get_timeout(self, timeout): + def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout: """Helper that always returns a :class:`urllib3.util.Timeout`""" - if timeout is _Default: + if timeout is _DEFAULT_TIMEOUT: return self.timeout.clone() if isinstance(timeout, Timeout): @@ -351,34 +355,40 @@ def _get_timeout(self, timeout): # can be removed later return Timeout.from_float(timeout) - def _raise_timeout(self, err, url, timeout_value): + def _raise_timeout( + self, + err: BaseSSLError | OSError | SocketTimeout, + url: str, + timeout_value: _TYPE_TIMEOUT | None, + ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % timeout_value - ) + self, url, f"Read timed out. (read timeout={timeout_value})" + ) from err - # See the above comment about EAGAIN in Python 3. In Python 2 we have - # to specifically catch it and throw the timeout error + # See the above comment about EAGAIN in Python 3. if hasattr(err, "errno") and err.errno in _blocking_errnos: raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % timeout_value - ) - - # Catch possible read timeouts thrown as SSL errors. If not the - # case, rethrow the original. We need to do this because of: - # http://bugs.python.org/issue10272 - if "timed out" in str(err) or "did not complete (read)" in str( - err - ): # Python < 2.7.4 - raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % timeout_value - ) + self, url, f"Read timed out. (read timeout={timeout_value})" + ) from err def _make_request( - self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw - ): + self, + conn: BaseHTTPConnection, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + retries: Retry | None = None, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + chunked: bool = False, + response_conn: BaseHTTPConnection | None = None, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. @@ -386,12 +396,61 @@ def _make_request( :param conn: a connection from one of our connection pools + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param url: + The URL to perform the request on. + + :param body: + Data to send in the request body, either :class:`str`, :class:`bytes`, + an iterable of :class:`str`/:class:`bytes`, or a file-like object. + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param retries: + Configure the number of retries to allow before raising a + :class:`~urllib3.exceptions.MaxRetryError` exception. + + Pass ``None`` to retry until you receive a response. Pass a + :class:`~urllib3.util.retry.Retry` object for fine-grained control + over different types of retries. + Pass an integer number to retry connection errors that many times, + but no other types of errors. Pass zero to never retry. + + If ``False``, then retries are disabled and any exception is raised + immediately. Also, instead of raising a MaxRetryError on redirects, + the redirect response will be returned. + + :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. + :param timeout: - Socket timeout in seconds for the request. This can be a - float or integer, which will set the same timeout value for - the socket connect and the socket read, or an instance of - :class:`urllib3.util.Timeout`, which gives you more fine-grained - control over your timeouts. + If specified, overrides the default timeout for this one + request. It may be a float (in seconds) or an instance of + :class:`urllib3.util.Timeout`. + + :param chunked: + If True, urllib3 will send the body using chunked transfer + encoding. Otherwise, urllib3 will send the body using the standard + content-length form. Defaults to False. + + :param response_conn: + Set this to ``None`` if you will handle releasing the connection or + set the connection to have the response release it. + + :param preload_content: + If True, the response's body will be preloaded during construction. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + + :param enforce_content_length: + Enforce content length checking. Body returned by server must match + value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 @@ -399,44 +458,66 @@ def _make_request( timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) - # Trigger any extra validation we need to do. try: - self._validate_conn(conn) - except (SocketTimeout, BaseSSLError) as e: - # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. - self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) - raise + # Trigger any extra validation we need to do. + try: + self._validate_conn(conn) + except (SocketTimeout, BaseSSLError) as e: + self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) + raise + + # _validate_conn() starts the connection to an HTTPS proxy + # so we need to wrap errors with 'ProxyError' here too. + except ( + OSError, + NewConnectionError, + TimeoutError, + BaseSSLError, + CertificateError, + SSLError, + ) as e: + new_e: Exception = e + if isinstance(e, (BaseSSLError, CertificateError)): + new_e = SSLError(e) + # If the connection didn't successfully connect to it's proxy + # then there + if isinstance( + new_e, (OSError, NewConnectionError, TimeoutError, SSLError) + ) and (conn and conn.proxy and not conn.has_connected_to_proxy): + new_e = _wrap_proxy_error(new_e, conn.proxy.scheme) + raise new_e # conn.request() calls http.client.*.request, not the method in # urllib3.request. It also calls makefile (recv) on the socket. try: - if chunked: - conn.request_chunked(method, url, **httplib_request_kw) - else: - conn.request(method, url, **httplib_request_kw) + conn.request( + method, + url, + body=body, + headers=headers, + chunked=chunked, + preload_content=preload_content, + decode_content=decode_content, + enforce_content_length=enforce_content_length, + ) # We are swallowing BrokenPipeError (errno.EPIPE) since the server is # legitimately able to close the connection after sending a valid response. # With this behaviour, the received response is still readable. except BrokenPipeError: - # Python 3 pass - except IOError as e: - # Python 2 and macOS/Linux - # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS + except OSError as e: + # MacOS/Linux + # EPROTOTYPE and ECONNRESET are needed on macOS # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ - if e.errno not in { - errno.EPIPE, - errno.ESHUTDOWN, - errno.EPROTOTYPE, - }: + # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE. + if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET: raise # Reset the timeout for the recv() on the socket read_timeout = timeout_obj.read_timeout - # App Engine doesn't have a sock attr - if getattr(conn, "sock", None): + if not conn.is_closed: # In Python 3 socket.py will catch EAGAIN and return None when you # try and read into the file pointer created by http.client, which # instead raises a BadStatusLine exception. Instead of catching @@ -444,33 +525,22 @@ def _make_request( # timeouts, check for a zero timeout before making the request. if read_timeout == 0: raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout + self, url, f"Read timed out. (read timeout={read_timeout})" ) - if read_timeout is Timeout.DEFAULT_TIMEOUT: - conn.sock.settimeout(socket.getdefaulttimeout()) - else: # None or a value - conn.sock.settimeout(read_timeout) + conn.timeout = read_timeout # Receive the response from the server try: - try: - # Python 2.7, use buffering of HTTP responses - httplib_response = conn.getresponse(buffering=True) - except TypeError: - # Python 3 - try: - httplib_response = conn.getresponse() - except BaseException as e: - # Remove the TypeError from the exception chain in - # Python 3 (including for exceptions like SystemExit). - # Otherwise it looks like a bug in the code. - six.raise_from(e, None) - except (SocketTimeout, BaseSSLError, SocketError) as e: + response = conn.getresponse() + except (BaseSSLError, OSError) as e: self._raise_timeout(err=e, url=url, timeout_value=read_timeout) raise - # AppEngine doesn't have a version attr. - http_version = getattr(conn, "_http_vsn_str", "HTTP/?") + # Set properties that are used by the pooling layer. + response.retries = retries + response._connection = response_conn # type: ignore[attr-defined] + response._pool = self # type: ignore[attr-defined] + log.debug( '%s://%s:%s "%s %s %s" %s %s', self.scheme, @@ -478,27 +548,14 @@ def _make_request( self.port, method, url, - http_version, - httplib_response.status, - httplib_response.length, + response.version_string, + response.status, + response.length_remaining, ) - try: - assert_header_parsing(httplib_response.msg) - except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 - log.warning( - "Failed to parse headers (url=%s): %s", - self._absolute_url(url), - hpe, - exc_info=True, - ) - - return httplib_response - - def _absolute_url(self, path): - return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url + return response - def close(self): + def close(self) -> None: """ Close all pooled connections and disable the pool. """ @@ -510,7 +567,7 @@ def close(self): # Close all the HTTPConnections in the pool. _close_pool_connections(old_pool) - def is_same_host(self, url): + def is_same_host(self, url: str) -> bool: """ Check if the given ``url`` is a member of the same host as this connection pool. @@ -519,7 +576,8 @@ def is_same_host(self, url): return True # TODO: Add optional support for socket.gethostbyname checking. - scheme, host, port = get_host(url) + scheme, _, host, port, *_ = parse_url(url) + scheme = scheme or "http" if host is not None: host = _normalize_host(host, scheme=scheme) @@ -531,22 +589,24 @@ def is_same_host(self, url): return (scheme, host, port) == (self.scheme, self.host, self.port) - def urlopen( + def urlopen( # type: ignore[override] self, - method, - url, - body=None, - headers=None, - retries=None, - redirect=True, - assert_same_host=True, - timeout=_Default, - pool_timeout=None, - release_conn=None, - chunked=False, - body_pos=None, - **response_kw - ): + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + retries: Retry | bool | int | None = None, + redirect: bool = True, + assert_same_host: bool = True, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + pool_timeout: int | None = None, + release_conn: bool | None = None, + chunked: bool = False, + body_pos: _TYPE_BODY_POSITION | None = None, + preload_content: bool = True, + decode_content: bool = True, + **response_kw: typing.Any, + ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -554,8 +614,8 @@ def urlopen( .. note:: - More commonly, it's appropriate to use a convenience method provided - by :class:`.RequestMethods`, such as :meth:`request`. + More commonly, it's appropriate to use a convenience method + such as :meth:`request`. .. note:: @@ -583,7 +643,7 @@ def urlopen( Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. - Pass ``None`` to retry until you receive a response. Pass a + If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, @@ -615,6 +675,13 @@ def urlopen( block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. + :param bool preload_content: + If True, the response's body will be preloaded into memory. + + :param bool decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if @@ -622,10 +689,10 @@ def urlopen( `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection - back into the pool. If None, it takes the value of - ``response_kw.get('preload_content', True)``. + back into the pool. If None, it takes the value of ``preload_content`` + which defaults to ``True``. - :param chunked: + :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. @@ -634,12 +701,7 @@ def urlopen( Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. - - :param \\**response_kw: - Additional parameters are passed to - :meth:`urllib3.response.HTTPResponse.from_httplib` """ - parsed_url = parse_url(url) destination_scheme = parsed_url.scheme @@ -650,7 +712,7 @@ def urlopen( retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: - release_conn = response_kw.get("preload_content", True) + release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): @@ -658,9 +720,9 @@ def urlopen( # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): - url = six.ensure_str(_encode_target(url)) + url = to_str(_encode_target(url)) else: - url = six.ensure_str(parsed_url.url) + url = to_str(parsed_url.url) conn = None @@ -683,8 +745,8 @@ def urlopen( # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: - headers = headers.copy() - headers.update(self.proxy_headers) + headers = headers.copy() # type: ignore[attr-defined] + headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. @@ -703,16 +765,26 @@ def urlopen( timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) - conn.timeout = timeout_obj.connect_timeout + conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] - is_new_proxy_conn = self.proxy is not None and not getattr( - conn, "sock", None - ) - if is_new_proxy_conn and http_tunnel_required: - self._prepare_proxy(conn) + # Is this a closed/new connection that requires CONNECT tunnelling? + if self.proxy is not None and http_tunnel_required and conn.is_closed: + try: + self._prepare_proxy(conn) + except (BaseSSLError, OSError, SocketTimeout) as e: + self._raise_timeout( + err=e, url=self.proxy.url, timeout_value=conn.timeout + ) + raise + + # If we're going to release the connection in ``finally:``, then + # the response doesn't need to know about the connection. Otherwise + # it will also try to release it and we'll have a double-release + # mess. + response_conn = conn if not release_conn else None - # Make the request on the httplib connection object. - httplib_response = self._make_request( + # Make the request on the HTTPConnection object + response = self._make_request( conn, method, url, @@ -720,24 +792,11 @@ def urlopen( body=body, headers=headers, chunked=chunked, - ) - - # If we're going to release the connection in ``finally:``, then - # the response doesn't need to know about the connection. Otherwise - # it will also try to release it and we'll have a double-release - # mess. - response_conn = conn if not release_conn else None - - # Pass method to Response for length checking - response_kw["request_method"] = method - - # Import httplib's response into our own wrapper object - response = self.ResponseCls.from_httplib( - httplib_response, - pool=self, - connection=response_conn, retries=retries, - **response_kw + response_conn=response_conn, + preload_content=preload_content, + decode_content=decode_content, + **response_kw, ) # Everything went great! @@ -752,52 +811,35 @@ def urlopen( except ( TimeoutError, HTTPException, - SocketError, + OSError, ProtocolError, BaseSSLError, SSLError, CertificateError, + ProxyError, ) as e: # Discard the connection for these exceptions. It will be # replaced during the next _get_conn() call. clean_exit = False - - def _is_ssl_error_message_from_http_proxy(ssl_error): - # We're trying to detect the message 'WRONG_VERSION_NUMBER' but - # SSLErrors are kinda all over the place when it comes to the message, - # so we try to cover our bases here! - message = " ".join(re.split("[^a-z]", str(ssl_error).lower())) - return ( - "wrong version number" in message or "unknown protocol" in message - ) - - # Try to detect a common user error with proxies which is to - # set an HTTP proxy to be HTTPS when it should be 'http://' - # (ie {'http': 'http://proxy', 'https': 'https://proxy'}) - # Instead we add a nice error message and point to a URL. - if ( - isinstance(e, BaseSSLError) - and self.proxy - and _is_ssl_error_message_from_http_proxy(e) - and conn.proxy - and conn.proxy.scheme == "https" - ): - e = ProxyError( - "Your proxy appears to only use HTTP and not HTTPS, " - "try changing your proxy URL to be HTTP. See: " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#https-proxy-error-http-proxy", - SSLError(e), - ) - elif isinstance(e, (BaseSSLError, CertificateError)): - e = SSLError(e) - elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: - e = ProxyError("Cannot connect to proxy.", e) - elif isinstance(e, (SocketError, HTTPException)): - e = ProtocolError("Connection aborted.", e) + new_e: Exception = e + if isinstance(e, (BaseSSLError, CertificateError)): + new_e = SSLError(e) + if isinstance( + new_e, + ( + OSError, + NewConnectionError, + TimeoutError, + SSLError, + HTTPException, + ), + ) and (conn and conn.proxy and not conn.has_connected_to_proxy): + new_e = _wrap_proxy_error(new_e, conn.proxy.scheme) + elif isinstance(new_e, (OSError, HTTPException)): + new_e = ProtocolError("Connection aborted.", new_e) retries = retries.increment( - method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] + method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2] ) retries.sleep() @@ -810,7 +852,9 @@ def _is_ssl_error_message_from_http_proxy(ssl_error): # to throw the connection away unless explicitly told not to. # Close the connection, set the variable to None, and make sure # we put the None back in the pool to avoid leaking it. - conn = conn and conn.close() + if conn: + conn.close() + conn = None release_this_conn = True if release_this_conn: @@ -837,7 +881,9 @@ def _is_ssl_error_message_from_http_proxy(ssl_error): release_conn=release_conn, chunked=chunked, body_pos=body_pos, - **response_kw + preload_content=preload_content, + decode_content=decode_content, + **response_kw, ) # Handle redirect? @@ -874,7 +920,9 @@ def _is_ssl_error_message_from_http_proxy(ssl_error): release_conn=release_conn, chunked=chunked, body_pos=body_pos, - **response_kw + preload_content=preload_content, + decode_content=decode_content, + **response_kw, ) # Check if we should retry the HTTP response. @@ -904,7 +952,9 @@ def _is_ssl_error_message_from_http_proxy(ssl_error): release_conn=release_conn, chunked=chunked, body_pos=body_pos, - **response_kw + preload_content=preload_content, + decode_content=decode_content, + **response_kw, ) return response @@ -925,37 +975,35 @@ class HTTPSConnectionPool(HTTPConnectionPool): """ scheme = "https" - ConnectionCls = HTTPSConnection + ConnectionCls: type[BaseHTTPSConnection] = HTTPSConnection def __init__( self, - host, - port=None, - strict=False, - timeout=Timeout.DEFAULT_TIMEOUT, - maxsize=1, - block=False, - headers=None, - retries=None, - _proxy=None, - _proxy_headers=None, - key_file=None, - cert_file=None, - cert_reqs=None, - key_password=None, - ca_certs=None, - ssl_version=None, - assert_hostname=None, - assert_fingerprint=None, - ca_cert_dir=None, - **conn_kw - ): - - HTTPConnectionPool.__init__( - self, + host: str, + port: int | None = None, + timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT, + maxsize: int = 1, + block: bool = False, + headers: typing.Mapping[str, str] | None = None, + retries: Retry | bool | int | None = None, + _proxy: Url | None = None, + _proxy_headers: typing.Mapping[str, str] | None = None, + key_file: str | None = None, + cert_file: str | None = None, + cert_reqs: int | str | None = None, + key_password: str | None = None, + ca_certs: str | None = None, + ssl_version: int | str | None = None, + ssl_minimum_version: ssl.TLSVersion | None = None, + ssl_maximum_version: ssl.TLSVersion | None = None, + assert_hostname: str | typing.Literal[False] | None = None, + assert_fingerprint: str | None = None, + ca_cert_dir: str | None = None, + **conn_kw: typing.Any, + ) -> None: + super().__init__( host, port, - strict, timeout, maxsize, block, @@ -963,7 +1011,7 @@ def __init__( retries, _proxy, _proxy_headers, - **conn_kw + **conn_kw, ) self.key_file = key_file @@ -973,47 +1021,29 @@ def __init__( self.ca_certs = ca_certs self.ca_cert_dir = ca_cert_dir self.ssl_version = ssl_version + self.ssl_minimum_version = ssl_minimum_version + self.ssl_maximum_version = ssl_maximum_version self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint - def _prepare_conn(self, conn): - """ - Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket` - and establish the tunnel if proxy is used. - """ - - if isinstance(conn, VerifiedHTTPSConnection): - conn.set_cert( - key_file=self.key_file, - key_password=self.key_password, - cert_file=self.cert_file, - cert_reqs=self.cert_reqs, - ca_certs=self.ca_certs, - ca_cert_dir=self.ca_cert_dir, - assert_hostname=self.assert_hostname, - assert_fingerprint=self.assert_fingerprint, - ) - conn.ssl_version = self.ssl_version - return conn - - def _prepare_proxy(self, conn): - """ - Establishes a tunnel connection through HTTP CONNECT. - - Tunnel connection is established early because otherwise httplib would - improperly set Host: header to proxy's IP:port. - """ - - conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) - - if self.proxy.scheme == "https": - conn.tls_in_tls_required = True + def _prepare_proxy(self, conn: HTTPSConnection) -> None: # type: ignore[override] + """Establishes a tunnel connection through HTTP CONNECT.""" + if self.proxy and self.proxy.scheme == "https": + tunnel_scheme = "https" + else: + tunnel_scheme = "http" + conn.set_tunnel( + scheme=tunnel_scheme, + host=self._tunnel_host, + port=self.port, + headers=self.proxy_headers, + ) conn.connect() - def _new_conn(self): + def _new_conn(self) -> BaseHTTPSConnection: """ - Return a fresh :class:`http.client.HTTPSConnection`. + Return a fresh :class:`urllib3.connection.HTTPConnection`. """ self.num_connections += 1 log.debug( @@ -1023,64 +1053,59 @@ def _new_conn(self): self.port or "443", ) - if not self.ConnectionCls or self.ConnectionCls is DummyConnection: - raise SSLError( + if not self.ConnectionCls or self.ConnectionCls is DummyConnection: # type: ignore[comparison-overlap] + raise ImportError( "Can't connect to HTTPS URL because the SSL module is not available." ) - actual_host = self.host + actual_host: str = self.host actual_port = self.port - if self.proxy is not None: + if self.proxy is not None and self.proxy.host is not None: actual_host = self.proxy.host actual_port = self.proxy.port - conn = self.ConnectionCls( + return self.ConnectionCls( host=actual_host, port=actual_port, timeout=self.timeout.connect_timeout, - strict=self.strict, cert_file=self.cert_file, key_file=self.key_file, key_password=self.key_password, - **self.conn_kw + cert_reqs=self.cert_reqs, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + assert_hostname=self.assert_hostname, + assert_fingerprint=self.assert_fingerprint, + ssl_version=self.ssl_version, + ssl_minimum_version=self.ssl_minimum_version, + ssl_maximum_version=self.ssl_maximum_version, + **self.conn_kw, ) - return self._prepare_conn(conn) - - def _validate_conn(self, conn): + def _validate_conn(self, conn: BaseHTTPConnection) -> None: """ Called right before a request is made, after the socket is created. """ - super(HTTPSConnectionPool, self)._validate_conn(conn) + super()._validate_conn(conn) # Force connect early to allow us to validate the connection. - if not getattr(conn, "sock", None): # AppEngine might not have `.sock` + if conn.is_closed: conn.connect() - if not conn.is_verified: - warnings.warn( - ( - "Unverified HTTPS request is being made to host '%s'. " - "Adding certificate verification is strongly advised. See: " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#ssl-warnings" % conn.host - ), - InsecureRequestWarning, - ) - - if getattr(conn, "proxy_is_verified", None) is False: + # TODO revise this, see https://github.com/urllib3/urllib3/issues/2791 + if not conn.is_verified and not conn.proxy_is_verified: warnings.warn( ( - "Unverified HTTPS connection done to an HTTPS proxy. " + f"Unverified HTTPS request is being made to host '{conn.host}'. " "Adding certificate verification is strongly advised. See: " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#ssl-warnings" + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#tls-warnings" ), InsecureRequestWarning, ) -def connection_from_url(url, **kw): +def connection_from_url(url: str, **kw: typing.Any) -> HTTPConnectionPool: """ Given a url, return an :class:`.ConnectionPool` instance of its host. @@ -1100,15 +1125,24 @@ def connection_from_url(url, **kw): >>> conn = connection_from_url('http://google.com/') >>> r = conn.request('GET', '/') """ - scheme, host, port = get_host(url) + scheme, _, host, port, *_ = parse_url(url) + scheme = scheme or "http" port = port or port_by_scheme.get(scheme, 80) if scheme == "https": - return HTTPSConnectionPool(host, port=port, **kw) + return HTTPSConnectionPool(host, port=port, **kw) # type: ignore[arg-type] else: - return HTTPConnectionPool(host, port=port, **kw) + return HTTPConnectionPool(host, port=port, **kw) # type: ignore[arg-type] + +@typing.overload +def _normalize_host(host: None, scheme: str | None) -> None: ... -def _normalize_host(host, scheme): + +@typing.overload +def _normalize_host(host: str, scheme: str | None) -> str: ... + + +def _normalize_host(host: str | None, scheme: str | None) -> str | None: """ Normalize hosts for comparisons and use with sockets. """ @@ -1121,12 +1155,19 @@ def _normalize_host(host, scheme): # Instead, we need to make sure we never pass ``None`` as the port. # However, for backward compatibility reasons we can't actually # *assert* that. See http://bugs.python.org/issue28539 - if host.startswith("[") and host.endswith("]"): + if host and host.startswith("[") and host.endswith("]"): host = host[1:-1] return host -def _close_pool_connections(pool): +def _url_from_pool( + pool: HTTPConnectionPool | HTTPSConnectionPool, path: str | None = None +) -> str: + """Returns the URL from a given connection pool. This is mainly used for testing and logging.""" + return Url(scheme=pool.scheme, host=pool.host, port=pool.port, path=path).url + + +def _close_pool_connections(pool: queue.LifoQueue[typing.Any]) -> None: """Drains a queue of connections and closes each one.""" try: while True: diff --git a/src/urllib3/contrib/_appengine_environ.py b/src/urllib3/contrib/_appengine_environ.py deleted file mode 100644 index 8765b90..0000000 --- a/src/urllib3/contrib/_appengine_environ.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -This module provides means to detect the App Engine environment. -""" - -import os - - -def is_appengine(): - return is_local_appengine() or is_prod_appengine() - - -def is_appengine_sandbox(): - """Reports if the app is running in the first generation sandbox. - - The second generation runtimes are technically still in a sandbox, but it - is much less restrictive, so generally you shouldn't need to check for it. - see https://cloud.google.com/appengine/docs/standard/runtimes - """ - return is_appengine() and os.environ["APPENGINE_RUNTIME"] == "python27" - - -def is_local_appengine(): - return "APPENGINE_RUNTIME" in os.environ and os.environ.get( - "SERVER_SOFTWARE", "" - ).startswith("Development/") - - -def is_prod_appengine(): - return "APPENGINE_RUNTIME" in os.environ and os.environ.get( - "SERVER_SOFTWARE", "" - ).startswith("Google App Engine/") - - -def is_prod_appengine_mvms(): - """Deprecated.""" - return False diff --git a/src/urllib3/contrib/_securetransport/bindings.py b/src/urllib3/contrib/_securetransport/bindings.py deleted file mode 100644 index 264d564..0000000 --- a/src/urllib3/contrib/_securetransport/bindings.py +++ /dev/null @@ -1,519 +0,0 @@ -""" -This module uses ctypes to bind a whole bunch of functions and constants from -SecureTransport. The goal here is to provide the low-level API to -SecureTransport. These are essentially the C-level functions and constants, and -they're pretty gross to work with. - -This code is a bastardised version of the code found in Will Bond's oscrypto -library. An enormous debt is owed to him for blazing this trail for us. For -that reason, this code should be considered to be covered both by urllib3's -license and by oscrypto's: - - Copyright (c) 2015-2016 Will Bond - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. -""" -from __future__ import absolute_import - -import platform -from ctypes import ( - CDLL, - CFUNCTYPE, - POINTER, - c_bool, - c_byte, - c_char_p, - c_int32, - c_long, - c_size_t, - c_uint32, - c_ulong, - c_void_p, -) -from ctypes.util import find_library - -from ...packages.six import raise_from - -if platform.system() != "Darwin": - raise ImportError("Only macOS is supported") - -version = platform.mac_ver()[0] -version_info = tuple(map(int, version.split("."))) -if version_info < (10, 8): - raise OSError( - "Only OS X 10.8 and newer are supported, not %s.%s" - % (version_info[0], version_info[1]) - ) - - -def load_cdll(name, macos10_16_path): - """Loads a CDLL by name, falling back to known path on 10.16+""" - try: - # Big Sur is technically 11 but we use 10.16 due to the Big Sur - # beta being labeled as 10.16. - if version_info >= (10, 16): - path = macos10_16_path - else: - path = find_library(name) - if not path: - raise OSError # Caught and reraised as 'ImportError' - return CDLL(path, use_errno=True) - except OSError: - raise_from(ImportError("The library %s failed to load" % name), None) - - -Security = load_cdll( - "Security", "/System/Library/Frameworks/Security.framework/Security" -) -CoreFoundation = load_cdll( - "CoreFoundation", - "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", -) - - -Boolean = c_bool -CFIndex = c_long -CFStringEncoding = c_uint32 -CFData = c_void_p -CFString = c_void_p -CFArray = c_void_p -CFMutableArray = c_void_p -CFDictionary = c_void_p -CFError = c_void_p -CFType = c_void_p -CFTypeID = c_ulong - -CFTypeRef = POINTER(CFType) -CFAllocatorRef = c_void_p - -OSStatus = c_int32 - -CFDataRef = POINTER(CFData) -CFStringRef = POINTER(CFString) -CFArrayRef = POINTER(CFArray) -CFMutableArrayRef = POINTER(CFMutableArray) -CFDictionaryRef = POINTER(CFDictionary) -CFArrayCallBacks = c_void_p -CFDictionaryKeyCallBacks = c_void_p -CFDictionaryValueCallBacks = c_void_p - -SecCertificateRef = POINTER(c_void_p) -SecExternalFormat = c_uint32 -SecExternalItemType = c_uint32 -SecIdentityRef = POINTER(c_void_p) -SecItemImportExportFlags = c_uint32 -SecItemImportExportKeyParameters = c_void_p -SecKeychainRef = POINTER(c_void_p) -SSLProtocol = c_uint32 -SSLCipherSuite = c_uint32 -SSLContextRef = POINTER(c_void_p) -SecTrustRef = POINTER(c_void_p) -SSLConnectionRef = c_uint32 -SecTrustResultType = c_uint32 -SecTrustOptionFlags = c_uint32 -SSLProtocolSide = c_uint32 -SSLConnectionType = c_uint32 -SSLSessionOption = c_uint32 - - -try: - Security.SecItemImport.argtypes = [ - CFDataRef, - CFStringRef, - POINTER(SecExternalFormat), - POINTER(SecExternalItemType), - SecItemImportExportFlags, - POINTER(SecItemImportExportKeyParameters), - SecKeychainRef, - POINTER(CFArrayRef), - ] - Security.SecItemImport.restype = OSStatus - - Security.SecCertificateGetTypeID.argtypes = [] - Security.SecCertificateGetTypeID.restype = CFTypeID - - Security.SecIdentityGetTypeID.argtypes = [] - Security.SecIdentityGetTypeID.restype = CFTypeID - - Security.SecKeyGetTypeID.argtypes = [] - Security.SecKeyGetTypeID.restype = CFTypeID - - Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef] - Security.SecCertificateCreateWithData.restype = SecCertificateRef - - Security.SecCertificateCopyData.argtypes = [SecCertificateRef] - Security.SecCertificateCopyData.restype = CFDataRef - - Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] - Security.SecCopyErrorMessageString.restype = CFStringRef - - Security.SecIdentityCreateWithCertificate.argtypes = [ - CFTypeRef, - SecCertificateRef, - POINTER(SecIdentityRef), - ] - Security.SecIdentityCreateWithCertificate.restype = OSStatus - - Security.SecKeychainCreate.argtypes = [ - c_char_p, - c_uint32, - c_void_p, - Boolean, - c_void_p, - POINTER(SecKeychainRef), - ] - Security.SecKeychainCreate.restype = OSStatus - - Security.SecKeychainDelete.argtypes = [SecKeychainRef] - Security.SecKeychainDelete.restype = OSStatus - - Security.SecPKCS12Import.argtypes = [ - CFDataRef, - CFDictionaryRef, - POINTER(CFArrayRef), - ] - Security.SecPKCS12Import.restype = OSStatus - - SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t)) - SSLWriteFunc = CFUNCTYPE( - OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t) - ) - - Security.SSLSetIOFuncs.argtypes = [SSLContextRef, SSLReadFunc, SSLWriteFunc] - Security.SSLSetIOFuncs.restype = OSStatus - - Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t] - Security.SSLSetPeerID.restype = OSStatus - - Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef] - Security.SSLSetCertificate.restype = OSStatus - - Security.SSLSetCertificateAuthorities.argtypes = [SSLContextRef, CFTypeRef, Boolean] - Security.SSLSetCertificateAuthorities.restype = OSStatus - - Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef] - Security.SSLSetConnection.restype = OSStatus - - Security.SSLSetPeerDomainName.argtypes = [SSLContextRef, c_char_p, c_size_t] - Security.SSLSetPeerDomainName.restype = OSStatus - - Security.SSLHandshake.argtypes = [SSLContextRef] - Security.SSLHandshake.restype = OSStatus - - Security.SSLRead.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] - Security.SSLRead.restype = OSStatus - - Security.SSLWrite.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] - Security.SSLWrite.restype = OSStatus - - Security.SSLClose.argtypes = [SSLContextRef] - Security.SSLClose.restype = OSStatus - - Security.SSLGetNumberSupportedCiphers.argtypes = [SSLContextRef, POINTER(c_size_t)] - Security.SSLGetNumberSupportedCiphers.restype = OSStatus - - Security.SSLGetSupportedCiphers.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite), - POINTER(c_size_t), - ] - Security.SSLGetSupportedCiphers.restype = OSStatus - - Security.SSLSetEnabledCiphers.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite), - c_size_t, - ] - Security.SSLSetEnabledCiphers.restype = OSStatus - - Security.SSLGetNumberEnabledCiphers.argtype = [SSLContextRef, POINTER(c_size_t)] - Security.SSLGetNumberEnabledCiphers.restype = OSStatus - - Security.SSLGetEnabledCiphers.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite), - POINTER(c_size_t), - ] - Security.SSLGetEnabledCiphers.restype = OSStatus - - Security.SSLGetNegotiatedCipher.argtypes = [SSLContextRef, POINTER(SSLCipherSuite)] - Security.SSLGetNegotiatedCipher.restype = OSStatus - - Security.SSLGetNegotiatedProtocolVersion.argtypes = [ - SSLContextRef, - POINTER(SSLProtocol), - ] - Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus - - Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)] - Security.SSLCopyPeerTrust.restype = OSStatus - - Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef] - Security.SecTrustSetAnchorCertificates.restype = OSStatus - - Security.SecTrustSetAnchorCertificatesOnly.argstypes = [SecTrustRef, Boolean] - Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus - - Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)] - Security.SecTrustEvaluate.restype = OSStatus - - Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef] - Security.SecTrustGetCertificateCount.restype = CFIndex - - Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex] - Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef - - Security.SSLCreateContext.argtypes = [ - CFAllocatorRef, - SSLProtocolSide, - SSLConnectionType, - ] - Security.SSLCreateContext.restype = SSLContextRef - - Security.SSLSetSessionOption.argtypes = [SSLContextRef, SSLSessionOption, Boolean] - Security.SSLSetSessionOption.restype = OSStatus - - Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol] - Security.SSLSetProtocolVersionMin.restype = OSStatus - - Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol] - Security.SSLSetProtocolVersionMax.restype = OSStatus - - try: - Security.SSLSetALPNProtocols.argtypes = [SSLContextRef, CFArrayRef] - Security.SSLSetALPNProtocols.restype = OSStatus - except AttributeError: - # Supported only in 10.12+ - pass - - Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] - Security.SecCopyErrorMessageString.restype = CFStringRef - - Security.SSLReadFunc = SSLReadFunc - Security.SSLWriteFunc = SSLWriteFunc - Security.SSLContextRef = SSLContextRef - Security.SSLProtocol = SSLProtocol - Security.SSLCipherSuite = SSLCipherSuite - Security.SecIdentityRef = SecIdentityRef - Security.SecKeychainRef = SecKeychainRef - Security.SecTrustRef = SecTrustRef - Security.SecTrustResultType = SecTrustResultType - Security.SecExternalFormat = SecExternalFormat - Security.OSStatus = OSStatus - - Security.kSecImportExportPassphrase = CFStringRef.in_dll( - Security, "kSecImportExportPassphrase" - ) - Security.kSecImportItemIdentity = CFStringRef.in_dll( - Security, "kSecImportItemIdentity" - ) - - # CoreFoundation time! - CoreFoundation.CFRetain.argtypes = [CFTypeRef] - CoreFoundation.CFRetain.restype = CFTypeRef - - CoreFoundation.CFRelease.argtypes = [CFTypeRef] - CoreFoundation.CFRelease.restype = None - - CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef] - CoreFoundation.CFGetTypeID.restype = CFTypeID - - CoreFoundation.CFStringCreateWithCString.argtypes = [ - CFAllocatorRef, - c_char_p, - CFStringEncoding, - ] - CoreFoundation.CFStringCreateWithCString.restype = CFStringRef - - CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding] - CoreFoundation.CFStringGetCStringPtr.restype = c_char_p - - CoreFoundation.CFStringGetCString.argtypes = [ - CFStringRef, - c_char_p, - CFIndex, - CFStringEncoding, - ] - CoreFoundation.CFStringGetCString.restype = c_bool - - CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex] - CoreFoundation.CFDataCreate.restype = CFDataRef - - CoreFoundation.CFDataGetLength.argtypes = [CFDataRef] - CoreFoundation.CFDataGetLength.restype = CFIndex - - CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef] - CoreFoundation.CFDataGetBytePtr.restype = c_void_p - - CoreFoundation.CFDictionaryCreate.argtypes = [ - CFAllocatorRef, - POINTER(CFTypeRef), - POINTER(CFTypeRef), - CFIndex, - CFDictionaryKeyCallBacks, - CFDictionaryValueCallBacks, - ] - CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef - - CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef] - CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef - - CoreFoundation.CFArrayCreate.argtypes = [ - CFAllocatorRef, - POINTER(CFTypeRef), - CFIndex, - CFArrayCallBacks, - ] - CoreFoundation.CFArrayCreate.restype = CFArrayRef - - CoreFoundation.CFArrayCreateMutable.argtypes = [ - CFAllocatorRef, - CFIndex, - CFArrayCallBacks, - ] - CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef - - CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p] - CoreFoundation.CFArrayAppendValue.restype = None - - CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef] - CoreFoundation.CFArrayGetCount.restype = CFIndex - - CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex] - CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p - - CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( - CoreFoundation, "kCFAllocatorDefault" - ) - CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll( - CoreFoundation, "kCFTypeArrayCallBacks" - ) - CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll( - CoreFoundation, "kCFTypeDictionaryKeyCallBacks" - ) - CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll( - CoreFoundation, "kCFTypeDictionaryValueCallBacks" - ) - - CoreFoundation.CFTypeRef = CFTypeRef - CoreFoundation.CFArrayRef = CFArrayRef - CoreFoundation.CFStringRef = CFStringRef - CoreFoundation.CFDictionaryRef = CFDictionaryRef - -except (AttributeError): - raise ImportError("Error initializing ctypes") - - -class CFConst(object): - """ - A class object that acts as essentially a namespace for CoreFoundation - constants. - """ - - kCFStringEncodingUTF8 = CFStringEncoding(0x08000100) - - -class SecurityConst(object): - """ - A class object that acts as essentially a namespace for Security constants. - """ - - kSSLSessionOptionBreakOnServerAuth = 0 - - kSSLProtocol2 = 1 - kSSLProtocol3 = 2 - kTLSProtocol1 = 4 - kTLSProtocol11 = 7 - kTLSProtocol12 = 8 - # SecureTransport does not support TLS 1.3 even if there's a constant for it - kTLSProtocol13 = 10 - kTLSProtocolMaxSupported = 999 - - kSSLClientSide = 1 - kSSLStreamType = 0 - - kSecFormatPEMSequence = 10 - - kSecTrustResultInvalid = 0 - kSecTrustResultProceed = 1 - # This gap is present on purpose: this was kSecTrustResultConfirm, which - # is deprecated. - kSecTrustResultDeny = 3 - kSecTrustResultUnspecified = 4 - kSecTrustResultRecoverableTrustFailure = 5 - kSecTrustResultFatalTrustFailure = 6 - kSecTrustResultOtherError = 7 - - errSSLProtocol = -9800 - errSSLWouldBlock = -9803 - errSSLClosedGraceful = -9805 - errSSLClosedNoNotify = -9816 - errSSLClosedAbort = -9806 - - errSSLXCertChainInvalid = -9807 - errSSLCrypto = -9809 - errSSLInternal = -9810 - errSSLCertExpired = -9814 - errSSLCertNotYetValid = -9815 - errSSLUnknownRootCert = -9812 - errSSLNoRootCert = -9813 - errSSLHostNameMismatch = -9843 - errSSLPeerHandshakeFail = -9824 - errSSLPeerUserCancelled = -9839 - errSSLWeakPeerEphemeralDHKey = -9850 - errSSLServerAuthCompleted = -9841 - errSSLRecordOverflow = -9847 - - errSecVerifyFailed = -67808 - errSecNoTrustSettings = -25263 - errSecItemNotFound = -25300 - errSecInvalidTrustSettings = -25262 - - # Cipher suites. We only pick the ones our default cipher string allows. - # Source: https://developer.apple.com/documentation/security/1550981-ssl_cipher_suite_values - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA9 - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA8 - TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F - TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 - TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B - TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 - TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 - TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 - TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D - TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C - TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D - TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003C - TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035 - TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F - TLS_AES_128_GCM_SHA256 = 0x1301 - TLS_AES_256_GCM_SHA384 = 0x1302 - TLS_AES_128_CCM_8_SHA256 = 0x1305 - TLS_AES_128_CCM_SHA256 = 0x1304 diff --git a/src/urllib3/contrib/_securetransport/low_level.py b/src/urllib3/contrib/_securetransport/low_level.py deleted file mode 100644 index fa0b245..0000000 --- a/src/urllib3/contrib/_securetransport/low_level.py +++ /dev/null @@ -1,397 +0,0 @@ -""" -Low-level helpers for the SecureTransport bindings. - -These are Python functions that are not directly related to the high-level APIs -but are necessary to get them to work. They include a whole bunch of low-level -CoreFoundation messing about and memory management. The concerns in this module -are almost entirely about trying to avoid memory leaks and providing -appropriate and useful assistance to the higher-level code. -""" -import base64 -import ctypes -import itertools -import os -import re -import ssl -import struct -import tempfile - -from .bindings import CFConst, CoreFoundation, Security - -# This regular expression is used to grab PEM data out of a PEM bundle. -_PEM_CERTS_RE = re.compile( - b"-----BEGIN CERTIFICATE-----\n(.*?)\n-----END CERTIFICATE-----", re.DOTALL -) - - -def _cf_data_from_bytes(bytestring): - """ - Given a bytestring, create a CFData object from it. This CFData object must - be CFReleased by the caller. - """ - return CoreFoundation.CFDataCreate( - CoreFoundation.kCFAllocatorDefault, bytestring, len(bytestring) - ) - - -def _cf_dictionary_from_tuples(tuples): - """ - Given a list of Python tuples, create an associated CFDictionary. - """ - dictionary_size = len(tuples) - - # We need to get the dictionary keys and values out in the same order. - keys = (t[0] for t in tuples) - values = (t[1] for t in tuples) - cf_keys = (CoreFoundation.CFTypeRef * dictionary_size)(*keys) - cf_values = (CoreFoundation.CFTypeRef * dictionary_size)(*values) - - return CoreFoundation.CFDictionaryCreate( - CoreFoundation.kCFAllocatorDefault, - cf_keys, - cf_values, - dictionary_size, - CoreFoundation.kCFTypeDictionaryKeyCallBacks, - CoreFoundation.kCFTypeDictionaryValueCallBacks, - ) - - -def _cfstr(py_bstr): - """ - Given a Python binary data, create a CFString. - The string must be CFReleased by the caller. - """ - c_str = ctypes.c_char_p(py_bstr) - cf_str = CoreFoundation.CFStringCreateWithCString( - CoreFoundation.kCFAllocatorDefault, - c_str, - CFConst.kCFStringEncodingUTF8, - ) - return cf_str - - -def _create_cfstring_array(lst): - """ - Given a list of Python binary data, create an associated CFMutableArray. - The array must be CFReleased by the caller. - - Raises an ssl.SSLError on failure. - """ - cf_arr = None - try: - cf_arr = CoreFoundation.CFArrayCreateMutable( - CoreFoundation.kCFAllocatorDefault, - 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), - ) - if not cf_arr: - raise MemoryError("Unable to allocate memory!") - for item in lst: - cf_str = _cfstr(item) - if not cf_str: - raise MemoryError("Unable to allocate memory!") - try: - CoreFoundation.CFArrayAppendValue(cf_arr, cf_str) - finally: - CoreFoundation.CFRelease(cf_str) - except BaseException as e: - if cf_arr: - CoreFoundation.CFRelease(cf_arr) - raise ssl.SSLError("Unable to allocate array: %s" % (e,)) - return cf_arr - - -def _cf_string_to_unicode(value): - """ - Creates a Unicode string from a CFString object. Used entirely for error - reporting. - - Yes, it annoys me quite a lot that this function is this complex. - """ - value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p)) - - string = CoreFoundation.CFStringGetCStringPtr( - value_as_void_p, CFConst.kCFStringEncodingUTF8 - ) - if string is None: - buffer = ctypes.create_string_buffer(1024) - result = CoreFoundation.CFStringGetCString( - value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8 - ) - if not result: - raise OSError("Error copying C string from CFStringRef") - string = buffer.value - if string is not None: - string = string.decode("utf-8") - return string - - -def _assert_no_error(error, exception_class=None): - """ - Checks the return code and throws an exception if there is an error to - report - """ - if error == 0: - return - - cf_error_string = Security.SecCopyErrorMessageString(error, None) - output = _cf_string_to_unicode(cf_error_string) - CoreFoundation.CFRelease(cf_error_string) - - if output is None or output == u"": - output = u"OSStatus %s" % error - - if exception_class is None: - exception_class = ssl.SSLError - - raise exception_class(output) - - -def _cert_array_from_pem(pem_bundle): - """ - Given a bundle of certs in PEM format, turns them into a CFArray of certs - that can be used to validate a cert chain. - """ - # Normalize the PEM bundle's line endings. - pem_bundle = pem_bundle.replace(b"\r\n", b"\n") - - der_certs = [ - base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle) - ] - if not der_certs: - raise ssl.SSLError("No root certificates specified") - - cert_array = CoreFoundation.CFArrayCreateMutable( - CoreFoundation.kCFAllocatorDefault, - 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), - ) - if not cert_array: - raise ssl.SSLError("Unable to allocate memory!") - - try: - for der_bytes in der_certs: - certdata = _cf_data_from_bytes(der_bytes) - if not certdata: - raise ssl.SSLError("Unable to allocate memory!") - cert = Security.SecCertificateCreateWithData( - CoreFoundation.kCFAllocatorDefault, certdata - ) - CoreFoundation.CFRelease(certdata) - if not cert: - raise ssl.SSLError("Unable to build cert object!") - - CoreFoundation.CFArrayAppendValue(cert_array, cert) - CoreFoundation.CFRelease(cert) - except Exception: - # We need to free the array before the exception bubbles further. - # We only want to do that if an error occurs: otherwise, the caller - # should free. - CoreFoundation.CFRelease(cert_array) - raise - - return cert_array - - -def _is_cert(item): - """ - Returns True if a given CFTypeRef is a certificate. - """ - expected = Security.SecCertificateGetTypeID() - return CoreFoundation.CFGetTypeID(item) == expected - - -def _is_identity(item): - """ - Returns True if a given CFTypeRef is an identity. - """ - expected = Security.SecIdentityGetTypeID() - return CoreFoundation.CFGetTypeID(item) == expected - - -def _temporary_keychain(): - """ - This function creates a temporary Mac keychain that we can use to work with - credentials. This keychain uses a one-time password and a temporary file to - store the data. We expect to have one keychain per socket. The returned - SecKeychainRef must be freed by the caller, including calling - SecKeychainDelete. - - Returns a tuple of the SecKeychainRef and the path to the temporary - directory that contains it. - """ - # Unfortunately, SecKeychainCreate requires a path to a keychain. This - # means we cannot use mkstemp to use a generic temporary file. Instead, - # we're going to create a temporary directory and a filename to use there. - # This filename will be 8 random bytes expanded into base64. We also need - # some random bytes to password-protect the keychain we're creating, so we - # ask for 40 random bytes. - random_bytes = os.urandom(40) - filename = base64.b16encode(random_bytes[:8]).decode("utf-8") - password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 - tempdirectory = tempfile.mkdtemp() - - keychain_path = os.path.join(tempdirectory, filename).encode("utf-8") - - # We now want to create the keychain itself. - keychain = Security.SecKeychainRef() - status = Security.SecKeychainCreate( - keychain_path, len(password), password, False, None, ctypes.byref(keychain) - ) - _assert_no_error(status) - - # Having created the keychain, we want to pass it off to the caller. - return keychain, tempdirectory - - -def _load_items_from_file(keychain, path): - """ - Given a single file, loads all the trust objects from it into arrays and - the keychain. - Returns a tuple of lists: the first list is a list of identities, the - second a list of certs. - """ - certificates = [] - identities = [] - result_array = None - - with open(path, "rb") as f: - raw_filedata = f.read() - - try: - filedata = CoreFoundation.CFDataCreate( - CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata) - ) - result_array = CoreFoundation.CFArrayRef() - result = Security.SecItemImport( - filedata, # cert data - None, # Filename, leaving it out for now - None, # What the type of the file is, we don't care - None, # what's in the file, we don't care - 0, # import flags - None, # key params, can include passphrase in the future - keychain, # The keychain to insert into - ctypes.byref(result_array), # Results - ) - _assert_no_error(result) - - # A CFArray is not very useful to us as an intermediary - # representation, so we are going to extract the objects we want - # and then free the array. We don't need to keep hold of keys: the - # keychain already has them! - result_count = CoreFoundation.CFArrayGetCount(result_array) - for index in range(result_count): - item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index) - item = ctypes.cast(item, CoreFoundation.CFTypeRef) - - if _is_cert(item): - CoreFoundation.CFRetain(item) - certificates.append(item) - elif _is_identity(item): - CoreFoundation.CFRetain(item) - identities.append(item) - finally: - if result_array: - CoreFoundation.CFRelease(result_array) - - CoreFoundation.CFRelease(filedata) - - return (identities, certificates) - - -def _load_client_cert_chain(keychain, *paths): - """ - Load certificates and maybe keys from a number of files. Has the end goal - of returning a CFArray containing one SecIdentityRef, and then zero or more - SecCertificateRef objects, suitable for use as a client certificate trust - chain. - """ - # Ok, the strategy. - # - # This relies on knowing that macOS will not give you a SecIdentityRef - # unless you have imported a key into a keychain. This is a somewhat - # artificial limitation of macOS (for example, it doesn't necessarily - # affect iOS), but there is nothing inside Security.framework that lets you - # get a SecIdentityRef without having a key in a keychain. - # - # So the policy here is we take all the files and iterate them in order. - # Each one will use SecItemImport to have one or more objects loaded from - # it. We will also point at a keychain that macOS can use to work with the - # private key. - # - # Once we have all the objects, we'll check what we actually have. If we - # already have a SecIdentityRef in hand, fab: we'll use that. Otherwise, - # we'll take the first certificate (which we assume to be our leaf) and - # ask the keychain to give us a SecIdentityRef with that cert's associated - # key. - # - # We'll then return a CFArray containing the trust chain: one - # SecIdentityRef and then zero-or-more SecCertificateRef objects. The - # responsibility for freeing this CFArray will be with the caller. This - # CFArray must remain alive for the entire connection, so in practice it - # will be stored with a single SSLSocket, along with the reference to the - # keychain. - certificates = [] - identities = [] - - # Filter out bad paths. - paths = (path for path in paths if path) - - try: - for file_path in paths: - new_identities, new_certs = _load_items_from_file(keychain, file_path) - identities.extend(new_identities) - certificates.extend(new_certs) - - # Ok, we have everything. The question is: do we have an identity? If - # not, we want to grab one from the first cert we have. - if not identities: - new_identity = Security.SecIdentityRef() - status = Security.SecIdentityCreateWithCertificate( - keychain, certificates[0], ctypes.byref(new_identity) - ) - _assert_no_error(status) - identities.append(new_identity) - - # We now want to release the original certificate, as we no longer - # need it. - CoreFoundation.CFRelease(certificates.pop(0)) - - # We now need to build a new CFArray that holds the trust chain. - trust_chain = CoreFoundation.CFArrayCreateMutable( - CoreFoundation.kCFAllocatorDefault, - 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), - ) - for item in itertools.chain(identities, certificates): - # ArrayAppendValue does a CFRetain on the item. That's fine, - # because the finally block will release our other refs to them. - CoreFoundation.CFArrayAppendValue(trust_chain, item) - - return trust_chain - finally: - for obj in itertools.chain(identities, certificates): - CoreFoundation.CFRelease(obj) - - -TLS_PROTOCOL_VERSIONS = { - "SSLv2": (0, 2), - "SSLv3": (3, 0), - "TLSv1": (3, 1), - "TLSv1.1": (3, 2), - "TLSv1.2": (3, 3), -} - - -def _build_tls_unknown_ca_alert(version): - """ - Builds a TLS alert record for an unknown CA. - """ - ver_maj, ver_min = TLS_PROTOCOL_VERSIONS[version] - severity_fatal = 0x02 - description_unknown_ca = 0x30 - msg = struct.pack(">BB", severity_fatal, description_unknown_ca) - msg_len = len(msg) - record_type_alert = 0x15 - record = struct.pack(">BBBH", record_type_alert, ver_maj, ver_min, msg_len) + msg - return record diff --git a/src/urllib3/contrib/appengine.py b/src/urllib3/contrib/appengine.py deleted file mode 100644 index a5a6d91..0000000 --- a/src/urllib3/contrib/appengine.py +++ /dev/null @@ -1,314 +0,0 @@ -""" -This module provides a pool manager that uses Google App Engine's -`URLFetch Service `_. - -Example usage:: - - from urllib3 import PoolManager - from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox - - if is_appengine_sandbox(): - # AppEngineManager uses AppEngine's URLFetch API behind the scenes - http = AppEngineManager() - else: - # PoolManager uses a socket-level API behind the scenes - http = PoolManager() - - r = http.request('GET', 'https://google.com/') - -There are `limitations `_ to the URLFetch service and it may not be -the best choice for your application. There are three options for using -urllib3 on Google App Engine: - -1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is - cost-effective in many circumstances as long as your usage is within the - limitations. -2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. - Sockets also have `limitations and restrictions - `_ and have a lower free quota than URLFetch. - To use sockets, be sure to specify the following in your ``app.yaml``:: - - env_variables: - GAE_USE_SOCKETS_HTTPLIB : 'true' - -3. If you are using `App Engine Flexible -`_, you can use the standard -:class:`PoolManager` without any configuration or special environment variables. -""" - -from __future__ import absolute_import - -import io -import logging -import warnings - -from ..exceptions import ( - HTTPError, - HTTPWarning, - MaxRetryError, - ProtocolError, - SSLError, - TimeoutError, -) -from ..packages.six.moves.urllib.parse import urljoin -from ..request import RequestMethods -from ..response import HTTPResponse -from ..util.retry import Retry -from ..util.timeout import Timeout -from . import _appengine_environ - -try: - from google.appengine.api import urlfetch -except ImportError: - urlfetch = None - - -log = logging.getLogger(__name__) - - -class AppEnginePlatformWarning(HTTPWarning): - pass - - -class AppEnginePlatformError(HTTPError): - pass - - -class AppEngineManager(RequestMethods): - """ - Connection manager for Google App Engine sandbox applications. - - This manager uses the URLFetch service directly instead of using the - emulated httplib, and is subject to URLFetch limitations as described in - the App Engine documentation `here - `_. - - Notably it will raise an :class:`AppEnginePlatformError` if: - * URLFetch is not available. - * If you attempt to use this on App Engine Flexible, as full socket - support is available. - * If a request size is more than 10 megabytes. - * If a response size is more than 32 megabytes. - * If you use an unsupported request method such as OPTIONS. - - Beyond those cases, it will raise normal urllib3 errors. - """ - - def __init__( - self, - headers=None, - retries=None, - validate_certificate=True, - urlfetch_retries=True, - ): - if not urlfetch: - raise AppEnginePlatformError( - "URLFetch is not available in this environment." - ) - - warnings.warn( - "urllib3 is using URLFetch on Google App Engine sandbox instead " - "of sockets. To use sockets directly instead of URLFetch see " - "https://urllib3.readthedocs.io/en/1.26.x/reference/urllib3.contrib.html.", - AppEnginePlatformWarning, - ) - - RequestMethods.__init__(self, headers) - self.validate_certificate = validate_certificate - self.urlfetch_retries = urlfetch_retries - - self.retries = retries or Retry.DEFAULT - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - # Return False to re-raise any potential exceptions - return False - - def urlopen( - self, - method, - url, - body=None, - headers=None, - retries=None, - redirect=True, - timeout=Timeout.DEFAULT_TIMEOUT, - **response_kw - ): - - retries = self._get_retries(retries, redirect) - - try: - follow_redirects = redirect and retries.redirect != 0 and retries.total - response = urlfetch.fetch( - url, - payload=body, - method=method, - headers=headers or {}, - allow_truncated=False, - follow_redirects=self.urlfetch_retries and follow_redirects, - deadline=self._get_absolute_timeout(timeout), - validate_certificate=self.validate_certificate, - ) - except urlfetch.DeadlineExceededError as e: - raise TimeoutError(self, e) - - except urlfetch.InvalidURLError as e: - if "too large" in str(e): - raise AppEnginePlatformError( - "URLFetch request too large, URLFetch only " - "supports requests up to 10mb in size.", - e, - ) - raise ProtocolError(e) - - except urlfetch.DownloadError as e: - if "Too many redirects" in str(e): - raise MaxRetryError(self, url, reason=e) - raise ProtocolError(e) - - except urlfetch.ResponseTooLargeError as e: - raise AppEnginePlatformError( - "URLFetch response too large, URLFetch only supports" - "responses up to 32mb in size.", - e, - ) - - except urlfetch.SSLCertificateError as e: - raise SSLError(e) - - except urlfetch.InvalidMethodError as e: - raise AppEnginePlatformError( - "URLFetch does not support method: %s" % method, e - ) - - http_response = self._urlfetch_response_to_http_response( - response, retries=retries, **response_kw - ) - - # Handle redirect? - redirect_location = redirect and http_response.get_redirect_location() - if redirect_location: - # Check for redirect response - if self.urlfetch_retries and retries.raise_on_redirect: - raise MaxRetryError(self, url, "too many redirects") - else: - if http_response.status == 303: - method = "GET" - - try: - retries = retries.increment( - method, url, response=http_response, _pool=self - ) - except MaxRetryError: - if retries.raise_on_redirect: - raise MaxRetryError(self, url, "too many redirects") - return http_response - - retries.sleep_for_retry(http_response) - log.debug("Redirecting %s -> %s", url, redirect_location) - redirect_url = urljoin(url, redirect_location) - return self.urlopen( - method, - redirect_url, - body, - headers, - retries=retries, - redirect=redirect, - timeout=timeout, - **response_kw - ) - - # Check if we should retry the HTTP response. - has_retry_after = bool(http_response.headers.get("Retry-After")) - if retries.is_retry(method, http_response.status, has_retry_after): - retries = retries.increment(method, url, response=http_response, _pool=self) - log.debug("Retry: %s", url) - retries.sleep(http_response) - return self.urlopen( - method, - url, - body=body, - headers=headers, - retries=retries, - redirect=redirect, - timeout=timeout, - **response_kw - ) - - return http_response - - def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): - - if is_prod_appengine(): - # Production GAE handles deflate encoding automatically, but does - # not remove the encoding header. - content_encoding = urlfetch_resp.headers.get("content-encoding") - - if content_encoding == "deflate": - del urlfetch_resp.headers["content-encoding"] - - transfer_encoding = urlfetch_resp.headers.get("transfer-encoding") - # We have a full response's content, - # so let's make sure we don't report ourselves as chunked data. - if transfer_encoding == "chunked": - encodings = transfer_encoding.split(",") - encodings.remove("chunked") - urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings) - - original_response = HTTPResponse( - # In order for decoding to work, we must present the content as - # a file-like object. - body=io.BytesIO(urlfetch_resp.content), - msg=urlfetch_resp.header_msg, - headers=urlfetch_resp.headers, - status=urlfetch_resp.status_code, - **response_kw - ) - - return HTTPResponse( - body=io.BytesIO(urlfetch_resp.content), - headers=urlfetch_resp.headers, - status=urlfetch_resp.status_code, - original_response=original_response, - **response_kw - ) - - def _get_absolute_timeout(self, timeout): - if timeout is Timeout.DEFAULT_TIMEOUT: - return None # Defer to URLFetch's default. - if isinstance(timeout, Timeout): - if timeout._read is not None or timeout._connect is not None: - warnings.warn( - "URLFetch does not support granular timeout settings, " - "reverting to total or default URLFetch timeout.", - AppEnginePlatformWarning, - ) - return timeout.total - return timeout - - def _get_retries(self, retries, redirect): - if not isinstance(retries, Retry): - retries = Retry.from_int(retries, redirect=redirect, default=self.retries) - - if retries.connect or retries.read or retries.redirect: - warnings.warn( - "URLFetch only supports total retries and does not " - "recognize connect, read, or redirect retry parameters.", - AppEnginePlatformWarning, - ) - - return retries - - -# Alias methods from _appengine_environ to maintain public API interface. - -is_appengine = _appengine_environ.is_appengine -is_appengine_sandbox = _appengine_environ.is_appengine_sandbox -is_local_appengine = _appengine_environ.is_local_appengine -is_prod_appengine = _appengine_environ.is_prod_appengine -is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms diff --git a/src/urllib3/contrib/emscripten/__init__.py b/src/urllib3/contrib/emscripten/__init__.py new file mode 100644 index 0000000..8a3c5be --- /dev/null +++ b/src/urllib3/contrib/emscripten/__init__.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import urllib3.connection + +from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection + + +def inject_into_urllib3() -> None: + # override connection classes to use emscripten specific classes + # n.b. mypy complains about the overriding of classes below + # if it isn't ignored + HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection + HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection + urllib3.connection.HTTPConnection = EmscriptenHTTPConnection # type: ignore[misc,assignment] + urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection # type: ignore[misc,assignment] diff --git a/src/urllib3/contrib/emscripten/connection.py b/src/urllib3/contrib/emscripten/connection.py new file mode 100644 index 0000000..41bfd27 --- /dev/null +++ b/src/urllib3/contrib/emscripten/connection.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import os +import typing + +# use http.client.HTTPException for consistency with non-emscripten +from http.client import HTTPException as HTTPException # noqa: F401 +from http.client import ResponseNotReady + +from ..._base_connection import _TYPE_BODY +from ...connection import HTTPConnection, ProxyConfig, port_by_scheme +from ...exceptions import TimeoutError +from ...response import BaseHTTPResponse +from ...util.connection import _TYPE_SOCKET_OPTIONS +from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT +from ...util.url import Url +from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request +from .request import EmscriptenRequest +from .response import EmscriptenHttpResponseWrapper, EmscriptenResponse + +if typing.TYPE_CHECKING: + from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection + + +class EmscriptenHTTPConnection: + default_port: typing.ClassVar[int] = port_by_scheme["http"] + default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS] + + timeout: None | (float) + + host: str + port: int + blocksize: int + source_address: tuple[str, int] | None + socket_options: _TYPE_SOCKET_OPTIONS | None + + proxy: Url | None + proxy_config: ProxyConfig | None + + is_verified: bool = False + proxy_is_verified: bool | None = None + + _response: EmscriptenResponse | None + + def __init__( + self, + host: str, + port: int = 0, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 8192, + socket_options: _TYPE_SOCKET_OPTIONS | None = None, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + ) -> None: + self.host = host + self.port = port + self.timeout = timeout if isinstance(timeout, float) else 0.0 + self.scheme = "http" + self._closed = True + self._response = None + # ignore these things because we don't + # have control over that stuff + self.proxy = None + self.proxy_config = None + self.blocksize = blocksize + self.source_address = None + self.socket_options = None + self.is_verified = False + + def set_tunnel( + self, + host: str, + port: int | None = 0, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: + pass + + def connect(self) -> None: + pass + + def request( + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + # We know *at least* botocore is depending on the order of the + # first 3 parameters so to be safe we only mark the later ones + # as keyword-only to ensure we have space to extend. + *, + chunked: bool = False, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + ) -> None: + self._closed = False + if url.startswith("/"): + # no scheme / host / port included, make a full url + url = f"{self.scheme}://{self.host}:{self.port}" + url + request = EmscriptenRequest( + url=url, + method=method, + timeout=self.timeout if self.timeout else 0, + decode_content=decode_content, + ) + request.set_body(body) + if headers: + for k, v in headers.items(): + request.set_header(k, v) + self._response = None + try: + if not preload_content: + self._response = send_streaming_request(request) + if self._response is None: + self._response = send_request(request) + except _TimeoutError as e: + raise TimeoutError(e.message) from e + except _RequestError as e: + raise HTTPException(e.message) from e + + def getresponse(self) -> BaseHTTPResponse: + if self._response is not None: + return EmscriptenHttpResponseWrapper( + internal_response=self._response, + url=self._response.request.url, + connection=self, + ) + else: + raise ResponseNotReady() + + def close(self) -> None: + self._closed = True + self._response = None + + @property + def is_closed(self) -> bool: + """Whether the connection either is brand new or has been previously closed. + If this property is True then both ``is_connected`` and ``has_connected_to_proxy`` + properties must be False. + """ + return self._closed + + @property + def is_connected(self) -> bool: + """Whether the connection is actively connected to any origin (proxy or target)""" + return True + + @property + def has_connected_to_proxy(self) -> bool: + """Whether the connection has successfully connected to its proxy. + This returns False if no proxy is in use. Used to determine whether + errors are coming from the proxy layer or from tunnelling to the target origin. + """ + return False + + +class EmscriptenHTTPSConnection(EmscriptenHTTPConnection): + default_port = port_by_scheme["https"] + # all this is basically ignored, as browser handles https + cert_reqs: int | str | None = None + ca_certs: str | None = None + ca_cert_dir: str | None = None + ca_cert_data: None | str | bytes = None + cert_file: str | None + key_file: str | None + key_password: str | None + ssl_context: typing.Any | None + ssl_version: int | str | None = None + ssl_minimum_version: int | None = None + ssl_maximum_version: int | None = None + assert_hostname: None | str | typing.Literal[False] + assert_fingerprint: str | None = None + + def __init__( + self, + host: str, + port: int = 0, + *, + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + blocksize: int = 16384, + socket_options: ( + None | _TYPE_SOCKET_OPTIONS + ) = HTTPConnection.default_socket_options, + proxy: Url | None = None, + proxy_config: ProxyConfig | None = None, + cert_reqs: int | str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + server_hostname: str | None = None, + ssl_context: typing.Any | None = None, + ca_certs: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, + ssl_version: int | str | None = None, # Deprecated + cert_file: str | None = None, + key_file: str | None = None, + key_password: str | None = None, + ) -> None: + super().__init__( + host, + port=port, + timeout=timeout, + source_address=source_address, + blocksize=blocksize, + socket_options=socket_options, + proxy=proxy, + proxy_config=proxy_config, + ) + self.scheme = "https" + + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password + self.ssl_context = ssl_context + self.server_hostname = server_hostname + self.assert_hostname = assert_hostname + self.assert_fingerprint = assert_fingerprint + self.ssl_version = ssl_version + self.ssl_minimum_version = ssl_minimum_version + self.ssl_maximum_version = ssl_maximum_version + self.ca_certs = ca_certs and os.path.expanduser(ca_certs) + self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) + self.ca_cert_data = ca_cert_data + + self.cert_reqs = None + + # The browser will automatically verify all requests. + # We have no control over that setting. + self.is_verified = True + + def set_cert( + self, + key_file: str | None = None, + cert_file: str | None = None, + cert_reqs: int | str | None = None, + key_password: str | None = None, + ca_certs: str | None = None, + assert_hostname: None | str | typing.Literal[False] = None, + assert_fingerprint: str | None = None, + ca_cert_dir: str | None = None, + ca_cert_data: None | str | bytes = None, + ) -> None: + pass + + +# verify that this class implements BaseHTTP(s) connection correctly +if typing.TYPE_CHECKING: + _supports_http_protocol: BaseHTTPConnection = EmscriptenHTTPConnection("", 0) + _supports_https_protocol: BaseHTTPSConnection = EmscriptenHTTPSConnection("", 0) diff --git a/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js b/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js new file mode 100644 index 0000000..243b862 --- /dev/null +++ b/src/urllib3/contrib/emscripten/emscripten_fetch_worker.js @@ -0,0 +1,110 @@ +let Status = { + SUCCESS_HEADER: -1, + SUCCESS_EOF: -2, + ERROR_TIMEOUT: -3, + ERROR_EXCEPTION: -4, +}; + +let connections = {}; +let nextConnectionID = 1; +const encoder = new TextEncoder(); + +self.addEventListener("message", async function (event) { + if (event.data.close) { + let connectionID = event.data.close; + delete connections[connectionID]; + return; + } else if (event.data.getMore) { + let connectionID = event.data.getMore; + let { curOffset, value, reader, intBuffer, byteBuffer } = + connections[connectionID]; + // if we still have some in buffer, then just send it back straight away + if (!value || curOffset >= value.length) { + // read another buffer if required + try { + let readResponse = await reader.read(); + + if (readResponse.done) { + // read everything - clear connection and return + delete connections[connectionID]; + Atomics.store(intBuffer, 0, Status.SUCCESS_EOF); + Atomics.notify(intBuffer, 0); + // finished reading successfully + // return from event handler + return; + } + curOffset = 0; + connections[connectionID].value = readResponse.value; + value = readResponse.value; + } catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } + + // send as much buffer as we can + let curLen = value.length - curOffset; + if (curLen > byteBuffer.length) { + curLen = byteBuffer.length; + } + byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0); + + Atomics.store(intBuffer, 0, curLen); // store current length in bytes + Atomics.notify(intBuffer, 0); + curOffset += curLen; + connections[connectionID].curOffset = curOffset; + + return; + } else { + // start fetch + let connectionID = nextConnectionID; + nextConnectionID += 1; + const intBuffer = new Int32Array(event.data.buffer); + const byteBuffer = new Uint8Array(event.data.buffer, 8); + try { + const response = await fetch(event.data.url, event.data.fetchParams); + // return the headers first via textencoder + var headers = []; + for (const pair of response.headers.entries()) { + headers.push([pair[0], pair[1]]); + } + let headerObj = { + headers: headers, + status: response.status, + connectionID, + }; + const headerText = JSON.stringify(headerObj); + let headerBytes = encoder.encode(headerText); + let written = headerBytes.length; + byteBuffer.set(headerBytes); + intBuffer[1] = written; + // make a connection + connections[connectionID] = { + reader: response.body.getReader(), + intBuffer: intBuffer, + byteBuffer: byteBuffer, + value: undefined, + curOffset: 0, + }; + // set header ready + Atomics.store(intBuffer, 0, Status.SUCCESS_HEADER); + Atomics.notify(intBuffer, 0); + // all fetching after this goes through a new postmessage call with getMore + // this allows for parallel requests + } catch (error) { + console.log("Request exception:", error); + let errorBytes = encoder.encode(error.message); + let written = errorBytes.length; + byteBuffer.set(errorBytes); + intBuffer[1] = written; + Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION); + Atomics.notify(intBuffer, 0); + } + } +}); +self.postMessage({ inited: true }); diff --git a/src/urllib3/contrib/emscripten/fetch.py b/src/urllib3/contrib/emscripten/fetch.py new file mode 100644 index 0000000..a514306 --- /dev/null +++ b/src/urllib3/contrib/emscripten/fetch.py @@ -0,0 +1,708 @@ +""" +Support for streaming http requests in emscripten. + +A few caveats - + +If your browser (or Node.js) has WebAssembly JavaScript Promise Integration enabled +https://github.com/WebAssembly/js-promise-integration/blob/main/proposals/js-promise-integration/Overview.md +*and* you launch pyodide using `pyodide.runPythonAsync`, this will fetch data using the +JavaScript asynchronous fetch api (wrapped via `pyodide.ffi.call_sync`). In this case +timeouts and streaming should just work. + +Otherwise, it uses a combination of XMLHttpRequest and a web-worker for streaming. + +This approach has several caveats: + +Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed. +Streaming only works if you're running pyodide in a web worker. + +Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch +operation, so it requires that you have crossOriginIsolation enabled, by serving over https +(or from localhost) with the two headers below set: + + Cross-Origin-Opener-Policy: same-origin + Cross-Origin-Embedder-Policy: require-corp + +You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in +JavaScript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole +request into a buffer and then returning it. it shows a warning in the JavaScript console in this case. + +Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once +control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch. + +NB: in this code, there are a lot of JavaScript objects. They are named js_* +to make it clear what type of object they are. +""" + +from __future__ import annotations + +import io +import json +from email.parser import Parser +from importlib.resources import files +from typing import TYPE_CHECKING, Any + +import js # type: ignore[import-not-found] +from pyodide.ffi import ( # type: ignore[import-not-found] + JsArray, + JsException, + JsProxy, + to_js, +) + +if TYPE_CHECKING: + from typing_extensions import Buffer + +from .request import EmscriptenRequest +from .response import EmscriptenResponse + +""" +There are some headers that trigger unintended CORS preflight requests. +See also https://github.com/koenvo/pyodide-http/issues/22 +""" +HEADERS_TO_IGNORE = ("user-agent",) + +SUCCESS_HEADER = -1 +SUCCESS_EOF = -2 +ERROR_TIMEOUT = -3 +ERROR_EXCEPTION = -4 + +_STREAMING_WORKER_CODE = ( + files(__package__) + .joinpath("emscripten_fetch_worker.js") + .read_text(encoding="utf-8") +) + + +class _RequestError(Exception): + def __init__( + self, + message: str | None = None, + *, + request: EmscriptenRequest | None = None, + response: EmscriptenResponse | None = None, + ): + self.request = request + self.response = response + self.message = message + super().__init__(self.message) + + +class _StreamingError(_RequestError): + pass + + +class _TimeoutError(_RequestError): + pass + + +def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy: + return to_js(dict_val, dict_converter=js.Object.fromEntries) + + +class _ReadStream(io.RawIOBase): + def __init__( + self, + int_buffer: JsArray, + byte_buffer: JsArray, + timeout: float, + worker: JsProxy, + connection_id: int, + request: EmscriptenRequest, + ): + self.int_buffer = int_buffer + self.byte_buffer = byte_buffer + self.read_pos = 0 + self.read_len = 0 + self.connection_id = connection_id + self.worker = worker + self.timeout = int(1000 * timeout) if timeout > 0 else None + self.is_live = True + self._is_closed = False + self.request: EmscriptenRequest | None = request + + def __del__(self) -> None: + self.close() + + # this is compatible with _base_connection + def is_closed(self) -> bool: + return self._is_closed + + # for compatibility with RawIOBase + @property + def closed(self) -> bool: + return self.is_closed() + + def close(self) -> None: + if self.is_closed(): + return + self.read_len = 0 + self.read_pos = 0 + self.int_buffer = None + self.byte_buffer = None + self._is_closed = True + self.request = None + if self.is_live: + self.worker.postMessage(_obj_from_dict({"close": self.connection_id})) + self.is_live = False + super().close() + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return False + + def seekable(self) -> bool: + return False + + def readinto(self, byte_obj: Buffer) -> int: + if not self.int_buffer: + raise _StreamingError( + "No buffer for stream in _ReadStream.readinto", + request=self.request, + response=None, + ) + if self.read_len == 0: + # wait for the worker to send something + js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT) + self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id})) + if ( + js.Atomics.wait(self.int_buffer, 0, ERROR_TIMEOUT, self.timeout) + == "timed-out" + ): + raise _TimeoutError + data_len = self.int_buffer[0] + if data_len > 0: + self.read_len = data_len + self.read_pos = 0 + elif data_len == ERROR_EXCEPTION: + string_len = self.int_buffer[1] + # decode the error string + js_decoder = js.TextDecoder.new() + json_str = js_decoder.decode(self.byte_buffer.slice(0, string_len)) + raise _StreamingError( + f"Exception thrown in fetch: {json_str}", + request=self.request, + response=None, + ) + else: + # EOF, free the buffers and return zero + # and free the request + self.is_live = False + self.close() + return 0 + # copy from int32array to python bytes + ret_length = min(self.read_len, len(memoryview(byte_obj))) + subarray = self.byte_buffer.subarray( + self.read_pos, self.read_pos + ret_length + ).to_py() + memoryview(byte_obj)[0:ret_length] = subarray + self.read_len -= ret_length + self.read_pos += ret_length + return ret_length + + +class _StreamingFetcher: + def __init__(self) -> None: + # make web-worker and data buffer on startup + self.streaming_ready = False + + js_data_blob = js.Blob.new( + to_js([_STREAMING_WORKER_CODE], create_pyproxies=False), + _obj_from_dict({"type": "application/javascript"}), + ) + + def promise_resolver(js_resolve_fn: JsProxy, js_reject_fn: JsProxy) -> None: + def onMsg(e: JsProxy) -> None: + self.streaming_ready = True + js_resolve_fn(e) + + def onErr(e: JsProxy) -> None: + js_reject_fn(e) # Defensive: never happens in ci + + self.js_worker.onmessage = onMsg + self.js_worker.onerror = onErr + + js_data_url = js.URL.createObjectURL(js_data_blob) + self.js_worker = js.globalThis.Worker.new(js_data_url) + self.js_worker_ready_promise = js.globalThis.Promise.new(promise_resolver) + + def send(self, request: EmscriptenRequest) -> EmscriptenResponse: + headers = { + k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE + } + + body = request.body + fetch_data = {"headers": headers, "body": to_js(body), "method": request.method} + # start the request off in the worker + timeout = int(1000 * request.timeout) if request.timeout > 0 else None + js_shared_buffer = js.SharedArrayBuffer.new(1048576) + js_int_buffer = js.Int32Array.new(js_shared_buffer) + js_byte_buffer = js.Uint8Array.new(js_shared_buffer, 8) + + js.Atomics.store(js_int_buffer, 0, ERROR_TIMEOUT) + js.Atomics.notify(js_int_buffer, 0) + js_absolute_url = js.URL.new(request.url, js.location).href + self.js_worker.postMessage( + _obj_from_dict( + { + "buffer": js_shared_buffer, + "url": js_absolute_url, + "fetchParams": fetch_data, + } + ) + ) + # wait for the worker to send something + js.Atomics.wait(js_int_buffer, 0, ERROR_TIMEOUT, timeout) + if js_int_buffer[0] == ERROR_TIMEOUT: + raise _TimeoutError( + "Timeout connecting to streaming request", + request=request, + response=None, + ) + elif js_int_buffer[0] == SUCCESS_HEADER: + # got response + # header length is in second int of intBuffer + string_len = js_int_buffer[1] + # decode the rest to a JSON string + js_decoder = js.TextDecoder.new() + # this does a copy (the slice) because decode can't work on shared array + # for some silly reason + json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len)) + # get it as an object + response_obj = json.loads(json_str) + return EmscriptenResponse( + request=request, + status_code=response_obj["status"], + headers=response_obj["headers"], + body=_ReadStream( + js_int_buffer, + js_byte_buffer, + request.timeout, + self.js_worker, + response_obj["connectionID"], + request, + ), + ) + elif js_int_buffer[0] == ERROR_EXCEPTION: + string_len = js_int_buffer[1] + # decode the error string + js_decoder = js.TextDecoder.new() + json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len)) + raise _StreamingError( + f"Exception thrown in fetch: {json_str}", request=request, response=None + ) + else: + raise _StreamingError( + f"Unknown status from worker in fetch: {js_int_buffer[0]}", + request=request, + response=None, + ) + + +class _JSPIReadStream(io.RawIOBase): + """ + A read stream that uses pyodide.ffi.run_sync to read from a JavaScript fetch + response. This requires support for WebAssembly JavaScript Promise Integration + in the containing browser, and for pyodide to be launched via runPythonAsync. + + :param js_read_stream: + The JavaScript stream reader + + :param timeout: + Timeout in seconds + + :param request: + The request we're handling + + :param response: + The response this stream relates to + + :param js_abort_controller: + A JavaScript AbortController object, used for timeouts + """ + + def __init__( + self, + js_read_stream: Any, + timeout: float, + request: EmscriptenRequest, + response: EmscriptenResponse, + js_abort_controller: Any, # JavaScript AbortController for timeouts + ): + self.js_read_stream = js_read_stream + self.timeout = timeout + self._is_closed = False + self._is_done = False + self.request: EmscriptenRequest | None = request + self.response: EmscriptenResponse | None = response + self.current_buffer = None + self.current_buffer_pos = 0 + self.js_abort_controller = js_abort_controller + + def __del__(self) -> None: + self.close() + + # this is compatible with _base_connection + def is_closed(self) -> bool: + return self._is_closed + + # for compatibility with RawIOBase + @property + def closed(self) -> bool: + return self.is_closed() + + def close(self) -> None: + if self.is_closed(): + return + self.read_len = 0 + self.read_pos = 0 + self.js_read_stream.cancel() + self.js_read_stream = None + self._is_closed = True + self._is_done = True + self.request = None + self.response = None + super().close() + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return False + + def seekable(self) -> bool: + return False + + def _get_next_buffer(self) -> bool: + result_js = _run_sync_with_timeout( + self.js_read_stream.read(), + self.timeout, + self.js_abort_controller, + request=self.request, + response=self.response, + ) + if result_js.done: + self._is_done = True + return False + else: + self.current_buffer = result_js.value.to_py() + self.current_buffer_pos = 0 + return True + + def readinto(self, byte_obj: Buffer) -> int: + if self.current_buffer is None: + if not self._get_next_buffer() or self.current_buffer is None: + self.close() + return 0 + ret_length = min( + len(byte_obj), len(self.current_buffer) - self.current_buffer_pos + ) + byte_obj[0:ret_length] = self.current_buffer[ + self.current_buffer_pos : self.current_buffer_pos + ret_length + ] + self.current_buffer_pos += ret_length + if self.current_buffer_pos == len(self.current_buffer): + self.current_buffer = None + return ret_length + + +# check if we are in a worker or not +def is_in_browser_main_thread() -> bool: + return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window + + +def is_cross_origin_isolated() -> bool: + return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated + + +def is_in_node() -> bool: + return ( + hasattr(js, "process") + and hasattr(js.process, "release") + and hasattr(js.process.release, "name") + and js.process.release.name == "node" + ) + + +def is_worker_available() -> bool: + return hasattr(js, "Worker") and hasattr(js, "Blob") + + +_fetcher: _StreamingFetcher | None = None + +if is_worker_available() and ( + (is_cross_origin_isolated() and not is_in_browser_main_thread()) + and (not is_in_node()) +): + _fetcher = _StreamingFetcher() +else: + _fetcher = None + + +NODE_JSPI_ERROR = ( + "urllib3 only works in Node.js with pyodide.runPythonAsync" + " and requires the flag --experimental-wasm-stack-switching in " + " versions of node <24." +) + + +def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | None: + if has_jspi(): + return send_jspi_request(request, True) + elif is_in_node(): + raise _RequestError( + message=NODE_JSPI_ERROR, + request=request, + response=None, + ) + + if _fetcher and streaming_ready(): + return _fetcher.send(request) + else: + _show_streaming_warning() + return None + + +_SHOWN_TIMEOUT_WARNING = False + + +def _show_timeout_warning() -> None: + global _SHOWN_TIMEOUT_WARNING + if not _SHOWN_TIMEOUT_WARNING: + _SHOWN_TIMEOUT_WARNING = True + message = "Warning: Timeout is not available on main browser thread" + js.console.warn(message) + + +_SHOWN_STREAMING_WARNING = False + + +def _show_streaming_warning() -> None: + global _SHOWN_STREAMING_WARNING + if not _SHOWN_STREAMING_WARNING: + _SHOWN_STREAMING_WARNING = True + message = "Can't stream HTTP requests because: \n" + if not is_cross_origin_isolated(): + message += " Page is not cross-origin isolated\n" + if is_in_browser_main_thread(): + message += " Python is running in main browser thread\n" + if not is_worker_available(): + message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in + if streaming_ready() is False: + message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch +is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`""" + from js import console + + console.warn(message) + + +def send_request(request: EmscriptenRequest) -> EmscriptenResponse: + if has_jspi(): + return send_jspi_request(request, False) + elif is_in_node(): + raise _RequestError( + message=NODE_JSPI_ERROR, + request=request, + response=None, + ) + try: + js_xhr = js.XMLHttpRequest.new() + + if not is_in_browser_main_thread(): + js_xhr.responseType = "arraybuffer" + if request.timeout: + js_xhr.timeout = int(request.timeout * 1000) + else: + js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15") + if request.timeout: + # timeout isn't available on the main thread - show a warning in console + # if it is set + _show_timeout_warning() + + js_xhr.open(request.method, request.url, False) + for name, value in request.headers.items(): + if name.lower() not in HEADERS_TO_IGNORE: + js_xhr.setRequestHeader(name, value) + + js_xhr.send(to_js(request.body)) + + headers = dict(Parser().parsestr(js_xhr.getAllResponseHeaders())) + + if not is_in_browser_main_thread(): + body = js_xhr.response.to_py().tobytes() + else: + body = js_xhr.response.encode("ISO-8859-15") + return EmscriptenResponse( + status_code=js_xhr.status, headers=headers, body=body, request=request + ) + except JsException as err: + if err.name == "TimeoutError": + raise _TimeoutError(err.message, request=request) + elif err.name == "NetworkError": + raise _RequestError(err.message, request=request) + else: + # general http error + raise _RequestError(err.message, request=request) + + +def send_jspi_request( + request: EmscriptenRequest, streaming: bool +) -> EmscriptenResponse: + """ + Send a request using WebAssembly JavaScript Promise Integration + to wrap the asynchronous JavaScript fetch api (experimental). + + :param request: + Request to send + + :param streaming: + Whether to stream the response + + :return: The response object + :rtype: EmscriptenResponse + """ + timeout = request.timeout + js_abort_controller = js.AbortController.new() + headers = {k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE} + req_body = request.body + fetch_data = { + "headers": headers, + "body": to_js(req_body), + "method": request.method, + "signal": js_abort_controller.signal, + } + # Call JavaScript fetch (async api, returns a promise) + fetcher_promise_js = js.fetch(request.url, _obj_from_dict(fetch_data)) + # Now suspend WebAssembly until we resolve that promise + # or time out. + response_js = _run_sync_with_timeout( + fetcher_promise_js, + timeout, + js_abort_controller, + request=request, + response=None, + ) + headers = {} + header_iter = response_js.headers.entries() + while True: + iter_value_js = header_iter.next() + if getattr(iter_value_js, "done", False): + break + else: + headers[str(iter_value_js.value[0])] = str(iter_value_js.value[1]) + status_code = response_js.status + body: bytes | io.RawIOBase = b"" + + response = EmscriptenResponse( + status_code=status_code, headers=headers, body=b"", request=request + ) + if streaming: + # get via inputstream + if response_js.body is not None: + # get a reader from the fetch response + body_stream_js = response_js.body.getReader() + body = _JSPIReadStream( + body_stream_js, timeout, request, response, js_abort_controller + ) + else: + # get directly via arraybuffer + # n.b. this is another async JavaScript call. + body = _run_sync_with_timeout( + response_js.arrayBuffer(), + timeout, + js_abort_controller, + request=request, + response=response, + ).to_py() + response.body = body + return response + + +def _run_sync_with_timeout( + promise: Any, + timeout: float, + js_abort_controller: Any, + request: EmscriptenRequest | None, + response: EmscriptenResponse | None, +) -> Any: + """ + Await a JavaScript promise synchronously with a timeout which is implemented + via the AbortController + + :param promise: + Javascript promise to await + + :param timeout: + Timeout in seconds + + :param js_abort_controller: + A JavaScript AbortController object, used on timeout + + :param request: + The request being handled + + :param response: + The response being handled (if it exists yet) + + :raises _TimeoutError: If the request times out + :raises _RequestError: If the request raises a JavaScript exception + + :return: The result of awaiting the promise. + """ + timer_id = None + if timeout > 0: + timer_id = js.setTimeout( + js_abort_controller.abort.bind(js_abort_controller), int(timeout * 1000) + ) + try: + from pyodide.ffi import run_sync + + # run_sync here uses WebAssembly JavaScript Promise Integration to + # suspend python until the JavaScript promise resolves. + return run_sync(promise) + except JsException as err: + if err.name == "AbortError": + raise _TimeoutError( + message="Request timed out", request=request, response=response + ) + else: + raise _RequestError(message=err.message, request=request, response=response) + finally: + if timer_id is not None: + js.clearTimeout(timer_id) + + +def has_jspi() -> bool: + """ + Return true if jspi can be used. + + This requires both browser support and also WebAssembly + to be in the correct state - i.e. that the javascript + call into python was async not sync. + + :return: True if jspi can be used. + :rtype: bool + """ + try: + from pyodide.ffi import can_run_sync, run_sync # noqa: F401 + + return bool(can_run_sync()) + except ImportError: + return False + + +def streaming_ready() -> bool | None: + if _fetcher: + return _fetcher.streaming_ready + else: + return None # no fetcher, return None to signify that + + +async def wait_for_streaming_ready() -> bool: + if _fetcher: + await _fetcher.js_worker_ready_promise + return True + else: + return False diff --git a/src/urllib3/contrib/emscripten/request.py b/src/urllib3/contrib/emscripten/request.py new file mode 100644 index 0000000..e692e69 --- /dev/null +++ b/src/urllib3/contrib/emscripten/request.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..._base_connection import _TYPE_BODY + + +@dataclass +class EmscriptenRequest: + method: str + url: str + params: dict[str, str] | None = None + body: _TYPE_BODY | None = None + headers: dict[str, str] = field(default_factory=dict) + timeout: float = 0 + decode_content: bool = True + + def set_header(self, name: str, value: str) -> None: + self.headers[name.capitalize()] = value + + def set_body(self, body: _TYPE_BODY | None) -> None: + self.body = body diff --git a/src/urllib3/contrib/emscripten/response.py b/src/urllib3/contrib/emscripten/response.py new file mode 100644 index 0000000..b32b402 --- /dev/null +++ b/src/urllib3/contrib/emscripten/response.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import json as _json +import logging +import typing +from contextlib import contextmanager +from dataclasses import dataclass +from http.client import HTTPException as HTTPException +from io import BytesIO, IOBase + +from ...exceptions import InvalidHeader, TimeoutError +from ...response import BaseHTTPResponse +from ...util.retry import Retry +from .request import EmscriptenRequest + +if typing.TYPE_CHECKING: + from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection + +log = logging.getLogger(__name__) + + +@dataclass +class EmscriptenResponse: + status_code: int + headers: dict[str, str] + body: IOBase | bytes + request: EmscriptenRequest + + +class EmscriptenHttpResponseWrapper(BaseHTTPResponse): + def __init__( + self, + internal_response: EmscriptenResponse, + url: str | None = None, + connection: BaseHTTPConnection | BaseHTTPSConnection | None = None, + ): + self._pool = None # set by pool class + self._body = None + self._response = internal_response + self._url = url + self._connection = connection + self._closed = False + super().__init__( + headers=internal_response.headers, + status=internal_response.status_code, + request_url=url, + version=0, + version_string="HTTP/?", + reason="", + decode_content=True, + ) + self.length_remaining = self._init_length(self._response.request.method) + self.length_is_certain = False + + @property + def url(self) -> str | None: + return self._url + + @url.setter + def url(self, url: str | None) -> None: + self._url = url + + @property + def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None: + return self._connection + + @property + def retries(self) -> Retry | None: + return self._retries + + @retries.setter + def retries(self, retries: Retry | None) -> None: + # Override the request_url if retries has a redirect location. + self._retries = retries + + def stream( + self, amt: int | None = 2**16, decode_content: bool | None = None + ) -> typing.Generator[bytes]: + """ + A generator wrapper for the read() method. A call will block until + ``amt`` bytes have been read from the connection or until the + connection is closed. + + :param amt: + How much of the content to read. The generator will return up to + much data per iteration, but may return less. This is particularly + likely when using compressed data. However, the empty string will + never be returned. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + while True: + data = self.read(amt=amt, decode_content=decode_content) + + if data: + yield data + else: + break + + def _init_length(self, request_method: str | None) -> int | None: + length: int | None + content_length: str | None = self.headers.get("content-length") + + if content_length is not None: + try: + # RFC 7230 section 3.3.2 specifies multiple content lengths can + # be sent in a single Content-Length header + # (e.g. Content-Length: 42, 42). This line ensures the values + # are all valid ints and that as long as the `set` length is 1, + # all values are the same. Otherwise, the header is invalid. + lengths = {int(val) for val in content_length.split(",")} + if len(lengths) > 1: + raise InvalidHeader( + "Content-Length contained multiple " + "unmatching values (%s)" % content_length + ) + length = lengths.pop() + except ValueError: + length = None + else: + if length < 0: + length = None + + else: # if content_length is None + length = None + + # Check for responses that shouldn't include a body + if ( + self.status in (204, 304) + or 100 <= self.status < 200 + or request_method == "HEAD" + ): + length = 0 + + return length + + def read( + self, + amt: int | None = None, + decode_content: bool | None = None, # ignored because browser decodes always + cache_content: bool = False, + ) -> bytes: + if ( + self._closed + or self._response is None + or (isinstance(self._response.body, IOBase) and self._response.body.closed) + ): + return b"" + + with self._error_catcher(): + # body has been preloaded as a string by XmlHttpRequest + if not isinstance(self._response.body, IOBase): + self.length_remaining = len(self._response.body) + self.length_is_certain = True + # wrap body in IOStream + self._response.body = BytesIO(self._response.body) + if amt is not None and amt >= 0: + # don't cache partial content + cache_content = False + data = self._response.body.read(amt) + if self.length_remaining is not None: + self.length_remaining = max(self.length_remaining - len(data), 0) + if (self.length_is_certain and self.length_remaining == 0) or len( + data + ) < amt: + # definitely finished reading, close response stream + self._response.body.close() + return typing.cast(bytes, data) + else: # read all we can (and cache it) + data = self._response.body.read() + if cache_content: + self._body = data + if self.length_remaining is not None: + self.length_remaining = max(self.length_remaining - len(data), 0) + if len(data) == 0 or ( + self.length_is_certain and self.length_remaining == 0 + ): + # definitely finished reading, close response stream + self._response.body.close() + return typing.cast(bytes, data) + + def read_chunked( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> typing.Generator[bytes]: + # chunked is handled by browser + while True: + bytes = self.read(amt, decode_content) + if not bytes: + break + yield bytes + + def release_conn(self) -> None: + if not self._pool or not self._connection: + return None + + self._pool._put_conn(self._connection) + self._connection = None + + def drain_conn(self) -> None: + self.close() + + @property + def data(self) -> bytes: + if self._body: + return self._body + else: + return self.read(cache_content=True) + + def json(self) -> typing.Any: + """ + Deserializes the body of the HTTP response as a Python object. + + The body of the HTTP response must be encoded using UTF-8, as per + `RFC 8529 Section 8.1 `_. + + To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to + your custom decoder instead. + + If the body of the HTTP response is not decodable to UTF-8, a + `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a + valid JSON document, a `json.JSONDecodeError` will be raised. + + Read more :ref:`here `. + + :returns: The body of the HTTP response as a Python object. + """ + data = self.data.decode("utf-8") + return _json.loads(data) + + def close(self) -> None: + if not self._closed: + if isinstance(self._response.body, IOBase): + self._response.body.close() + if self._connection: + self._connection.close() + self._connection = None + self._closed = True + + @contextmanager + def _error_catcher(self) -> typing.Generator[None]: + """ + Catch Emscripten specific exceptions thrown by fetch.py, + instead re-raising urllib3 variants, so that low-level exceptions + are not leaked in the high-level api. + + On exit, release the connection back to the pool. + """ + from .fetch import _RequestError, _TimeoutError # avoid circular import + + clean_exit = False + + try: + yield + # If no exception is thrown, we should avoid cleaning up + # unnecessarily. + clean_exit = True + except _TimeoutError as e: + raise TimeoutError(str(e)) + except _RequestError as e: + raise HTTPException(str(e)) + finally: + # If we didn't terminate cleanly, we need to throw away our + # connection. + if not clean_exit: + # The response may not be closed but we're not going to use it + # anymore so close it now + if ( + isinstance(self._response.body, IOBase) + and not self._response.body.closed + ): + self._response.body.close() + # release the connection back to the pool + self.release_conn() + else: + # If we have read everything from the response stream, + # return the connection back to the pool. + if ( + isinstance(self._response.body, IOBase) + and self._response.body.closed + ): + self.release_conn() diff --git a/src/urllib3/contrib/ntlmpool.py b/src/urllib3/contrib/ntlmpool.py deleted file mode 100644 index 4716657..0000000 --- a/src/urllib3/contrib/ntlmpool.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -NTLM authenticating pool, contributed by erikcederstran - -Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 -""" -from __future__ import absolute_import - -import warnings -from logging import getLogger - -from ntlm import ntlm - -from .. import HTTPSConnectionPool -from ..packages.six.moves.http_client import HTTPSConnection - -warnings.warn( - "The 'urllib3.contrib.ntlmpool' module is deprecated and will be removed " - "in urllib3 v2.0 release, urllib3 is not able to support it properly due " - "to reasons listed in issue: https://github.com/urllib3/urllib3/issues/2282. " - "If you are a user of this module please comment in the mentioned issue.", - DeprecationWarning, -) - -log = getLogger(__name__) - - -class NTLMConnectionPool(HTTPSConnectionPool): - """ - Implements an NTLM authentication version of an urllib3 connection pool - """ - - scheme = "https" - - def __init__(self, user, pw, authurl, *args, **kwargs): - """ - authurl is a random URL on the server that is protected by NTLM. - user is the Windows user, probably in the DOMAIN\\username format. - pw is the password for the user. - """ - super(NTLMConnectionPool, self).__init__(*args, **kwargs) - self.authurl = authurl - self.rawuser = user - user_parts = user.split("\\", 1) - self.domain = user_parts[0].upper() - self.user = user_parts[1] - self.pw = pw - - def _new_conn(self): - # Performs the NTLM handshake that secures the connection. The socket - # must be kept open while requests are performed. - self.num_connections += 1 - log.debug( - "Starting NTLM HTTPS connection no. %d: https://%s%s", - self.num_connections, - self.host, - self.authurl, - ) - - headers = {"Connection": "Keep-Alive"} - req_header = "Authorization" - resp_header = "www-authenticate" - - conn = HTTPSConnection(host=self.host, port=self.port) - - # Send negotiation message - headers[req_header] = "NTLM %s" % ntlm.create_NTLM_NEGOTIATE_MESSAGE( - self.rawuser - ) - log.debug("Request headers: %s", headers) - conn.request("GET", self.authurl, None, headers) - res = conn.getresponse() - reshdr = dict(res.headers) - log.debug("Response status: %s %s", res.status, res.reason) - log.debug("Response headers: %s", reshdr) - log.debug("Response data: %s [...]", res.read(100)) - - # Remove the reference to the socket, so that it can not be closed by - # the response object (we want to keep the socket open) - res.fp = None - - # Server should respond with a challenge message - auth_header_values = reshdr[resp_header].split(", ") - auth_header_value = None - for s in auth_header_values: - if s[:5] == "NTLM ": - auth_header_value = s[5:] - if auth_header_value is None: - raise Exception( - "Unexpected %s response header: %s" % (resp_header, reshdr[resp_header]) - ) - - # Send authentication message - ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE( - auth_header_value - ) - auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE( - ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags - ) - headers[req_header] = "NTLM %s" % auth_msg - log.debug("Request headers: %s", headers) - conn.request("GET", self.authurl, None, headers) - res = conn.getresponse() - log.debug("Response status: %s %s", res.status, res.reason) - log.debug("Response headers: %s", dict(res.headers)) - log.debug("Response data: %s [...]", res.read()[:100]) - if res.status != 200: - if res.status == 401: - raise Exception("Server rejected request: wrong username or password") - raise Exception("Wrong server response: %s %s" % (res.status, res.reason)) - - res.fp = None - log.debug("Connection established") - return conn - - def urlopen( - self, - method, - url, - body=None, - headers=None, - retries=3, - redirect=True, - assert_same_host=True, - ): - if headers is None: - headers = {} - headers["Connection"] = "Keep-Alive" - return super(NTLMConnectionPool, self).urlopen( - method, url, body, headers, retries, redirect, assert_same_host - ) diff --git a/src/urllib3/contrib/pyopenssl.py b/src/urllib3/contrib/pyopenssl.py index 1ed214b..ed65430 100644 --- a/src/urllib3/contrib/pyopenssl.py +++ b/src/urllib3/contrib/pyopenssl.py @@ -1,17 +1,17 @@ """ -TLS with SNI_-support for Python 2. Follow these instructions if you would -like to verify TLS certificates in Python 2. Note, the default libraries do -*not* do certificate checking; you need to do additional work to validate -certificates yourself. +Module for using pyOpenSSL as a TLS backend. This module was relevant before +the standard library ``ssl`` module supported SNI, but now that we've dropped +support for Python 2.7 all relevant Python versions support SNI so +**this module is no longer recommended**. This needs the following packages installed: * `pyOpenSSL`_ (tested with 16.0.0) * `cryptography`_ (minimum 1.3.4, from pyopenssl) -* `idna`_ (minimum 2.0, from cryptography) +* `idna`_ (minimum 2.0) -However, pyopenssl depends on cryptography, which depends on idna, so while we -use all three directly here we end up having relatively few packages required. +However, pyOpenSSL depends on cryptography, so while we use all three directly here we +end up having relatively few packages required. You can install them with the following command: @@ -33,75 +33,46 @@ except ImportError: pass -Now you can use :mod:`urllib3` as you normally would, and it will support SNI -when the required modules are installed. - -Activating this module also has the positive side effect of disabling SSL/TLS -compression in Python 2 (see `CRIME attack`_). - -.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication -.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) .. _pyopenssl: https://www.pyopenssl.org .. _cryptography: https://cryptography.io .. _idna: https://github.com/kjd/idna """ -from __future__ import absolute_import -import OpenSSL.crypto -import OpenSSL.SSL +from __future__ import annotations + +import OpenSSL.SSL # type: ignore[import-untyped] from cryptography import x509 -from cryptography.hazmat.backends.openssl import backend as openssl_backend try: - from cryptography.x509 import UnsupportedExtension + from cryptography.x509 import UnsupportedExtension # type: ignore[attr-defined] except ImportError: # UnsupportedExtension is gone in cryptography >= 2.1.0 - class UnsupportedExtension(Exception): + class UnsupportedExtension(Exception): # type: ignore[no-redef] pass +import logging +import ssl +import typing from io import BytesIO -from socket import error as SocketError +from socket import socket as socket_cls from socket import timeout -try: # Platform-specific: Python 2 - from socket import _fileobject -except ImportError: # Platform-specific: Python 3 - _fileobject = None - from ..packages.backports.makefile import backport_makefile +from .. import util -import logging -import ssl -import sys -import warnings +if typing.TYPE_CHECKING: + from OpenSSL.crypto import X509 # type: ignore[import-untyped] -from .. import util -from ..packages import six -from ..util.ssl_ import PROTOCOL_TLS_CLIENT - -warnings.warn( - "'urllib3.contrib.pyopenssl' module is deprecated and will be removed " - "in a future release of urllib3 2.x. Read more in this issue: " - "https://github.com/urllib3/urllib3/issues/2680", - category=DeprecationWarning, - stacklevel=2, -) __all__ = ["inject_into_urllib3", "extract_from_urllib3"] -# SNI always works. -HAS_SNI = True - # Map from urllib3 to PyOpenSSL compatible parameter-values. -_openssl_versions = { - util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, - PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, +_openssl_versions: dict[int, int] = { + util.ssl_.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined] + util.ssl_.PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined] ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } -if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"): - _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD - if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"): _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD @@ -115,43 +86,77 @@ class UnsupportedExtension(Exception): ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -_openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items()) +_openssl_to_stdlib_verify = {v: k for k, v in _stdlib_to_openssl_verify.items()} + +# The SSLvX values are the most likely to be missing in the future +# but we check them all just to be sure. +_OP_NO_SSLv2_OR_SSLv3: int = getattr(OpenSSL.SSL, "OP_NO_SSLv2", 0) | getattr( + OpenSSL.SSL, "OP_NO_SSLv3", 0 +) +_OP_NO_TLSv1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1", 0) +_OP_NO_TLSv1_1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_1", 0) +_OP_NO_TLSv1_2: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_2", 0) +_OP_NO_TLSv1_3: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_3", 0) + +_openssl_to_ssl_minimum_version: dict[int, int] = { + ssl.TLSVersion.MINIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3, + ssl.TLSVersion.TLSv1: _OP_NO_SSLv2_OR_SSLv3, + ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1, + ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1, + ssl.TLSVersion.TLSv1_3: ( + _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 + ), + ssl.TLSVersion.MAXIMUM_SUPPORTED: ( + _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 + ), +} +_openssl_to_ssl_maximum_version: dict[int, int] = { + ssl.TLSVersion.MINIMUM_SUPPORTED: ( + _OP_NO_SSLv2_OR_SSLv3 + | _OP_NO_TLSv1 + | _OP_NO_TLSv1_1 + | _OP_NO_TLSv1_2 + | _OP_NO_TLSv1_3 + ), + ssl.TLSVersion.TLSv1: ( + _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3 + ), + ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3, + ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_3, + ssl.TLSVersion.TLSv1_3: _OP_NO_SSLv2_OR_SSLv3, + ssl.TLSVersion.MAXIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3, +} # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 -orig_util_HAS_SNI = util.HAS_SNI orig_util_SSLContext = util.ssl_.SSLContext log = logging.getLogger(__name__) -def inject_into_urllib3(): +def inject_into_urllib3() -> None: "Monkey-patch urllib3 with PyOpenSSL-backed SSL-support." _validate_dependencies_met() - util.SSLContext = PyOpenSSLContext - util.ssl_.SSLContext = PyOpenSSLContext - util.HAS_SNI = HAS_SNI - util.ssl_.HAS_SNI = HAS_SNI + util.SSLContext = PyOpenSSLContext # type: ignore[assignment] + util.ssl_.SSLContext = PyOpenSSLContext # type: ignore[assignment] util.IS_PYOPENSSL = True util.ssl_.IS_PYOPENSSL = True -def extract_from_urllib3(): +def extract_from_urllib3() -> None: "Undo monkey-patching by :func:`inject_into_urllib3`." util.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext - util.HAS_SNI = orig_util_HAS_SNI - util.ssl_.HAS_SNI = orig_util_HAS_SNI util.IS_PYOPENSSL = False util.ssl_.IS_PYOPENSSL = False -def _validate_dependencies_met(): +def _validate_dependencies_met() -> None: """ Verifies that PyOpenSSL's package-level dependencies have been met. Throws `ImportError` if they are not met. @@ -177,7 +182,7 @@ def _validate_dependencies_met(): ) -def _dnsname_to_stdlib(name): +def _dnsname_to_stdlib(name: str) -> str | None: """ Converts a dNSName SubjectAlternativeName field to the form used by the standard library on the given Python version. @@ -191,7 +196,7 @@ def _dnsname_to_stdlib(name): the name given should be skipped. """ - def idna_encode(name): + def idna_encode(name: str) -> bytes | None: """ Borrowed wholesale from the Python Cryptography Project. It turns out that we can't just safely call `idna.encode`: it can explode for @@ -200,7 +205,7 @@ def idna_encode(name): import idna try: - for prefix in [u"*.", u"."]: + for prefix in ["*.", "."]: if name.startswith(prefix): name = name[len(prefix) :] return prefix.encode("ascii") + idna.encode(name) @@ -212,24 +217,17 @@ def idna_encode(name): if ":" in name: return name - name = idna_encode(name) - if name is None: + encoded_name = idna_encode(name) + if encoded_name is None: return None - elif sys.version_info >= (3, 0): - name = name.decode("utf-8") - return name + return encoded_name.decode("utf-8") -def get_subj_alt_name(peer_cert): +def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]: """ Given an PyOpenSSL certificate, provides all the subject alternative names. """ - # Pass the cert to cryptography, which has much better APIs for this. - if hasattr(peer_cert, "to_cryptography"): - cert = peer_cert.to_cryptography() - else: - der = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, peer_cert) - cert = x509.load_der_x509_certificate(der, openssl_backend) + cert = peer_cert.to_cryptography() # We want to find the SAN extension. Ask Cryptography to locate it (it's # faster than looping in Python) @@ -273,93 +271,94 @@ def get_subj_alt_name(peer_cert): return names -class WrappedSocket(object): - """API-compatibility wrapper for Python OpenSSL's Connection-class. +class WrappedSocket: + """API-compatibility wrapper for Python OpenSSL's Connection-class.""" - Note: _makefile_refs, _drop() and _reuse() are needed for the garbage - collector of pypy. - """ - - def __init__(self, connection, socket, suppress_ragged_eofs=True): + def __init__( + self, + connection: OpenSSL.SSL.Connection, + socket: socket_cls, + suppress_ragged_eofs: bool = True, + ) -> None: self.connection = connection self.socket = socket self.suppress_ragged_eofs = suppress_ragged_eofs - self._makefile_refs = 0 + self._io_refs = 0 self._closed = False - def fileno(self): + def fileno(self) -> int: return self.socket.fileno() # Copy-pasted from Python 3.5 source code - def _decref_socketios(self): - if self._makefile_refs > 0: - self._makefile_refs -= 1 + def _decref_socketios(self) -> None: + if self._io_refs > 0: + self._io_refs -= 1 if self._closed: self.close() - def recv(self, *args, **kwargs): + def recv(self, *args: typing.Any, **kwargs: typing.Any) -> bytes: try: data = self.connection.recv(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): return b"" else: - raise SocketError(str(e)) + raise OSError(e.args[0], str(e)) from e except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b"" else: raise - except OpenSSL.SSL.WantReadError: + except OpenSSL.SSL.WantReadError as e: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout("The read operation timed out") + raise timeout("The read operation timed out") from e else: return self.recv(*args, **kwargs) # TLS 1.3 post-handshake authentication except OpenSSL.SSL.Error as e: - raise ssl.SSLError("read error: %r" % e) + raise ssl.SSLError(f"read error: {e!r}") from e else: - return data + return data # type: ignore[no-any-return] - def recv_into(self, *args, **kwargs): + def recv_into(self, *args: typing.Any, **kwargs: typing.Any) -> int: try: - return self.connection.recv_into(*args, **kwargs) + return self.connection.recv_into(*args, **kwargs) # type: ignore[no-any-return] except OpenSSL.SSL.SysCallError as e: if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): return 0 else: - raise SocketError(str(e)) + raise OSError(e.args[0], str(e)) from e except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return 0 else: raise - except OpenSSL.SSL.WantReadError: + except OpenSSL.SSL.WantReadError as e: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout("The read operation timed out") + raise timeout("The read operation timed out") from e else: return self.recv_into(*args, **kwargs) # TLS 1.3 post-handshake authentication except OpenSSL.SSL.Error as e: - raise ssl.SSLError("read error: %r" % e) + raise ssl.SSLError(f"read error: {e!r}") from e - def settimeout(self, timeout): + def settimeout(self, timeout: float) -> None: return self.socket.settimeout(timeout) - def _send_until_done(self, data): + def _send_until_done(self, data: bytes) -> int: while True: try: - return self.connection.send(data) - except OpenSSL.SSL.WantWriteError: + return self.connection.send(data) # type: ignore[no-any-return] + except OpenSSL.SSL.WantWriteError as e: if not util.wait_for_write(self.socket, self.socket.gettimeout()): - raise timeout() + raise timeout() from e continue except OpenSSL.SSL.SysCallError as e: - raise SocketError(str(e)) + raise OSError(e.args[0], str(e)) from e - def sendall(self, data): + def sendall(self, data: bytes) -> None: total_sent = 0 while total_sent < len(data): sent = self._send_until_done( @@ -367,135 +366,141 @@ def sendall(self, data): ) total_sent += sent - def shutdown(self): - # FIXME rethrow compatible exceptions should we ever use this - self.connection.shutdown() + def shutdown(self, how: int) -> None: + try: + self.connection.shutdown() + except OpenSSL.SSL.Error as e: + raise ssl.SSLError(f"shutdown error: {e!r}") from e - def close(self): - if self._makefile_refs < 1: - try: - self._closed = True - return self.connection.close() - except OpenSSL.SSL.Error: - return - else: - self._makefile_refs -= 1 + def close(self) -> None: + self._closed = True + if self._io_refs <= 0: + self._real_close() - def getpeercert(self, binary_form=False): + def _real_close(self) -> None: + try: + return self.connection.close() # type: ignore[no-any-return] + except OpenSSL.SSL.Error: + return + + def getpeercert( + self, binary_form: bool = False + ) -> dict[str, list[typing.Any]] | None: x509 = self.connection.get_peer_certificate() if not x509: - return x509 + return x509 # type: ignore[no-any-return] if binary_form: - return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) + return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) # type: ignore[no-any-return] return { - "subject": ((("commonName", x509.get_subject().CN),),), + "subject": ((("commonName", x509.get_subject().CN),),), # type: ignore[dict-item] "subjectAltName": get_subj_alt_name(x509), } - def version(self): - return self.connection.get_protocol_version_name() - - def _reuse(self): - self._makefile_refs += 1 - - def _drop(self): - if self._makefile_refs < 1: - self.close() - else: - self._makefile_refs -= 1 - - -if _fileobject: # Platform-specific: Python 2 + def version(self) -> str: + return self.connection.get_protocol_version_name() # type: ignore[no-any-return] - def makefile(self, mode, bufsize=-1): - self._makefile_refs += 1 - return _fileobject(self, mode, bufsize, close=True) + def selected_alpn_protocol(self) -> str | None: + alpn_proto = self.connection.get_alpn_proto_negotiated() + return alpn_proto.decode() if alpn_proto else None -else: # Platform-specific: Python 3 - makefile = backport_makefile -WrappedSocket.makefile = makefile +WrappedSocket.makefile = socket_cls.makefile # type: ignore[attr-defined] -class PyOpenSSLContext(object): +class PyOpenSSLContext: """ I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible for translating the interface of the standard library ``SSLContext`` object to calls into PyOpenSSL. """ - def __init__(self, protocol): + def __init__(self, protocol: int) -> None: self.protocol = _openssl_versions[protocol] self._ctx = OpenSSL.SSL.Context(self.protocol) self._options = 0 self.check_hostname = False + self._minimum_version: int = ssl.TLSVersion.MINIMUM_SUPPORTED + self._maximum_version: int = ssl.TLSVersion.MAXIMUM_SUPPORTED @property - def options(self): + def options(self) -> int: return self._options @options.setter - def options(self, value): + def options(self, value: int) -> None: self._options = value - self._ctx.set_options(value) + self._set_ctx_options() @property - def verify_mode(self): + def verify_mode(self) -> int: return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()] @verify_mode.setter - def verify_mode(self, value): + def verify_mode(self, value: ssl.VerifyMode) -> None: self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback) - def set_default_verify_paths(self): + def set_default_verify_paths(self) -> None: self._ctx.set_default_verify_paths() - def set_ciphers(self, ciphers): - if isinstance(ciphers, six.text_type): + def set_ciphers(self, ciphers: bytes | str) -> None: + if isinstance(ciphers, str): ciphers = ciphers.encode("utf-8") self._ctx.set_cipher_list(ciphers) - def load_verify_locations(self, cafile=None, capath=None, cadata=None): + def load_verify_locations( + self, + cafile: str | None = None, + capath: str | None = None, + cadata: bytes | None = None, + ) -> None: if cafile is not None: - cafile = cafile.encode("utf-8") + cafile = cafile.encode("utf-8") # type: ignore[assignment] if capath is not None: - capath = capath.encode("utf-8") + capath = capath.encode("utf-8") # type: ignore[assignment] try: self._ctx.load_verify_locations(cafile, capath) if cadata is not None: self._ctx.load_verify_locations(BytesIO(cadata)) except OpenSSL.SSL.Error as e: - raise ssl.SSLError("unable to load trusted certificates: %r" % e) + raise ssl.SSLError(f"unable to load trusted certificates: {e!r}") from e - def load_cert_chain(self, certfile, keyfile=None, password=None): - self._ctx.use_certificate_chain_file(certfile) - if password is not None: - if not isinstance(password, six.binary_type): - password = password.encode("utf-8") - self._ctx.set_passwd_cb(lambda *_: password) - self._ctx.use_privatekey_file(keyfile or certfile) + def load_cert_chain( + self, + certfile: str, + keyfile: str | None = None, + password: str | None = None, + ) -> None: + try: + self._ctx.use_certificate_chain_file(certfile) + if password is not None: + if not isinstance(password, bytes): + password = password.encode("utf-8") # type: ignore[assignment] + self._ctx.set_passwd_cb(lambda *_: password) + self._ctx.use_privatekey_file(keyfile or certfile) + except OpenSSL.SSL.Error as e: + raise ssl.SSLError(f"Unable to load certificate chain: {e!r}") from e - def set_alpn_protocols(self, protocols): - protocols = [six.ensure_binary(p) for p in protocols] - return self._ctx.set_alpn_protos(protocols) + def set_alpn_protocols(self, protocols: list[bytes | str]) -> None: + protocols = [util.util.to_bytes(p, "ascii") for p in protocols] + return self._ctx.set_alpn_protos(protocols) # type: ignore[no-any-return] def wrap_socket( self, - sock, - server_side=False, - do_handshake_on_connect=True, - suppress_ragged_eofs=True, - server_hostname=None, - ): + sock: socket_cls, + server_side: bool = False, + do_handshake_on_connect: bool = True, + suppress_ragged_eofs: bool = True, + server_hostname: bytes | str | None = None, + ) -> WrappedSocket: cnx = OpenSSL.SSL.Connection(self._ctx, sock) - if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 - server_hostname = server_hostname.encode("utf-8") - - if server_hostname is not None: + # If server_hostname is an IP, don't use it for SNI, per RFC6066 Section 3 + if server_hostname and not util.ssl_.is_ipaddress(server_hostname): + if isinstance(server_hostname, str): + server_hostname = server_hostname.encode("utf-8") cnx.set_tlsext_host_name(server_hostname) cnx.set_connect_state() @@ -503,16 +508,47 @@ def wrap_socket( while True: try: cnx.do_handshake() - except OpenSSL.SSL.WantReadError: + except OpenSSL.SSL.WantReadError as e: if not util.wait_for_read(sock, sock.gettimeout()): - raise timeout("select timed out") + raise timeout("select timed out") from e continue except OpenSSL.SSL.Error as e: - raise ssl.SSLError("bad handshake: %r" % e) + raise ssl.SSLError(f"bad handshake: {e!r}") from e break return WrappedSocket(cnx, sock) + def _set_ctx_options(self) -> None: + self._ctx.set_options( + self._options + | _openssl_to_ssl_minimum_version[self._minimum_version] + | _openssl_to_ssl_maximum_version[self._maximum_version] + ) + + @property + def minimum_version(self) -> int: + return self._minimum_version -def _verify_callback(cnx, x509, err_no, err_depth, return_code): + @minimum_version.setter + def minimum_version(self, minimum_version: int) -> None: + self._minimum_version = minimum_version + self._set_ctx_options() + + @property + def maximum_version(self) -> int: + return self._maximum_version + + @maximum_version.setter + def maximum_version(self, maximum_version: int) -> None: + self._maximum_version = maximum_version + self._set_ctx_options() + + +def _verify_callback( + cnx: OpenSSL.SSL.Connection, + x509: X509, + err_no: int, + err_depth: int, + return_code: int, +) -> bool: return err_no == 0 diff --git a/src/urllib3/contrib/securetransport.py b/src/urllib3/contrib/securetransport.py deleted file mode 100644 index e311c0c..0000000 --- a/src/urllib3/contrib/securetransport.py +++ /dev/null @@ -1,920 +0,0 @@ -""" -SecureTranport support for urllib3 via ctypes. - -This makes platform-native TLS available to urllib3 users on macOS without the -use of a compiler. This is an important feature because the Python Package -Index is moving to become a TLSv1.2-or-higher server, and the default OpenSSL -that ships with macOS is not capable of doing TLSv1.2. The only way to resolve -this is to give macOS users an alternative solution to the problem, and that -solution is to use SecureTransport. - -We use ctypes here because this solution must not require a compiler. That's -because pip is not allowed to require a compiler either. - -This is not intended to be a seriously long-term solution to this problem. -The hope is that PEP 543 will eventually solve this issue for us, at which -point we can retire this contrib module. But in the short term, we need to -solve the impending tire fire that is Python on Mac without this kind of -contrib module. So...here we are. - -To use this module, simply import and inject it:: - - import urllib3.contrib.securetransport - urllib3.contrib.securetransport.inject_into_urllib3() - -Happy TLSing! - -This code is a bastardised version of the code found in Will Bond's oscrypto -library. An enormous debt is owed to him for blazing this trail for us. For -that reason, this code should be considered to be covered both by urllib3's -license and by oscrypto's: - -.. code-block:: - - Copyright (c) 2015-2016 Will Bond - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. -""" -from __future__ import absolute_import - -import contextlib -import ctypes -import errno -import os.path -import shutil -import socket -import ssl -import struct -import threading -import weakref - -from .. import util -from ..packages import six -from ..util.ssl_ import PROTOCOL_TLS_CLIENT -from ._securetransport.bindings import CoreFoundation, Security, SecurityConst -from ._securetransport.low_level import ( - _assert_no_error, - _build_tls_unknown_ca_alert, - _cert_array_from_pem, - _create_cfstring_array, - _load_client_cert_chain, - _temporary_keychain, -) - -try: # Platform-specific: Python 2 - from socket import _fileobject -except ImportError: # Platform-specific: Python 3 - _fileobject = None - from ..packages.backports.makefile import backport_makefile - -__all__ = ["inject_into_urllib3", "extract_from_urllib3"] - -# SNI always works -HAS_SNI = True - -orig_util_HAS_SNI = util.HAS_SNI -orig_util_SSLContext = util.ssl_.SSLContext - -# This dictionary is used by the read callback to obtain a handle to the -# calling wrapped socket. This is a pretty silly approach, but for now it'll -# do. I feel like I should be able to smuggle a handle to the wrapped socket -# directly in the SSLConnectionRef, but for now this approach will work I -# guess. -# -# We need to lock around this structure for inserts, but we don't do it for -# reads/writes in the callbacks. The reasoning here goes as follows: -# -# 1. It is not possible to call into the callbacks before the dictionary is -# populated, so once in the callback the id must be in the dictionary. -# 2. The callbacks don't mutate the dictionary, they only read from it, and -# so cannot conflict with any of the insertions. -# -# This is good: if we had to lock in the callbacks we'd drastically slow down -# the performance of this code. -_connection_refs = weakref.WeakValueDictionary() -_connection_ref_lock = threading.Lock() - -# Limit writes to 16kB. This is OpenSSL's limit, but we'll cargo-cult it over -# for no better reason than we need *a* limit, and this one is right there. -SSL_WRITE_BLOCKSIZE = 16384 - -# This is our equivalent of util.ssl_.DEFAULT_CIPHERS, but expanded out to -# individual cipher suites. We need to do this because this is how -# SecureTransport wants them. -CIPHER_SUITES = [ - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, - SecurityConst.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, - SecurityConst.TLS_AES_256_GCM_SHA384, - SecurityConst.TLS_AES_128_GCM_SHA256, - SecurityConst.TLS_RSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_RSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_AES_128_CCM_8_SHA256, - SecurityConst.TLS_AES_128_CCM_SHA256, - SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA, -] - -# Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of -# TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. -# TLSv1 to 1.2 are supported on macOS 10.8+ -_protocol_to_min_max = { - util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), - PROTOCOL_TLS_CLIENT: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), -} - -if hasattr(ssl, "PROTOCOL_SSLv2"): - _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = ( - SecurityConst.kSSLProtocol2, - SecurityConst.kSSLProtocol2, - ) -if hasattr(ssl, "PROTOCOL_SSLv3"): - _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = ( - SecurityConst.kSSLProtocol3, - SecurityConst.kSSLProtocol3, - ) -if hasattr(ssl, "PROTOCOL_TLSv1"): - _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = ( - SecurityConst.kTLSProtocol1, - SecurityConst.kTLSProtocol1, - ) -if hasattr(ssl, "PROTOCOL_TLSv1_1"): - _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = ( - SecurityConst.kTLSProtocol11, - SecurityConst.kTLSProtocol11, - ) -if hasattr(ssl, "PROTOCOL_TLSv1_2"): - _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = ( - SecurityConst.kTLSProtocol12, - SecurityConst.kTLSProtocol12, - ) - - -def inject_into_urllib3(): - """ - Monkey-patch urllib3 with SecureTransport-backed SSL-support. - """ - util.SSLContext = SecureTransportContext - util.ssl_.SSLContext = SecureTransportContext - util.HAS_SNI = HAS_SNI - util.ssl_.HAS_SNI = HAS_SNI - util.IS_SECURETRANSPORT = True - util.ssl_.IS_SECURETRANSPORT = True - - -def extract_from_urllib3(): - """ - Undo monkey-patching by :func:`inject_into_urllib3`. - """ - util.SSLContext = orig_util_SSLContext - util.ssl_.SSLContext = orig_util_SSLContext - util.HAS_SNI = orig_util_HAS_SNI - util.ssl_.HAS_SNI = orig_util_HAS_SNI - util.IS_SECURETRANSPORT = False - util.ssl_.IS_SECURETRANSPORT = False - - -def _read_callback(connection_id, data_buffer, data_length_pointer): - """ - SecureTransport read callback. This is called by ST to request that data - be returned from the socket. - """ - wrapped_socket = None - try: - wrapped_socket = _connection_refs.get(connection_id) - if wrapped_socket is None: - return SecurityConst.errSSLInternal - base_socket = wrapped_socket.socket - - requested_length = data_length_pointer[0] - - timeout = wrapped_socket.gettimeout() - error = None - read_count = 0 - - try: - while read_count < requested_length: - if timeout is None or timeout >= 0: - if not util.wait_for_read(base_socket, timeout): - raise socket.error(errno.EAGAIN, "timed out") - - remaining = requested_length - read_count - buffer = (ctypes.c_char * remaining).from_address( - data_buffer + read_count - ) - chunk_size = base_socket.recv_into(buffer, remaining) - read_count += chunk_size - if not chunk_size: - if not read_count: - return SecurityConst.errSSLClosedGraceful - break - except (socket.error) as e: - error = e.errno - - if error is not None and error != errno.EAGAIN: - data_length_pointer[0] = read_count - if error == errno.ECONNRESET or error == errno.EPIPE: - return SecurityConst.errSSLClosedAbort - raise - - data_length_pointer[0] = read_count - - if read_count != requested_length: - return SecurityConst.errSSLWouldBlock - - return 0 - except Exception as e: - if wrapped_socket is not None: - wrapped_socket._exception = e - return SecurityConst.errSSLInternal - - -def _write_callback(connection_id, data_buffer, data_length_pointer): - """ - SecureTransport write callback. This is called by ST to request that data - actually be sent on the network. - """ - wrapped_socket = None - try: - wrapped_socket = _connection_refs.get(connection_id) - if wrapped_socket is None: - return SecurityConst.errSSLInternal - base_socket = wrapped_socket.socket - - bytes_to_write = data_length_pointer[0] - data = ctypes.string_at(data_buffer, bytes_to_write) - - timeout = wrapped_socket.gettimeout() - error = None - sent = 0 - - try: - while sent < bytes_to_write: - if timeout is None or timeout >= 0: - if not util.wait_for_write(base_socket, timeout): - raise socket.error(errno.EAGAIN, "timed out") - chunk_sent = base_socket.send(data) - sent += chunk_sent - - # This has some needless copying here, but I'm not sure there's - # much value in optimising this data path. - data = data[chunk_sent:] - except (socket.error) as e: - error = e.errno - - if error is not None and error != errno.EAGAIN: - data_length_pointer[0] = sent - if error == errno.ECONNRESET or error == errno.EPIPE: - return SecurityConst.errSSLClosedAbort - raise - - data_length_pointer[0] = sent - - if sent != bytes_to_write: - return SecurityConst.errSSLWouldBlock - - return 0 - except Exception as e: - if wrapped_socket is not None: - wrapped_socket._exception = e - return SecurityConst.errSSLInternal - - -# We need to keep these two objects references alive: if they get GC'd while -# in use then SecureTransport could attempt to call a function that is in freed -# memory. That would be...uh...bad. Yeah, that's the word. Bad. -_read_callback_pointer = Security.SSLReadFunc(_read_callback) -_write_callback_pointer = Security.SSLWriteFunc(_write_callback) - - -class WrappedSocket(object): - """ - API-compatibility wrapper for Python's OpenSSL wrapped socket object. - - Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage - collector of PyPy. - """ - - def __init__(self, socket): - self.socket = socket - self.context = None - self._makefile_refs = 0 - self._closed = False - self._exception = None - self._keychain = None - self._keychain_dir = None - self._client_cert_chain = None - - # We save off the previously-configured timeout and then set it to - # zero. This is done because we use select and friends to handle the - # timeouts, but if we leave the timeout set on the lower socket then - # Python will "kindly" call select on that socket again for us. Avoid - # that by forcing the timeout to zero. - self._timeout = self.socket.gettimeout() - self.socket.settimeout(0) - - @contextlib.contextmanager - def _raise_on_error(self): - """ - A context manager that can be used to wrap calls that do I/O from - SecureTransport. If any of the I/O callbacks hit an exception, this - context manager will correctly propagate the exception after the fact. - This avoids silently swallowing those exceptions. - - It also correctly forces the socket closed. - """ - self._exception = None - - # We explicitly don't catch around this yield because in the unlikely - # event that an exception was hit in the block we don't want to swallow - # it. - yield - if self._exception is not None: - exception, self._exception = self._exception, None - self.close() - raise exception - - def _set_ciphers(self): - """ - Sets up the allowed ciphers. By default this matches the set in - util.ssl_.DEFAULT_CIPHERS, at least as supported by macOS. This is done - custom and doesn't allow changing at this time, mostly because parsing - OpenSSL cipher strings is going to be a freaking nightmare. - """ - ciphers = (Security.SSLCipherSuite * len(CIPHER_SUITES))(*CIPHER_SUITES) - result = Security.SSLSetEnabledCiphers( - self.context, ciphers, len(CIPHER_SUITES) - ) - _assert_no_error(result) - - def _set_alpn_protocols(self, protocols): - """ - Sets up the ALPN protocols on the context. - """ - if not protocols: - return - protocols_arr = _create_cfstring_array(protocols) - try: - result = Security.SSLSetALPNProtocols(self.context, protocols_arr) - _assert_no_error(result) - finally: - CoreFoundation.CFRelease(protocols_arr) - - def _custom_validate(self, verify, trust_bundle): - """ - Called when we have set custom validation. We do this in two cases: - first, when cert validation is entirely disabled; and second, when - using a custom trust DB. - Raises an SSLError if the connection is not trusted. - """ - # If we disabled cert validation, just say: cool. - if not verify: - return - - successes = ( - SecurityConst.kSecTrustResultUnspecified, - SecurityConst.kSecTrustResultProceed, - ) - try: - trust_result = self._evaluate_trust(trust_bundle) - if trust_result in successes: - return - reason = "error code: %d" % (trust_result,) - except Exception as e: - # Do not trust on error - reason = "exception: %r" % (e,) - - # SecureTransport does not send an alert nor shuts down the connection. - rec = _build_tls_unknown_ca_alert(self.version()) - self.socket.sendall(rec) - # close the connection immediately - # l_onoff = 1, activate linger - # l_linger = 0, linger for 0 seoncds - opts = struct.pack("ii", 1, 0) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, opts) - self.close() - raise ssl.SSLError("certificate verify failed, %s" % reason) - - def _evaluate_trust(self, trust_bundle): - # We want data in memory, so load it up. - if os.path.isfile(trust_bundle): - with open(trust_bundle, "rb") as f: - trust_bundle = f.read() - - cert_array = None - trust = Security.SecTrustRef() - - try: - # Get a CFArray that contains the certs we want. - cert_array = _cert_array_from_pem(trust_bundle) - - # Ok, now the hard part. We want to get the SecTrustRef that ST has - # created for this connection, shove our CAs into it, tell ST to - # ignore everything else it knows, and then ask if it can build a - # chain. This is a buuuunch of code. - result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) - _assert_no_error(result) - if not trust: - raise ssl.SSLError("Failed to copy trust reference") - - result = Security.SecTrustSetAnchorCertificates(trust, cert_array) - _assert_no_error(result) - - result = Security.SecTrustSetAnchorCertificatesOnly(trust, True) - _assert_no_error(result) - - trust_result = Security.SecTrustResultType() - result = Security.SecTrustEvaluate(trust, ctypes.byref(trust_result)) - _assert_no_error(result) - finally: - if trust: - CoreFoundation.CFRelease(trust) - - if cert_array is not None: - CoreFoundation.CFRelease(cert_array) - - return trust_result.value - - def handshake( - self, - server_hostname, - verify, - trust_bundle, - min_version, - max_version, - client_cert, - client_key, - client_key_passphrase, - alpn_protocols, - ): - """ - Actually performs the TLS handshake. This is run automatically by - wrapped socket, and shouldn't be needed in user code. - """ - # First, we do the initial bits of connection setup. We need to create - # a context, set its I/O funcs, and set the connection reference. - self.context = Security.SSLCreateContext( - None, SecurityConst.kSSLClientSide, SecurityConst.kSSLStreamType - ) - result = Security.SSLSetIOFuncs( - self.context, _read_callback_pointer, _write_callback_pointer - ) - _assert_no_error(result) - - # Here we need to compute the handle to use. We do this by taking the - # id of self modulo 2**31 - 1. If this is already in the dictionary, we - # just keep incrementing by one until we find a free space. - with _connection_ref_lock: - handle = id(self) % 2147483647 - while handle in _connection_refs: - handle = (handle + 1) % 2147483647 - _connection_refs[handle] = self - - result = Security.SSLSetConnection(self.context, handle) - _assert_no_error(result) - - # If we have a server hostname, we should set that too. - if server_hostname: - if not isinstance(server_hostname, bytes): - server_hostname = server_hostname.encode("utf-8") - - result = Security.SSLSetPeerDomainName( - self.context, server_hostname, len(server_hostname) - ) - _assert_no_error(result) - - # Setup the ciphers. - self._set_ciphers() - - # Setup the ALPN protocols. - self._set_alpn_protocols(alpn_protocols) - - # Set the minimum and maximum TLS versions. - result = Security.SSLSetProtocolVersionMin(self.context, min_version) - _assert_no_error(result) - - result = Security.SSLSetProtocolVersionMax(self.context, max_version) - _assert_no_error(result) - - # If there's a trust DB, we need to use it. We do that by telling - # SecureTransport to break on server auth. We also do that if we don't - # want to validate the certs at all: we just won't actually do any - # authing in that case. - if not verify or trust_bundle is not None: - result = Security.SSLSetSessionOption( - self.context, SecurityConst.kSSLSessionOptionBreakOnServerAuth, True - ) - _assert_no_error(result) - - # If there's a client cert, we need to use it. - if client_cert: - self._keychain, self._keychain_dir = _temporary_keychain() - self._client_cert_chain = _load_client_cert_chain( - self._keychain, client_cert, client_key - ) - result = Security.SSLSetCertificate(self.context, self._client_cert_chain) - _assert_no_error(result) - - while True: - with self._raise_on_error(): - result = Security.SSLHandshake(self.context) - - if result == SecurityConst.errSSLWouldBlock: - raise socket.timeout("handshake timed out") - elif result == SecurityConst.errSSLServerAuthCompleted: - self._custom_validate(verify, trust_bundle) - continue - else: - _assert_no_error(result) - break - - def fileno(self): - return self.socket.fileno() - - # Copy-pasted from Python 3.5 source code - def _decref_socketios(self): - if self._makefile_refs > 0: - self._makefile_refs -= 1 - if self._closed: - self.close() - - def recv(self, bufsiz): - buffer = ctypes.create_string_buffer(bufsiz) - bytes_read = self.recv_into(buffer, bufsiz) - data = buffer[:bytes_read] - return data - - def recv_into(self, buffer, nbytes=None): - # Read short on EOF. - if self._closed: - return 0 - - if nbytes is None: - nbytes = len(buffer) - - buffer = (ctypes.c_char * nbytes).from_buffer(buffer) - processed_bytes = ctypes.c_size_t(0) - - with self._raise_on_error(): - result = Security.SSLRead( - self.context, buffer, nbytes, ctypes.byref(processed_bytes) - ) - - # There are some result codes that we want to treat as "not always - # errors". Specifically, those are errSSLWouldBlock, - # errSSLClosedGraceful, and errSSLClosedNoNotify. - if result == SecurityConst.errSSLWouldBlock: - # If we didn't process any bytes, then this was just a time out. - # However, we can get errSSLWouldBlock in situations when we *did* - # read some data, and in those cases we should just read "short" - # and return. - if processed_bytes.value == 0: - # Timed out, no data read. - raise socket.timeout("recv timed out") - elif result in ( - SecurityConst.errSSLClosedGraceful, - SecurityConst.errSSLClosedNoNotify, - ): - # The remote peer has closed this connection. We should do so as - # well. Note that we don't actually return here because in - # principle this could actually be fired along with return data. - # It's unlikely though. - self.close() - else: - _assert_no_error(result) - - # Ok, we read and probably succeeded. We should return whatever data - # was actually read. - return processed_bytes.value - - def settimeout(self, timeout): - self._timeout = timeout - - def gettimeout(self): - return self._timeout - - def send(self, data): - processed_bytes = ctypes.c_size_t(0) - - with self._raise_on_error(): - result = Security.SSLWrite( - self.context, data, len(data), ctypes.byref(processed_bytes) - ) - - if result == SecurityConst.errSSLWouldBlock and processed_bytes.value == 0: - # Timed out - raise socket.timeout("send timed out") - else: - _assert_no_error(result) - - # We sent, and probably succeeded. Tell them how much we sent. - return processed_bytes.value - - def sendall(self, data): - total_sent = 0 - while total_sent < len(data): - sent = self.send(data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]) - total_sent += sent - - def shutdown(self): - with self._raise_on_error(): - Security.SSLClose(self.context) - - def close(self): - # TODO: should I do clean shutdown here? Do I have to? - if self._makefile_refs < 1: - self._closed = True - if self.context: - CoreFoundation.CFRelease(self.context) - self.context = None - if self._client_cert_chain: - CoreFoundation.CFRelease(self._client_cert_chain) - self._client_cert_chain = None - if self._keychain: - Security.SecKeychainDelete(self._keychain) - CoreFoundation.CFRelease(self._keychain) - shutil.rmtree(self._keychain_dir) - self._keychain = self._keychain_dir = None - return self.socket.close() - else: - self._makefile_refs -= 1 - - def getpeercert(self, binary_form=False): - # Urgh, annoying. - # - # Here's how we do this: - # - # 1. Call SSLCopyPeerTrust to get hold of the trust object for this - # connection. - # 2. Call SecTrustGetCertificateAtIndex for index 0 to get the leaf. - # 3. To get the CN, call SecCertificateCopyCommonName and process that - # string so that it's of the appropriate type. - # 4. To get the SAN, we need to do something a bit more complex: - # a. Call SecCertificateCopyValues to get the data, requesting - # kSecOIDSubjectAltName. - # b. Mess about with this dictionary to try to get the SANs out. - # - # This is gross. Really gross. It's going to be a few hundred LoC extra - # just to repeat something that SecureTransport can *already do*. So my - # operating assumption at this time is that what we want to do is - # instead to just flag to urllib3 that it shouldn't do its own hostname - # validation when using SecureTransport. - if not binary_form: - raise ValueError("SecureTransport only supports dumping binary certs") - trust = Security.SecTrustRef() - certdata = None - der_bytes = None - - try: - # Grab the trust store. - result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) - _assert_no_error(result) - if not trust: - # Probably we haven't done the handshake yet. No biggie. - return None - - cert_count = Security.SecTrustGetCertificateCount(trust) - if not cert_count: - # Also a case that might happen if we haven't handshaked. - # Handshook? Handshaken? - return None - - leaf = Security.SecTrustGetCertificateAtIndex(trust, 0) - assert leaf - - # Ok, now we want the DER bytes. - certdata = Security.SecCertificateCopyData(leaf) - assert certdata - - data_length = CoreFoundation.CFDataGetLength(certdata) - data_buffer = CoreFoundation.CFDataGetBytePtr(certdata) - der_bytes = ctypes.string_at(data_buffer, data_length) - finally: - if certdata: - CoreFoundation.CFRelease(certdata) - if trust: - CoreFoundation.CFRelease(trust) - - return der_bytes - - def version(self): - protocol = Security.SSLProtocol() - result = Security.SSLGetNegotiatedProtocolVersion( - self.context, ctypes.byref(protocol) - ) - _assert_no_error(result) - if protocol.value == SecurityConst.kTLSProtocol13: - raise ssl.SSLError("SecureTransport does not support TLS 1.3") - elif protocol.value == SecurityConst.kTLSProtocol12: - return "TLSv1.2" - elif protocol.value == SecurityConst.kTLSProtocol11: - return "TLSv1.1" - elif protocol.value == SecurityConst.kTLSProtocol1: - return "TLSv1" - elif protocol.value == SecurityConst.kSSLProtocol3: - return "SSLv3" - elif protocol.value == SecurityConst.kSSLProtocol2: - return "SSLv2" - else: - raise ssl.SSLError("Unknown TLS version: %r" % protocol) - - def _reuse(self): - self._makefile_refs += 1 - - def _drop(self): - if self._makefile_refs < 1: - self.close() - else: - self._makefile_refs -= 1 - - -if _fileobject: # Platform-specific: Python 2 - - def makefile(self, mode, bufsize=-1): - self._makefile_refs += 1 - return _fileobject(self, mode, bufsize, close=True) - -else: # Platform-specific: Python 3 - - def makefile(self, mode="r", buffering=None, *args, **kwargs): - # We disable buffering with SecureTransport because it conflicts with - # the buffering that ST does internally (see issue #1153 for more). - buffering = 0 - return backport_makefile(self, mode, buffering, *args, **kwargs) - - -WrappedSocket.makefile = makefile - - -class SecureTransportContext(object): - """ - I am a wrapper class for the SecureTransport library, to translate the - interface of the standard library ``SSLContext`` object to calls into - SecureTransport. - """ - - def __init__(self, protocol): - self._min_version, self._max_version = _protocol_to_min_max[protocol] - self._options = 0 - self._verify = False - self._trust_bundle = None - self._client_cert = None - self._client_key = None - self._client_key_passphrase = None - self._alpn_protocols = None - - @property - def check_hostname(self): - """ - SecureTransport cannot have its hostname checking disabled. For more, - see the comment on getpeercert() in this file. - """ - return True - - @check_hostname.setter - def check_hostname(self, value): - """ - SecureTransport cannot have its hostname checking disabled. For more, - see the comment on getpeercert() in this file. - """ - pass - - @property - def options(self): - # TODO: Well, crap. - # - # So this is the bit of the code that is the most likely to cause us - # trouble. Essentially we need to enumerate all of the SSL options that - # users might want to use and try to see if we can sensibly translate - # them, or whether we should just ignore them. - return self._options - - @options.setter - def options(self, value): - # TODO: Update in line with above. - self._options = value - - @property - def verify_mode(self): - return ssl.CERT_REQUIRED if self._verify else ssl.CERT_NONE - - @verify_mode.setter - def verify_mode(self, value): - self._verify = True if value == ssl.CERT_REQUIRED else False - - def set_default_verify_paths(self): - # So, this has to do something a bit weird. Specifically, what it does - # is nothing. - # - # This means that, if we had previously had load_verify_locations - # called, this does not undo that. We need to do that because it turns - # out that the rest of the urllib3 code will attempt to load the - # default verify paths if it hasn't been told about any paths, even if - # the context itself was sometime earlier. We resolve that by just - # ignoring it. - pass - - def load_default_certs(self): - return self.set_default_verify_paths() - - def set_ciphers(self, ciphers): - # For now, we just require the default cipher string. - if ciphers != util.ssl_.DEFAULT_CIPHERS: - raise ValueError("SecureTransport doesn't support custom cipher strings") - - def load_verify_locations(self, cafile=None, capath=None, cadata=None): - # OK, we only really support cadata and cafile. - if capath is not None: - raise ValueError("SecureTransport does not support cert directories") - - # Raise if cafile does not exist. - if cafile is not None: - with open(cafile): - pass - - self._trust_bundle = cafile or cadata - - def load_cert_chain(self, certfile, keyfile=None, password=None): - self._client_cert = certfile - self._client_key = keyfile - self._client_cert_passphrase = password - - def set_alpn_protocols(self, protocols): - """ - Sets the ALPN protocols that will later be set on the context. - - Raises a NotImplementedError if ALPN is not supported. - """ - if not hasattr(Security, "SSLSetALPNProtocols"): - raise NotImplementedError( - "SecureTransport supports ALPN only in macOS 10.12+" - ) - self._alpn_protocols = [six.ensure_binary(p) for p in protocols] - - def wrap_socket( - self, - sock, - server_side=False, - do_handshake_on_connect=True, - suppress_ragged_eofs=True, - server_hostname=None, - ): - # So, what do we do here? Firstly, we assert some properties. This is a - # stripped down shim, so there is some functionality we don't support. - # See PEP 543 for the real deal. - assert not server_side - assert do_handshake_on_connect - assert suppress_ragged_eofs - - # Ok, we're good to go. Now we want to create the wrapped socket object - # and store it in the appropriate place. - wrapped_socket = WrappedSocket(sock) - - # Now we can handshake - wrapped_socket.handshake( - server_hostname, - self._verify, - self._trust_bundle, - self._min_version, - self._max_version, - self._client_cert, - self._client_key, - self._client_key_passphrase, - self._alpn_protocols, - ) - return wrapped_socket diff --git a/src/urllib3/contrib/socks.py b/src/urllib3/contrib/socks.py index c326e80..c62b5e0 100644 --- a/src/urllib3/contrib/socks.py +++ b/src/urllib3/contrib/socks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ This module contains provisional support for SOCKS proxies from within urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and @@ -38,10 +37,11 @@ proxy_url="socks5h://:@proxy-host" """ -from __future__ import absolute_import + +from __future__ import annotations try: - import socks + import socks # type: ignore[import-not-found] except ImportError: import warnings @@ -51,13 +51,13 @@ ( "SOCKS support in urllib3 requires the installation of optional " "dependencies: specifically, PySocks. For more information, see " - "https://urllib3.readthedocs.io/en/1.26.x/contrib.html#socks-proxies" + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html#socks-proxies" ), DependencyWarning, ) raise -from socket import error as SocketError +import typing from socket import timeout as SocketTimeout from ..connection import HTTPConnection, HTTPSConnection @@ -69,7 +69,16 @@ try: import ssl except ImportError: - ssl = None + ssl = None # type: ignore[assignment] + + +class _TYPE_SOCKS_OPTIONS(typing.TypedDict): + socks_version: int + proxy_host: str | None + proxy_port: str | None + username: str | None + password: str | None + rdns: bool class SOCKSConnection(HTTPConnection): @@ -77,15 +86,20 @@ class SOCKSConnection(HTTPConnection): A plain-text HTTP connection that connects via a SOCKS proxy. """ - def __init__(self, *args, **kwargs): - self._socks_options = kwargs.pop("_socks_options") - super(SOCKSConnection, self).__init__(*args, **kwargs) - - def _new_conn(self): + def __init__( + self, + _socks_options: _TYPE_SOCKS_OPTIONS, + *args: typing.Any, + **kwargs: typing.Any, + ) -> None: + self._socks_options = _socks_options + super().__init__(*args, **kwargs) + + def _new_conn(self) -> socks.socksocket: """ Establish a new connection via the SOCKS proxy. """ - extra_kw = {} + extra_kw: dict[str, typing.Any] = {} if self.source_address: extra_kw["source_address"] = self.source_address @@ -102,15 +116,14 @@ def _new_conn(self): proxy_password=self._socks_options["password"], proxy_rdns=self._socks_options["rdns"], timeout=self.timeout, - **extra_kw + **extra_kw, ) - except SocketTimeout: + except SocketTimeout as e: raise ConnectTimeoutError( self, - "Connection to %s timed out. (connect timeout=%s)" - % (self.host, self.timeout), - ) + f"Connection to {self.host} timed out. (connect timeout={self.timeout})", + ) from e except socks.ProxyError as e: # This is fragile as hell, but it seems to be the only way to raise @@ -120,22 +133,23 @@ def _new_conn(self): if isinstance(error, SocketTimeout): raise ConnectTimeoutError( self, - "Connection to %s timed out. (connect timeout=%s)" - % (self.host, self.timeout), - ) + f"Connection to {self.host} timed out. (connect timeout={self.timeout})", + ) from e else: + # Adding `from e` messes with coverage somehow, so it's omitted. + # See #2386. raise NewConnectionError( - self, "Failed to establish a new connection: %s" % error + self, f"Failed to establish a new connection: {error}" ) else: raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e - ) + self, f"Failed to establish a new connection: {e}" + ) from e - except SocketError as e: # Defensive: PySocks should catch all these. + except OSError as e: # Defensive: PySocks should catch all these. raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e - ) + self, f"Failed to establish a new connection: {e}" + ) from e return conn @@ -169,12 +183,12 @@ class SOCKSProxyManager(PoolManager): def __init__( self, - proxy_url, - username=None, - password=None, - num_pools=10, - headers=None, - **connection_pool_kw + proxy_url: str, + username: str | None = None, + password: str | None = None, + num_pools: int = 10, + headers: typing.Mapping[str, str] | None = None, + **connection_pool_kw: typing.Any, ): parsed = parse_url(proxy_url) @@ -195,7 +209,7 @@ def __init__( socks_version = socks.PROXY_TYPE_SOCKS4 rdns = True else: - raise ValueError("Unable to determine SOCKS version from %s" % proxy_url) + raise ValueError(f"Unable to determine SOCKS version from {proxy_url}") self.proxy_url = proxy_url @@ -209,8 +223,6 @@ def __init__( } connection_pool_kw["_socks_options"] = socks_options - super(SOCKSProxyManager, self).__init__( - num_pools, headers, **connection_pool_kw - ) + super().__init__(num_pools, headers, **connection_pool_kw) self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme diff --git a/src/urllib3/exceptions.py b/src/urllib3/exceptions.py index cba6f3f..0394578 100644 --- a/src/urllib3/exceptions.py +++ b/src/urllib3/exceptions.py @@ -1,6 +1,16 @@ -from __future__ import absolute_import +from __future__ import annotations -from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead +import socket +import typing +import warnings +from email.errors import MessageDefect +from http.client import IncompleteRead as httplib_IncompleteRead + +if typing.TYPE_CHECKING: + from .connection import HTTPConnection + from .connectionpool import ConnectionPool + from .response import HTTPResponse + from .util.retry import Retry # Base Exceptions @@ -8,23 +18,22 @@ class HTTPError(Exception): """Base exception used by this module.""" - pass - class HTTPWarning(Warning): """Base warning used by this module.""" - pass + +_TYPE_REDUCE_RESULT = tuple[typing.Callable[..., object], tuple[object, ...]] class PoolError(HTTPError): """Base exception for errors caused within a pool.""" - def __init__(self, pool, message): + def __init__(self, pool: ConnectionPool, message: str) -> None: self.pool = pool - HTTPError.__init__(self, "%s: %s" % (pool, message)) + super().__init__(f"{pool}: {message}") - def __reduce__(self): + def __reduce__(self) -> _TYPE_REDUCE_RESULT: # For pickling purposes. return self.__class__, (None, None) @@ -32,11 +41,11 @@ def __reduce__(self): class RequestError(PoolError): """Base exception for PoolErrors that have associated URLs.""" - def __init__(self, pool, url, message): + def __init__(self, pool: ConnectionPool, url: str, message: str) -> None: self.url = url - PoolError.__init__(self, pool, message) + super().__init__(pool, message) - def __reduce__(self): + def __reduce__(self) -> _TYPE_REDUCE_RESULT: # For pickling purposes. return self.__class__, (None, self.url, None) @@ -44,28 +53,25 @@ def __reduce__(self): class SSLError(HTTPError): """Raised when SSL certificate fails in an HTTPS connection.""" - pass - class ProxyError(HTTPError): """Raised when the connection to a proxy fails.""" - def __init__(self, message, error, *args): - super(ProxyError, self).__init__(message, error, *args) + # The original error is also available as __cause__. + original_error: Exception + + def __init__(self, message: str, error: Exception) -> None: + super().__init__(message, error) self.original_error = error class DecodeError(HTTPError): """Raised when automatic decoding based on Content-Type fails.""" - pass - class ProtocolError(HTTPError): """Raised when something unexpected happens mid-request/response.""" - pass - #: Renamed to ProtocolError but aliased for backwards compatibility. ConnectionError = ProtocolError @@ -79,33 +85,36 @@ class MaxRetryError(RequestError): :param pool: The connection pool :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool` - :param string url: The requested Url - :param exceptions.Exception reason: The underlying error + :param str url: The requested Url + :param reason: The underlying error + :type reason: :class:`Exception` """ - def __init__(self, pool, url, reason=None): + def __init__( + self, pool: ConnectionPool, url: str, reason: Exception | None = None + ) -> None: self.reason = reason - message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason) + message = f"Max retries exceeded with url: {url} (Caused by {reason!r})" - RequestError.__init__(self, pool, url, message) + super().__init__(pool, url, message) class HostChangedError(RequestError): """Raised when an existing pool gets a request for a foreign host.""" - def __init__(self, pool, url, retries=3): - message = "Tried to open a foreign host with url: %s" % url - RequestError.__init__(self, pool, url, message) + def __init__( + self, pool: ConnectionPool, url: str, retries: Retry | int = 3 + ) -> None: + message = f"Tried to open a foreign host with url: {url}" + super().__init__(pool, url, message) self.retries = retries class TimeoutStateError(HTTPError): """Raised when passing an invalid state to a timeout""" - pass - class TimeoutError(HTTPError): """Raised when a socket timeout error occurs. @@ -114,53 +123,74 @@ class TimeoutError(HTTPError): ` and :exc:`ConnectTimeoutErrors `. """ - pass - class ReadTimeoutError(TimeoutError, RequestError): """Raised when a socket timeout occurs while receiving data from a server""" - pass - # This timeout error does not have a URL attached and needs to inherit from the # base HTTPError class ConnectTimeoutError(TimeoutError): """Raised when a socket timeout occurs while connecting to a server""" - pass - -class NewConnectionError(ConnectTimeoutError, PoolError): +class NewConnectionError(ConnectTimeoutError, HTTPError): """Raised when we fail to establish a new connection. Usually ECONNREFUSED.""" - pass + def __init__(self, conn: HTTPConnection, message: str) -> None: + self.conn = conn + super().__init__(f"{conn}: {message}") + + def __reduce__(self) -> _TYPE_REDUCE_RESULT: + # For pickling purposes. + return self.__class__, (None, None) + + @property + def pool(self) -> HTTPConnection: + warnings.warn( + "The 'pool' property is deprecated and will be removed " + "in urllib3 v2.1.0. Use 'conn' instead.", + DeprecationWarning, + stacklevel=2, + ) + + return self.conn + + +class NameResolutionError(NewConnectionError): + """Raised when host name resolution fails.""" + + def __init__(self, host: str, conn: HTTPConnection, reason: socket.gaierror): + message = f"Failed to resolve '{host}' ({reason})" + super().__init__(conn, message) + + def __reduce__(self) -> _TYPE_REDUCE_RESULT: + # For pickling purposes. + return self.__class__, (None, None, None) class EmptyPoolError(PoolError): """Raised when a pool runs out of connections and no more are allowed.""" - pass + +class FullPoolError(PoolError): + """Raised when we try to add a connection to a full pool in blocking mode.""" class ClosedPoolError(PoolError): """Raised when a request enters a pool after the pool has been closed.""" - pass - class LocationValueError(ValueError, HTTPError): """Raised when there is something wrong with a given URL input.""" - pass - class LocationParseError(LocationValueError): """Raised when get_host or similar fails to parse the URL input.""" - def __init__(self, location): - message = "Failed to parse: %s" % location - HTTPError.__init__(self, message) + def __init__(self, location: str) -> None: + message = f"Failed to parse: {location}" + super().__init__(message) self.location = location @@ -168,9 +198,9 @@ def __init__(self, location): class URLSchemeUnknown(LocationValueError): """Raised when a URL input has an unsupported scheme.""" - def __init__(self, scheme): - message = "Not supported URL scheme %s" % scheme - super(URLSchemeUnknown, self).__init__(message) + def __init__(self, scheme: str): + message = f"Not supported URL scheme {scheme}" + super().__init__(message) self.scheme = scheme @@ -185,38 +215,22 @@ class ResponseError(HTTPError): class SecurityWarning(HTTPWarning): """Warned when performing security reducing actions""" - pass - - -class SubjectAltNameWarning(SecurityWarning): - """Warned when connecting to a host with a certificate missing a SAN.""" - - pass - class InsecureRequestWarning(SecurityWarning): """Warned when making an unverified HTTPS request.""" - pass + +class NotOpenSSLWarning(SecurityWarning): + """Warned when using unsupported SSL library""" class SystemTimeWarning(SecurityWarning): """Warned when system time is suspected to be wrong""" - pass - class InsecurePlatformWarning(SecurityWarning): """Warned when certain TLS/SSL configuration is not available on a platform.""" - pass - - -class SNIMissingWarning(HTTPWarning): - """Warned when making a HTTPS request without SNI available.""" - - pass - class DependencyWarning(HTTPWarning): """ @@ -224,14 +238,10 @@ class DependencyWarning(HTTPWarning): dependencies. """ - pass - class ResponseNotChunked(ProtocolError, ValueError): """Response needs to be chunked in order to read it as chunks.""" - pass - class BodyNotHttplibCompatible(HTTPError): """ @@ -239,8 +249,6 @@ class BodyNotHttplibCompatible(HTTPError): (have an fp attribute which returns raw chunks) for read_chunked(). """ - pass - class IncompleteRead(HTTPError, httplib_IncompleteRead): """ @@ -250,10 +258,14 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead): for ``partial`` to avoid creating large objects on streamed reads. """ - def __init__(self, partial, expected): - super(IncompleteRead, self).__init__(partial, expected) + partial: int # type: ignore[assignment] + expected: int - def __repr__(self): + def __init__(self, partial: int, expected: int) -> None: + self.partial = partial + self.expected = expected + + def __repr__(self) -> str: return "IncompleteRead(%i bytes read, %i more expected)" % ( self.partial, self.expected, @@ -263,14 +275,13 @@ def __repr__(self): class InvalidChunkLength(HTTPError, httplib_IncompleteRead): """Invalid chunk length in a chunked response.""" - def __init__(self, response, length): - super(InvalidChunkLength, self).__init__( - response.tell(), response.length_remaining - ) + def __init__(self, response: HTTPResponse, length: bytes) -> None: + self.partial: int = response.tell() # type: ignore[assignment] + self.expected: int | None = response.length_remaining self.response = response self.length = length - def __repr__(self): + def __repr__(self) -> str: return "InvalidChunkLength(got length %r, %i bytes read)" % ( self.length, self.partial, @@ -280,15 +291,13 @@ def __repr__(self): class InvalidHeader(HTTPError): """The header provided was somehow invalid.""" - pass - class ProxySchemeUnknown(AssertionError, URLSchemeUnknown): """ProxyManager does not support the supplied scheme""" # TODO(t-8ch): Stop inheriting from AssertionError in v2.0. - def __init__(self, scheme): + def __init__(self, scheme: str | None) -> None: # 'localhost' is here because our URL parser parses # localhost:8080 -> scheme=localhost, remove if we fix this. if scheme == "localhost": @@ -296,28 +305,23 @@ def __init__(self, scheme): if scheme is None: message = "Proxy URL had no scheme, should start with http:// or https://" else: - message = ( - "Proxy URL had unsupported scheme %s, should use http:// or https://" - % scheme - ) - super(ProxySchemeUnknown, self).__init__(message) + message = f"Proxy URL had unsupported scheme {scheme}, should use http:// or https://" + super().__init__(message) class ProxySchemeUnsupported(ValueError): """Fetching HTTPS resources through HTTPS proxies is unsupported""" - pass - class HeaderParsingError(HTTPError): """Raised by assert_header_parsing, but we convert it to a log.warning statement.""" - def __init__(self, defects, unparsed_data): - message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data) - super(HeaderParsingError, self).__init__(message) + def __init__( + self, defects: list[MessageDefect], unparsed_data: bytes | str | None + ) -> None: + message = f"{defects or 'Unknown'}, unparsed data: {unparsed_data!r}" + super().__init__(message) class UnrewindableBodyError(HTTPError): """urllib3 encountered an error when trying to rewind a body""" - - pass diff --git a/src/urllib3/fields.py b/src/urllib3/fields.py index 9d630f4..97c4730 100644 --- a/src/urllib3/fields.py +++ b/src/urllib3/fields.py @@ -1,13 +1,20 @@ -from __future__ import absolute_import +from __future__ import annotations import email.utils import mimetypes -import re +import typing -from .packages import six +_TYPE_FIELD_VALUE = typing.Union[str, bytes] +_TYPE_FIELD_VALUE_TUPLE = typing.Union[ + _TYPE_FIELD_VALUE, + tuple[str, _TYPE_FIELD_VALUE], + tuple[str, _TYPE_FIELD_VALUE, str], +] -def guess_content_type(filename, default="application/octet-stream"): +def guess_content_type( + filename: str | None, default: str = "application/octet-stream" +) -> str: """ Guess the "Content-Type" of a file. @@ -21,7 +28,7 @@ def guess_content_type(filename, default="application/octet-stream"): return default -def format_header_param_rfc2231(name, value): +def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str: """ Helper function to format and quote a single header parameter using the strategy defined in RFC 2231. @@ -34,14 +41,28 @@ def format_header_param_rfc2231(name, value): The name of the parameter, a string expected to be ASCII only. :param value: The value of the parameter, provided as ``bytes`` or `str``. - :ret: + :returns: An RFC-2231-formatted unicode string. + + .. deprecated:: 2.0.0 + Will be removed in urllib3 v2.1.0. This is not valid for + ``multipart/form-data`` header parameters. """ - if isinstance(value, six.binary_type): + import warnings + + warnings.warn( + "'format_header_param_rfc2231' is deprecated and will be " + "removed in urllib3 v2.1.0. This is not valid for " + "multipart/form-data header parameters.", + DeprecationWarning, + stacklevel=2, + ) + + if isinstance(value, bytes): value = value.decode("utf-8") if not any(ch in value for ch in '"\\\r\n'): - result = u'%s="%s"' % (name, value) + result = f'{name}="{value}"' try: result.encode("ascii") except (UnicodeEncodeError, UnicodeDecodeError): @@ -49,81 +70,87 @@ def format_header_param_rfc2231(name, value): else: return result - if six.PY2: # Python 2: - value = value.encode("utf-8") - - # encode_rfc2231 accepts an encoded string and returns an ascii-encoded - # string in Python 2 but accepts and returns unicode strings in Python 3 value = email.utils.encode_rfc2231(value, "utf-8") - value = "%s*=%s" % (name, value) - - if six.PY2: # Python 2: - value = value.decode("utf-8") + value = f"{name}*={value}" return value -_HTML5_REPLACEMENTS = { - u"\u0022": u"%22", - # Replace "\" with "\\". - u"\u005C": u"\u005C\u005C", -} - -# All control characters from 0x00 to 0x1F *except* 0x1B. -_HTML5_REPLACEMENTS.update( - { - six.unichr(cc): u"%{:02X}".format(cc) - for cc in range(0x00, 0x1F + 1) - if cc not in (0x1B,) - } -) - - -def _replace_multiple(value, needles_and_replacements): - def replacer(match): - return needles_and_replacements[match.group(0)] - - pattern = re.compile( - r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()]) - ) - - result = pattern.sub(replacer, value) - - return result - - -def format_header_param_html5(name, value): +def format_multipart_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str: """ - Helper function to format and quote a single header parameter using the - HTML5 strategy. + Format and quote a single multipart header parameter. - Particularly useful for header parameters which might contain - non-ASCII values, like file names. This follows the `HTML5 Working Draft - Section 4.10.22.7`_ and matches the behavior of curl and modern browsers. + This follows the `WHATWG HTML Standard`_ as of 2021/06/10, matching + the behavior of current browser and curl versions. Values are + assumed to be UTF-8. The ``\\n``, ``\\r``, and ``"`` characters are + percent encoded. - .. _HTML5 Working Draft Section 4.10.22.7: - https://w3c.github.io/html/sec-forms.html#multipart-form-data + .. _WHATWG HTML Standard: + https://html.spec.whatwg.org/multipage/ + form-control-infrastructure.html#multipart-form-data :param name: - The name of the parameter, a string expected to be ASCII only. + The name of the parameter, an ASCII-only ``str``. :param value: - The value of the parameter, provided as ``bytes`` or `str``. - :ret: - A unicode string, stripped of troublesome characters. + The value of the parameter, a ``str`` or UTF-8 encoded + ``bytes``. + :returns: + A string ``name="value"`` with the escaped value. + + .. versionchanged:: 2.0.0 + Matches the WHATWG HTML Standard as of 2021/06/10. Control + characters are no longer percent encoded. + + .. versionchanged:: 2.0.0 + Renamed from ``format_header_param_html5`` and + ``format_header_param``. The old names will be removed in + urllib3 v2.1.0. """ - if isinstance(value, six.binary_type): + if isinstance(value, bytes): value = value.decode("utf-8") - value = _replace_multiple(value, _HTML5_REPLACEMENTS) + # percent encode \n \r " + value = value.translate({10: "%0A", 13: "%0D", 34: "%22"}) + return f'{name}="{value}"' - return u'%s="%s"' % (name, value) + +def format_header_param_html5(name: str, value: _TYPE_FIELD_VALUE) -> str: + """ + .. deprecated:: 2.0.0 + Renamed to :func:`format_multipart_header_param`. Will be + removed in urllib3 v2.1.0. + """ + import warnings + + warnings.warn( + "'format_header_param_html5' has been renamed to " + "'format_multipart_header_param'. The old name will be " + "removed in urllib3 v2.1.0.", + DeprecationWarning, + stacklevel=2, + ) + return format_multipart_header_param(name, value) -# For backwards-compatibility. -format_header_param = format_header_param_html5 +def format_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str: + """ + .. deprecated:: 2.0.0 + Renamed to :func:`format_multipart_header_param`. Will be + removed in urllib3 v2.1.0. + """ + import warnings + + warnings.warn( + "'format_header_param' has been renamed to " + "'format_multipart_header_param'. The old name will be " + "removed in urllib3 v2.1.0.", + DeprecationWarning, + stacklevel=2, + ) + return format_multipart_header_param(name, value) -class RequestField(object): +class RequestField: """ A data container for request body parameters. @@ -135,29 +162,47 @@ class RequestField(object): An optional filename of the request field. Must be unicode. :param headers: An optional dict-like object of headers to initially use for the field. - :param header_formatter: - An optional callable that is used to encode and format the headers. By - default, this is :func:`format_header_param_html5`. + + .. versionchanged:: 2.0.0 + The ``header_formatter`` parameter is deprecated and will + be removed in urllib3 v2.1.0. """ def __init__( self, - name, - data, - filename=None, - headers=None, - header_formatter=format_header_param_html5, + name: str, + data: _TYPE_FIELD_VALUE, + filename: str | None = None, + headers: typing.Mapping[str, str] | None = None, + header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None, ): self._name = name self._filename = filename self.data = data - self.headers = {} + self.headers: dict[str, str | None] = {} if headers: self.headers = dict(headers) - self.header_formatter = header_formatter + + if header_formatter is not None: + import warnings + + warnings.warn( + "The 'header_formatter' parameter is deprecated and " + "will be removed in urllib3 v2.1.0.", + DeprecationWarning, + stacklevel=2, + ) + self.header_formatter = header_formatter + else: + self.header_formatter = format_multipart_header_param @classmethod - def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5): + def from_tuples( + cls, + fieldname: str, + value: _TYPE_FIELD_VALUE_TUPLE, + header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None, + ) -> RequestField: """ A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. @@ -174,6 +219,10 @@ def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html Field names and filenames must be unicode. """ + filename: str | None + content_type: str | None + data: _TYPE_FIELD_VALUE + if isinstance(value, tuple): if len(value) == 3: filename, data, content_type = value @@ -192,20 +241,29 @@ def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html return request_param - def _render_part(self, name, value): + def _render_part(self, name: str, value: _TYPE_FIELD_VALUE) -> str: """ - Overridable helper function to format a single header parameter. By - default, this calls ``self.header_formatter``. + Override this method to change how each multipart header + parameter is formatted. By default, this calls + :func:`format_multipart_header_param`. :param name: - The name of the parameter, a string expected to be ASCII only. + The name of the parameter, an ASCII-only ``str``. :param value: - The value of the parameter, provided as a unicode string. - """ + The value of the parameter, a ``str`` or UTF-8 encoded + ``bytes``. + :meta public: + """ return self.header_formatter(name, value) - def _render_parts(self, header_parts): + def _render_parts( + self, + header_parts: ( + dict[str, _TYPE_FIELD_VALUE | None] + | typing.Sequence[tuple[str, _TYPE_FIELD_VALUE | None]] + ), + ) -> str: """ Helper function to format and quote a single header. @@ -216,18 +274,21 @@ def _render_parts(self, header_parts): A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format as `k1="v1"; k2="v2"; ...`. """ + iterable: typing.Iterable[tuple[str, _TYPE_FIELD_VALUE | None]] + parts = [] - iterable = header_parts if isinstance(header_parts, dict): iterable = header_parts.items() + else: + iterable = header_parts for name, value in iterable: if value is not None: parts.append(self._render_part(name, value)) - return u"; ".join(parts) + return "; ".join(parts) - def render_headers(self): + def render_headers(self) -> str: """ Renders the headers for this request field. """ @@ -236,39 +297,45 @@ def render_headers(self): sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"] for sort_key in sort_keys: if self.headers.get(sort_key, False): - lines.append(u"%s: %s" % (sort_key, self.headers[sort_key])) + lines.append(f"{sort_key}: {self.headers[sort_key]}") for header_name, header_value in self.headers.items(): if header_name not in sort_keys: if header_value: - lines.append(u"%s: %s" % (header_name, header_value)) + lines.append(f"{header_name}: {header_value}") - lines.append(u"\r\n") - return u"\r\n".join(lines) + lines.append("\r\n") + return "\r\n".join(lines) def make_multipart( - self, content_disposition=None, content_type=None, content_location=None - ): + self, + content_disposition: str | None = None, + content_type: str | None = None, + content_location: str | None = None, + ) -> None: """ Makes this request field into a multipart request field. This method overrides "Content-Disposition", "Content-Type" and "Content-Location" headers to the request parameter. + :param content_disposition: + The 'Content-Disposition' of the request body. Defaults to 'form-data' :param content_type: The 'Content-Type' of the request body. :param content_location: The 'Content-Location' of the request body. """ - self.headers["Content-Disposition"] = content_disposition or u"form-data" - self.headers["Content-Disposition"] += u"; ".join( + content_disposition = (content_disposition or "form-data") + "; ".join( [ - u"", + "", self._render_parts( - ((u"name", self._name), (u"filename", self._filename)) + (("name", self._name), ("filename", self._filename)) ), ] ) + + self.headers["Content-Disposition"] = content_disposition self.headers["Content-Type"] = content_type self.headers["Content-Location"] = content_location diff --git a/src/urllib3/filepost.py b/src/urllib3/filepost.py index 36c9252..14f70b0 100644 --- a/src/urllib3/filepost.py +++ b/src/urllib3/filepost.py @@ -1,28 +1,32 @@ -from __future__ import absolute_import +from __future__ import annotations import binascii import codecs import os +import typing from io import BytesIO -from .fields import RequestField -from .packages import six -from .packages.six import b +from .fields import _TYPE_FIELD_VALUE_TUPLE, RequestField writer = codecs.lookup("utf-8")[3] +_TYPE_FIELDS_SEQUENCE = typing.Sequence[ + typing.Union[tuple[str, _TYPE_FIELD_VALUE_TUPLE], RequestField] +] +_TYPE_FIELDS = typing.Union[ + _TYPE_FIELDS_SEQUENCE, + typing.Mapping[str, _TYPE_FIELD_VALUE_TUPLE], +] -def choose_boundary(): + +def choose_boundary() -> str: """ Our embarrassingly-simple replacement for mimetools.choose_boundary. """ - boundary = binascii.hexlify(os.urandom(16)) - if not six.PY2: - boundary = boundary.decode("ascii") - return boundary + return binascii.hexlify(os.urandom(16)).decode() -def iter_field_objects(fields): +def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]: """ Iterate over fields. @@ -30,42 +34,29 @@ def iter_field_objects(fields): :class:`~urllib3.fields.RequestField`. """ - if isinstance(fields, dict): - i = six.iteritems(fields) + iterable: typing.Iterable[RequestField | tuple[str, _TYPE_FIELD_VALUE_TUPLE]] + + if isinstance(fields, typing.Mapping): + iterable = fields.items() else: - i = iter(fields) + iterable = fields - for field in i: + for field in iterable: if isinstance(field, RequestField): yield field else: yield RequestField.from_tuples(*field) -def iter_fields(fields): - """ - .. deprecated:: 1.6 - - Iterate over fields. - - The addition of :class:`~urllib3.fields.RequestField` makes this function - obsolete. Instead, use :func:`iter_field_objects`, which returns - :class:`~urllib3.fields.RequestField` objects. - - Supports list of (k, v) tuples and dicts. - """ - if isinstance(fields, dict): - return ((k, v) for k, v in six.iteritems(fields)) - - return ((k, v) for k, v in fields) - - -def encode_multipart_formdata(fields, boundary=None): +def encode_multipart_formdata( + fields: _TYPE_FIELDS, boundary: str | None = None +) -> tuple[bytes, str]: """ Encode a dictionary of ``fields`` using the multipart/form-data MIME format. :param fields: Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`). + Values are processed by :func:`urllib3.fields.RequestField.from_tuples`. :param boundary: If not specified, then a random boundary will be generated using @@ -76,7 +67,7 @@ def encode_multipart_formdata(fields, boundary=None): boundary = choose_boundary() for field in iter_field_objects(fields): - body.write(b("--%s\r\n" % (boundary))) + body.write(f"--{boundary}\r\n".encode("latin-1")) writer(body).write(field.render_headers()) data = field.data @@ -84,15 +75,15 @@ def encode_multipart_formdata(fields, boundary=None): if isinstance(data, int): data = str(data) # Backwards compatibility - if isinstance(data, six.text_type): + if isinstance(data, str): writer(body).write(data) else: body.write(data) body.write(b"\r\n") - body.write(b("--%s--\r\n" % (boundary))) + body.write(f"--{boundary}--\r\n".encode("latin-1")) - content_type = str("multipart/form-data; boundary=%s" % boundary) + content_type = f"multipart/form-data; boundary={boundary}" return body.getvalue(), content_type diff --git a/src/urllib3/http2/__init__.py b/src/urllib3/http2/__init__.py new file mode 100644 index 0000000..133e1d8 --- /dev/null +++ b/src/urllib3/http2/__init__.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from importlib.metadata import version + +__all__ = [ + "inject_into_urllib3", + "extract_from_urllib3", +] + +import typing + +orig_HTTPSConnection: typing.Any = None + + +def inject_into_urllib3() -> None: + # First check if h2 version is valid + h2_version = version("h2") + if not h2_version.startswith("4."): + raise ImportError( + "urllib3 v2 supports h2 version 4.x.x, currently " + f"the 'h2' module is compiled with {h2_version!r}. " + "See: https://github.com/urllib3/urllib3/issues/3290" + ) + + # Import here to avoid circular dependencies. + from .. import connection as urllib3_connection + from .. import util as urllib3_util + from ..connectionpool import HTTPSConnectionPool + from ..util import ssl_ as urllib3_util_ssl + from .connection import HTTP2Connection + + global orig_HTTPSConnection + orig_HTTPSConnection = urllib3_connection.HTTPSConnection + + HTTPSConnectionPool.ConnectionCls = HTTP2Connection + urllib3_connection.HTTPSConnection = HTTP2Connection # type: ignore[misc] + + # TODO: Offer 'http/1.1' as well, but for testing purposes this is handy. + urllib3_util.ALPN_PROTOCOLS = ["h2"] + urllib3_util_ssl.ALPN_PROTOCOLS = ["h2"] + + +def extract_from_urllib3() -> None: + from .. import connection as urllib3_connection + from .. import util as urllib3_util + from ..connectionpool import HTTPSConnectionPool + from ..util import ssl_ as urllib3_util_ssl + + HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection + urllib3_connection.HTTPSConnection = orig_HTTPSConnection # type: ignore[misc] + + urllib3_util.ALPN_PROTOCOLS = ["http/1.1"] + urllib3_util_ssl.ALPN_PROTOCOLS = ["http/1.1"] diff --git a/src/urllib3/http2/connection.py b/src/urllib3/http2/connection.py new file mode 100644 index 0000000..f486145 --- /dev/null +++ b/src/urllib3/http2/connection.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +import logging +import re +import threading +import types +import typing + +import h2.config # type: ignore[import-untyped] +import h2.connection # type: ignore[import-untyped] +import h2.events # type: ignore[import-untyped] + +from .._base_connection import _TYPE_BODY +from .._collections import HTTPHeaderDict +from ..connection import HTTPSConnection, _get_default_user_agent +from ..exceptions import ConnectionError +from ..response import BaseHTTPResponse + +orig_HTTPSConnection = HTTPSConnection + +T = typing.TypeVar("T") + +log = logging.getLogger(__name__) + +RE_IS_LEGAL_HEADER_NAME = re.compile(rb"^[!#$%&'*+\-.^_`|~0-9a-z]+$") +RE_IS_ILLEGAL_HEADER_VALUE = re.compile(rb"[\0\x00\x0a\x0d\r\n]|^[ \r\n\t]|[ \r\n\t]$") + + +def _is_legal_header_name(name: bytes) -> bool: + """ + "An implementation that validates fields according to the definitions in Sections + 5.1 and 5.5 of [HTTP] only needs an additional check that field names do not + include uppercase characters." (https://httpwg.org/specs/rfc9113.html#n-field-validity) + + `http.client._is_legal_header_name` does not validate the field name according to the + HTTP 1.1 spec, so we do that here, in addition to checking for uppercase characters. + + This does not allow for the `:` character in the header name, so should not + be used to validate pseudo-headers. + """ + return bool(RE_IS_LEGAL_HEADER_NAME.match(name)) + + +def _is_illegal_header_value(value: bytes) -> bool: + """ + "A field value MUST NOT contain the zero value (ASCII NUL, 0x00), line feed + (ASCII LF, 0x0a), or carriage return (ASCII CR, 0x0d) at any position. A field + value MUST NOT start or end with an ASCII whitespace character (ASCII SP or HTAB, + 0x20 or 0x09)." (https://httpwg.org/specs/rfc9113.html#n-field-validity) + """ + return bool(RE_IS_ILLEGAL_HEADER_VALUE.search(value)) + + +class _LockedObject(typing.Generic[T]): + """ + A wrapper class that hides a specific object behind a lock. + The goal here is to provide a simple way to protect access to an object + that cannot safely be simultaneously accessed from multiple threads. The + intended use of this class is simple: take hold of it with a context + manager, which returns the protected object. + """ + + __slots__ = ( + "lock", + "_obj", + ) + + def __init__(self, obj: T): + self.lock = threading.RLock() + self._obj = obj + + def __enter__(self) -> T: + self.lock.acquire() + return self._obj + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> None: + self.lock.release() + + +class HTTP2Connection(HTTPSConnection): + def __init__( + self, host: str, port: int | None = None, **kwargs: typing.Any + ) -> None: + self._h2_conn = self._new_h2_conn() + self._h2_stream: int | None = None + self._headers: list[tuple[bytes, bytes]] = [] + + if "proxy" in kwargs or "proxy_config" in kwargs: # Defensive: + raise NotImplementedError("Proxies aren't supported with HTTP/2") + + super().__init__(host, port, **kwargs) + + if self._tunnel_host is not None: + raise NotImplementedError("Tunneling isn't supported with HTTP/2") + + def _new_h2_conn(self) -> _LockedObject[h2.connection.H2Connection]: + config = h2.config.H2Configuration(client_side=True) + return _LockedObject(h2.connection.H2Connection(config=config)) + + def connect(self) -> None: + super().connect() + with self._h2_conn as conn: + conn.initiate_connection() + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + + def putrequest( # type: ignore[override] + self, + method: str, + url: str, + **kwargs: typing.Any, + ) -> None: + """putrequest + This deviates from the HTTPConnection method signature since we never need to override + sending accept-encoding headers or the host header. + """ + if "skip_host" in kwargs: + raise NotImplementedError("`skip_host` isn't supported") + if "skip_accept_encoding" in kwargs: + raise NotImplementedError("`skip_accept_encoding` isn't supported") + + self._request_url = url or "/" + self._validate_path(url) # type: ignore[attr-defined] + + if ":" in self.host: + authority = f"[{self.host}]:{self.port or 443}" + else: + authority = f"{self.host}:{self.port or 443}" + + self._headers.append((b":scheme", b"https")) + self._headers.append((b":method", method.encode())) + self._headers.append((b":authority", authority.encode())) + self._headers.append((b":path", url.encode())) + + with self._h2_conn as conn: + self._h2_stream = conn.get_next_available_stream_id() + + def putheader(self, header: str | bytes, *values: str | bytes) -> None: + # TODO SKIPPABLE_HEADERS from urllib3 are ignored. + header = header.encode() if isinstance(header, str) else header + header = header.lower() # A lot of upstream code uses capitalized headers. + if not _is_legal_header_name(header): + raise ValueError(f"Illegal header name {str(header)}") + + for value in values: + value = value.encode() if isinstance(value, str) else value + if _is_illegal_header_value(value): + raise ValueError(f"Illegal header value {str(value)}") + self._headers.append((header, value)) + + def endheaders(self, message_body: typing.Any = None) -> None: # type: ignore[override] + if self._h2_stream is None: + raise ConnectionError("Must call `putrequest` first.") + + with self._h2_conn as conn: + conn.send_headers( + stream_id=self._h2_stream, + headers=self._headers, + end_stream=(message_body is None), + ) + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + self._headers = [] # Reset headers for the next request. + + def send(self, data: typing.Any) -> None: + """Send data to the server. + `data` can be: `str`, `bytes`, an iterable, or file-like objects + that support a .read() method. + """ + if self._h2_stream is None: + raise ConnectionError("Must call `putrequest` first.") + + with self._h2_conn as conn: + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + + if hasattr(data, "read"): # file-like objects + while True: + chunk = data.read(self.blocksize) + if not chunk: + break + if isinstance(chunk, str): + chunk = chunk.encode() # pragma: no cover + conn.send_data(self._h2_stream, chunk, end_stream=False) + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + conn.end_stream(self._h2_stream) + return + + if isinstance(data, str): # str -> bytes + data = data.encode() + + try: + if isinstance(data, bytes): + conn.send_data(self._h2_stream, data, end_stream=True) + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + else: + for chunk in data: + conn.send_data(self._h2_stream, chunk, end_stream=False) + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + conn.end_stream(self._h2_stream) + except TypeError: + raise TypeError( + "`data` should be str, bytes, iterable, or file. got %r" + % type(data) + ) + + def set_tunnel( + self, + host: str, + port: int | None = None, + headers: typing.Mapping[str, str] | None = None, + scheme: str = "http", + ) -> None: + raise NotImplementedError( + "HTTP/2 does not support setting up a tunnel through a proxy" + ) + + def getresponse( # type: ignore[override] + self, + ) -> HTTP2Response: + status = None + data = bytearray() + with self._h2_conn as conn: + end_stream = False + while not end_stream: + # TODO: Arbitrary read value. + if received_data := self.sock.recv(65535): + events = conn.receive_data(received_data) + for event in events: + if isinstance(event, h2.events.ResponseReceived): + headers = HTTPHeaderDict() + for header, value in event.headers: + if header == b":status": + status = int(value.decode()) + else: + headers.add( + header.decode("ascii"), value.decode("ascii") + ) + + elif isinstance(event, h2.events.DataReceived): + data += event.data + conn.acknowledge_received_data( + event.flow_controlled_length, event.stream_id + ) + + elif isinstance(event, h2.events.StreamEnded): + end_stream = True + + if data_to_send := conn.data_to_send(): + self.sock.sendall(data_to_send) + + assert status is not None + return HTTP2Response( + status=status, + headers=headers, + request_url=self._request_url, + data=bytes(data), + ) + + def request( # type: ignore[override] + self, + method: str, + url: str, + body: _TYPE_BODY | None = None, + headers: typing.Mapping[str, str] | None = None, + *, + preload_content: bool = True, + decode_content: bool = True, + enforce_content_length: bool = True, + **kwargs: typing.Any, + ) -> None: + """Send an HTTP/2 request""" + if "chunked" in kwargs: + # TODO this is often present from upstream. + # raise NotImplementedError("`chunked` isn't supported with HTTP/2") + pass + + if self.sock is not None: + self.sock.settimeout(self.timeout) + + self.putrequest(method, url) + + headers = headers or {} + for k, v in headers.items(): + if k.lower() == "transfer-encoding" and v == "chunked": + continue + else: + self.putheader(k, v) + + if b"user-agent" not in dict(self._headers): + self.putheader(b"user-agent", _get_default_user_agent()) + + if body: + self.endheaders(message_body=body) + self.send(body) + else: + self.endheaders() + + def close(self) -> None: + with self._h2_conn as conn: + try: + conn.close_connection() + if data := conn.data_to_send(): + self.sock.sendall(data) + except Exception: + pass + + # Reset all our HTTP/2 connection state. + self._h2_conn = self._new_h2_conn() + self._h2_stream = None + self._headers = [] + + super().close() + + +class HTTP2Response(BaseHTTPResponse): + # TODO: This is a woefully incomplete response object, but works for non-streaming. + def __init__( + self, + status: int, + headers: HTTPHeaderDict, + request_url: str, + data: bytes, + decode_content: bool = False, # TODO: support decoding + ) -> None: + super().__init__( + status=status, + headers=headers, + # Following CPython, we map HTTP versions to major * 10 + minor integers + version=20, + version_string="HTTP/2", + # No reason phrase in HTTP/2 + reason=None, + decode_content=decode_content, + request_url=request_url, + ) + self._data = data + self.length_remaining = 0 + + @property + def data(self) -> bytes: + return self._data + + def get_redirect_location(self) -> None: + return None + + def close(self) -> None: + pass diff --git a/src/urllib3/http2/probe.py b/src/urllib3/http2/probe.py new file mode 100644 index 0000000..9ea9007 --- /dev/null +++ b/src/urllib3/http2/probe.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import threading + + +class _HTTP2ProbeCache: + __slots__ = ( + "_lock", + "_cache_locks", + "_cache_values", + ) + + def __init__(self) -> None: + self._lock = threading.Lock() + self._cache_locks: dict[tuple[str, int], threading.RLock] = {} + self._cache_values: dict[tuple[str, int], bool | None] = {} + + def acquire_and_get(self, host: str, port: int) -> bool | None: + # By the end of this block we know that + # _cache_[values,locks] is available. + value = None + with self._lock: + key = (host, port) + try: + value = self._cache_values[key] + # If it's a known value we return right away. + if value is not None: + return value + except KeyError: + self._cache_locks[key] = threading.RLock() + self._cache_values[key] = None + + # If the value is unknown, we acquire the lock to signal + # to the requesting thread that the probe is in progress + # or that the current thread needs to return their findings. + key_lock = self._cache_locks[key] + key_lock.acquire() + try: + # If the by the time we get the lock the value has been + # updated we want to return the updated value. + value = self._cache_values[key] + + # In case an exception like KeyboardInterrupt is raised here. + except BaseException as e: # Defensive: + assert not isinstance(e, KeyError) # KeyError shouldn't be possible. + key_lock.release() + raise + + return value + + def set_and_release( + self, host: str, port: int, supports_http2: bool | None + ) -> None: + key = (host, port) + key_lock = self._cache_locks[key] + with key_lock: # Uses an RLock, so can be locked again from same thread. + if supports_http2 is None and self._cache_values[key] is not None: + raise ValueError( + "Cannot reset HTTP/2 support for origin after value has been set." + ) # Defensive: not expected in normal usage + + self._cache_values[key] = supports_http2 + key_lock.release() + + def _values(self) -> dict[tuple[str, int], bool | None]: + """This function is for testing purposes only. Gets the current state of the probe cache""" + with self._lock: + return {k: v for k, v in self._cache_values.items()} + + def _reset(self) -> None: + """This function is for testing purposes only. Reset the cache values""" + with self._lock: + self._cache_locks = {} + self._cache_values = {} + + +_HTTP2_PROBE_CACHE = _HTTP2ProbeCache() + +set_and_release = _HTTP2_PROBE_CACHE.set_and_release +acquire_and_get = _HTTP2_PROBE_CACHE.acquire_and_get +_values = _HTTP2_PROBE_CACHE._values +_reset = _HTTP2_PROBE_CACHE._reset + +__all__ = [ + "set_and_release", + "acquire_and_get", +] diff --git a/src/urllib3/packages/__init__.py b/src/urllib3/packages/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/urllib3/packages/backports/__init__.py b/src/urllib3/packages/backports/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/urllib3/packages/backports/makefile.py b/src/urllib3/packages/backports/makefile.py deleted file mode 100644 index b8fb215..0000000 --- a/src/urllib3/packages/backports/makefile.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -""" -backports.makefile -~~~~~~~~~~~~~~~~~~ - -Backports the Python 3 ``socket.makefile`` method for use with anything that -wants to create a "fake" socket object. -""" -import io -from socket import SocketIO - - -def backport_makefile( - self, mode="r", buffering=None, encoding=None, errors=None, newline=None -): - """ - Backport of ``socket.makefile`` from Python 3.5. - """ - if not set(mode) <= {"r", "w", "b"}: - raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) - writing = "w" in mode - reading = "r" in mode or not writing - assert reading or writing - binary = "b" in mode - rawmode = "" - if reading: - rawmode += "r" - if writing: - rawmode += "w" - raw = SocketIO(self, rawmode) - self._makefile_refs += 1 - if buffering is None: - buffering = -1 - if buffering < 0: - buffering = io.DEFAULT_BUFFER_SIZE - if buffering == 0: - if not binary: - raise ValueError("unbuffered streams must be binary") - return raw - if reading and writing: - buffer = io.BufferedRWPair(raw, raw, buffering) - elif reading: - buffer = io.BufferedReader(raw, buffering) - else: - assert writing - buffer = io.BufferedWriter(raw, buffering) - if binary: - return buffer - text = io.TextIOWrapper(buffer, encoding, errors, newline) - text.mode = mode - return text diff --git a/src/urllib3/packages/backports/weakref_finalize.py b/src/urllib3/packages/backports/weakref_finalize.py deleted file mode 100644 index a2f2966..0000000 --- a/src/urllib3/packages/backports/weakref_finalize.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- -""" -backports.weakref_finalize -~~~~~~~~~~~~~~~~~~ - -Backports the Python 3 ``weakref.finalize`` method. -""" -from __future__ import absolute_import - -import itertools -import sys -from weakref import ref - -__all__ = ["weakref_finalize"] - - -class weakref_finalize(object): - """Class for finalization of weakrefable objects - finalize(obj, func, *args, **kwargs) returns a callable finalizer - object which will be called when obj is garbage collected. The - first time the finalizer is called it evaluates func(*arg, **kwargs) - and returns the result. After this the finalizer is dead, and - calling it just returns None. - When the program exits any remaining finalizers for which the - atexit attribute is true will be run in reverse order of creation. - By default atexit is true. - """ - - # Finalizer objects don't have any state of their own. They are - # just used as keys to lookup _Info objects in the registry. This - # ensures that they cannot be part of a ref-cycle. - - __slots__ = () - _registry = {} - _shutdown = False - _index_iter = itertools.count() - _dirty = False - _registered_with_atexit = False - - class _Info(object): - __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") - - def __init__(self, obj, func, *args, **kwargs): - if not self._registered_with_atexit: - # We may register the exit function more than once because - # of a thread race, but that is harmless - import atexit - - atexit.register(self._exitfunc) - weakref_finalize._registered_with_atexit = True - info = self._Info() - info.weakref = ref(obj, self) - info.func = func - info.args = args - info.kwargs = kwargs or None - info.atexit = True - info.index = next(self._index_iter) - self._registry[self] = info - weakref_finalize._dirty = True - - def __call__(self, _=None): - """If alive then mark as dead and return func(*args, **kwargs); - otherwise return None""" - info = self._registry.pop(self, None) - if info and not self._shutdown: - return info.func(*info.args, **(info.kwargs or {})) - - def detach(self): - """If alive then mark as dead and return (obj, func, args, kwargs); - otherwise return None""" - info = self._registry.get(self) - obj = info and info.weakref() - if obj is not None and self._registry.pop(self, None): - return (obj, info.func, info.args, info.kwargs or {}) - - def peek(self): - """If alive then return (obj, func, args, kwargs); - otherwise return None""" - info = self._registry.get(self) - obj = info and info.weakref() - if obj is not None: - return (obj, info.func, info.args, info.kwargs or {}) - - @property - def alive(self): - """Whether finalizer is alive""" - return self in self._registry - - @property - def atexit(self): - """Whether finalizer should be called at exit""" - info = self._registry.get(self) - return bool(info) and info.atexit - - @atexit.setter - def atexit(self, value): - info = self._registry.get(self) - if info: - info.atexit = bool(value) - - def __repr__(self): - info = self._registry.get(self) - obj = info and info.weakref() - if obj is None: - return "<%s object at %#x; dead>" % (type(self).__name__, id(self)) - else: - return "<%s object at %#x; for %r at %#x>" % ( - type(self).__name__, - id(self), - type(obj).__name__, - id(obj), - ) - - @classmethod - def _select_for_exit(cls): - # Return live finalizers marked for exit, oldest first - L = [(f, i) for (f, i) in cls._registry.items() if i.atexit] - L.sort(key=lambda item: item[1].index) - return [f for (f, i) in L] - - @classmethod - def _exitfunc(cls): - # At shutdown invoke finalizers for which atexit is true. - # This is called once all other non-daemonic threads have been - # joined. - reenable_gc = False - try: - if cls._registry: - import gc - - if gc.isenabled(): - reenable_gc = True - gc.disable() - pending = None - while True: - if pending is None or weakref_finalize._dirty: - pending = cls._select_for_exit() - weakref_finalize._dirty = False - if not pending: - break - f = pending.pop() - try: - # gc is disabled, so (assuming no daemonic - # threads) the following is the only line in - # this function which might trigger creation - # of a new finalizer - f() - except Exception: - sys.excepthook(*sys.exc_info()) - assert f not in cls._registry - finally: - # prevent any more finalizers from executing during shutdown - weakref_finalize._shutdown = True - if reenable_gc: - gc.enable() diff --git a/src/urllib3/packages/six.py b/src/urllib3/packages/six.py deleted file mode 100644 index f099a3d..0000000 --- a/src/urllib3/packages/six.py +++ /dev/null @@ -1,1076 +0,0 @@ -# Copyright (c) 2010-2020 Benjamin Peterson -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -"""Utilities for writing code that runs on Python 2 and 3""" - -from __future__ import absolute_import - -import functools -import itertools -import operator -import sys -import types - -__author__ = "Benjamin Peterson " -__version__ = "1.16.0" - - -# Useful for very coarse version differentiation. -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 -PY34 = sys.version_info[0:2] >= (3, 4) - -if PY3: - string_types = (str,) - integer_types = (int,) - class_types = (type,) - text_type = str - binary_type = bytes - - MAXSIZE = sys.maxsize -else: - string_types = (basestring,) - integer_types = (int, long) - class_types = (type, types.ClassType) - text_type = unicode - binary_type = str - - if sys.platform.startswith("java"): - # Jython always uses 32 bits. - MAXSIZE = int((1 << 31) - 1) - else: - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - def __len__(self): - return 1 << 31 - - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) - del X - -if PY34: - from importlib.util import spec_from_loader -else: - spec_from_loader = None - - -def _add_doc(func, doc): - """Add documentation to a function.""" - func.__doc__ = doc - - -def _import_module(name): - """Import module, returning the module after the last dot.""" - __import__(name) - return sys.modules[name] - - -class _LazyDescr(object): - def __init__(self, name): - self.name = name - - def __get__(self, obj, tp): - result = self._resolve() - setattr(obj, self.name, result) # Invokes __set__. - try: - # This is a bit ugly, but it avoids running this again by - # removing this descriptor. - delattr(obj.__class__, self.name) - except AttributeError: - pass - return result - - -class MovedModule(_LazyDescr): - def __init__(self, name, old, new=None): - super(MovedModule, self).__init__(name) - if PY3: - if new is None: - new = name - self.mod = new - else: - self.mod = old - - def _resolve(self): - return _import_module(self.mod) - - def __getattr__(self, attr): - _module = self._resolve() - value = getattr(_module, attr) - setattr(self, attr, value) - return value - - -class _LazyModule(types.ModuleType): - def __init__(self, name): - super(_LazyModule, self).__init__(name) - self.__doc__ = self.__class__.__doc__ - - def __dir__(self): - attrs = ["__doc__", "__name__"] - attrs += [attr.name for attr in self._moved_attributes] - return attrs - - # Subclasses should override this - _moved_attributes = [] - - -class MovedAttribute(_LazyDescr): - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): - super(MovedAttribute, self).__init__(name) - if PY3: - if new_mod is None: - new_mod = name - self.mod = new_mod - if new_attr is None: - if old_attr is None: - new_attr = name - else: - new_attr = old_attr - self.attr = new_attr - else: - self.mod = old_mod - if old_attr is None: - old_attr = name - self.attr = old_attr - - def _resolve(self): - module = _import_module(self.mod) - return getattr(module, self.attr) - - -class _SixMetaPathImporter(object): - - """ - A meta path importer to import six.moves and its submodules. - - This class implements a PEP302 finder and loader. It should be compatible - with Python 2.5 and all existing versions of Python3 - """ - - def __init__(self, six_module_name): - self.name = six_module_name - self.known_modules = {} - - def _add_module(self, mod, *fullnames): - for fullname in fullnames: - self.known_modules[self.name + "." + fullname] = mod - - def _get_module(self, fullname): - return self.known_modules[self.name + "." + fullname] - - def find_module(self, fullname, path=None): - if fullname in self.known_modules: - return self - return None - - def find_spec(self, fullname, path, target=None): - if fullname in self.known_modules: - return spec_from_loader(fullname, self) - return None - - def __get_module(self, fullname): - try: - return self.known_modules[fullname] - except KeyError: - raise ImportError("This loader does not know module " + fullname) - - def load_module(self, fullname): - try: - # in case of a reload - return sys.modules[fullname] - except KeyError: - pass - mod = self.__get_module(fullname) - if isinstance(mod, MovedModule): - mod = mod._resolve() - else: - mod.__loader__ = self - sys.modules[fullname] = mod - return mod - - def is_package(self, fullname): - """ - Return true, if the named module is a package. - - We need this method to get correct spec objects with - Python 3.4 (see PEP451) - """ - return hasattr(self.__get_module(fullname), "__path__") - - def get_code(self, fullname): - """Return None - - Required, if is_package is implemented""" - self.__get_module(fullname) # eventually raises ImportError - return None - - get_source = get_code # same as get_code - - def create_module(self, spec): - return self.load_module(spec.name) - - def exec_module(self, module): - pass - - -_importer = _SixMetaPathImporter(__name__) - - -class _MovedItems(_LazyModule): - - """Lazy loading of moved objects""" - - __path__ = [] # mark as package - - -_moved_attributes = [ - MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), - MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute( - "filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse" - ), - MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), - MovedAttribute("intern", "__builtin__", "sys"), - MovedAttribute("map", "itertools", "builtins", "imap", "map"), - MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), - MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), - MovedAttribute("getoutput", "commands", "subprocess"), - MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute( - "reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload" - ), - MovedAttribute("reduce", "__builtin__", "functools"), - MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), - MovedAttribute("StringIO", "StringIO", "io"), - MovedAttribute("UserDict", "UserDict", "collections"), - MovedAttribute("UserList", "UserList", "collections"), - MovedAttribute("UserString", "UserString", "collections"), - MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute( - "zip_longest", "itertools", "itertools", "izip_longest", "zip_longest" - ), - MovedModule("builtins", "__builtin__"), - MovedModule("configparser", "ConfigParser"), - MovedModule( - "collections_abc", - "collections", - "collections.abc" if sys.version_info >= (3, 3) else "collections", - ), - MovedModule("copyreg", "copy_reg"), - MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), - MovedModule( - "_dummy_thread", - "dummy_thread", - "_dummy_thread" if sys.version_info < (3, 9) else "_thread", - ), - MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), - MovedModule("http_cookies", "Cookie", "http.cookies"), - MovedModule("html_entities", "htmlentitydefs", "html.entities"), - MovedModule("html_parser", "HTMLParser", "html.parser"), - MovedModule("http_client", "httplib", "http.client"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), - MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), - MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule( - "email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart" - ), - MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), - MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), - MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), - MovedModule("cPickle", "cPickle", "pickle"), - MovedModule("queue", "Queue"), - MovedModule("reprlib", "repr"), - MovedModule("socketserver", "SocketServer"), - MovedModule("_thread", "thread", "_thread"), - MovedModule("tkinter", "Tkinter"), - MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), - MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), - MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), - MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), - MovedModule("tkinter_tix", "Tix", "tkinter.tix"), - MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), - MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), - MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog"), - MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), - MovedModule("tkinter_font", "tkFont", "tkinter.font"), - MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"), - MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), - MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), - MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), - MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), - MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), - MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), -] -# Add windows specific modules. -if sys.platform == "win32": - _moved_attributes += [ - MovedModule("winreg", "_winreg"), - ] - -for attr in _moved_attributes: - setattr(_MovedItems, attr.name, attr) - if isinstance(attr, MovedModule): - _importer._add_module(attr, "moves." + attr.name) -del attr - -_MovedItems._moved_attributes = _moved_attributes - -moves = _MovedItems(__name__ + ".moves") -_importer._add_module(moves, "moves") - - -class Module_six_moves_urllib_parse(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_parse""" - - -_urllib_parse_moved_attributes = [ - MovedAttribute("ParseResult", "urlparse", "urllib.parse"), - MovedAttribute("SplitResult", "urlparse", "urllib.parse"), - MovedAttribute("parse_qs", "urlparse", "urllib.parse"), - MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), - MovedAttribute("urldefrag", "urlparse", "urllib.parse"), - MovedAttribute("urljoin", "urlparse", "urllib.parse"), - MovedAttribute("urlparse", "urlparse", "urllib.parse"), - MovedAttribute("urlsplit", "urlparse", "urllib.parse"), - MovedAttribute("urlunparse", "urlparse", "urllib.parse"), - MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), - MovedAttribute("quote", "urllib", "urllib.parse"), - MovedAttribute("quote_plus", "urllib", "urllib.parse"), - MovedAttribute("unquote", "urllib", "urllib.parse"), - MovedAttribute("unquote_plus", "urllib", "urllib.parse"), - MovedAttribute( - "unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes" - ), - MovedAttribute("urlencode", "urllib", "urllib.parse"), - MovedAttribute("splitquery", "urllib", "urllib.parse"), - MovedAttribute("splittag", "urllib", "urllib.parse"), - MovedAttribute("splituser", "urllib", "urllib.parse"), - MovedAttribute("splitvalue", "urllib", "urllib.parse"), - MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), - MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), - MovedAttribute("uses_params", "urlparse", "urllib.parse"), - MovedAttribute("uses_query", "urlparse", "urllib.parse"), - MovedAttribute("uses_relative", "urlparse", "urllib.parse"), -] -for attr in _urllib_parse_moved_attributes: - setattr(Module_six_moves_urllib_parse, attr.name, attr) -del attr - -Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes - -_importer._add_module( - Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), - "moves.urllib_parse", - "moves.urllib.parse", -) - - -class Module_six_moves_urllib_error(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_error""" - - -_urllib_error_moved_attributes = [ - MovedAttribute("URLError", "urllib2", "urllib.error"), - MovedAttribute("HTTPError", "urllib2", "urllib.error"), - MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), -] -for attr in _urllib_error_moved_attributes: - setattr(Module_six_moves_urllib_error, attr.name, attr) -del attr - -Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes - -_importer._add_module( - Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), - "moves.urllib_error", - "moves.urllib.error", -) - - -class Module_six_moves_urllib_request(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_request""" - - -_urllib_request_moved_attributes = [ - MovedAttribute("urlopen", "urllib2", "urllib.request"), - MovedAttribute("install_opener", "urllib2", "urllib.request"), - MovedAttribute("build_opener", "urllib2", "urllib.request"), - MovedAttribute("pathname2url", "urllib", "urllib.request"), - MovedAttribute("url2pathname", "urllib", "urllib.request"), - MovedAttribute("getproxies", "urllib", "urllib.request"), - MovedAttribute("Request", "urllib2", "urllib.request"), - MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), - MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), - MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), - MovedAttribute("BaseHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), - MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), - MovedAttribute("FileHandler", "urllib2", "urllib.request"), - MovedAttribute("FTPHandler", "urllib2", "urllib.request"), - MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), - MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), - MovedAttribute("urlretrieve", "urllib", "urllib.request"), - MovedAttribute("urlcleanup", "urllib", "urllib.request"), - MovedAttribute("URLopener", "urllib", "urllib.request"), - MovedAttribute("FancyURLopener", "urllib", "urllib.request"), - MovedAttribute("proxy_bypass", "urllib", "urllib.request"), - MovedAttribute("parse_http_list", "urllib2", "urllib.request"), - MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), -] -for attr in _urllib_request_moved_attributes: - setattr(Module_six_moves_urllib_request, attr.name, attr) -del attr - -Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes - -_importer._add_module( - Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), - "moves.urllib_request", - "moves.urllib.request", -) - - -class Module_six_moves_urllib_response(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_response""" - - -_urllib_response_moved_attributes = [ - MovedAttribute("addbase", "urllib", "urllib.response"), - MovedAttribute("addclosehook", "urllib", "urllib.response"), - MovedAttribute("addinfo", "urllib", "urllib.response"), - MovedAttribute("addinfourl", "urllib", "urllib.response"), -] -for attr in _urllib_response_moved_attributes: - setattr(Module_six_moves_urllib_response, attr.name, attr) -del attr - -Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes - -_importer._add_module( - Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), - "moves.urllib_response", - "moves.urllib.response", -) - - -class Module_six_moves_urllib_robotparser(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_robotparser""" - - -_urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), -] -for attr in _urllib_robotparser_moved_attributes: - setattr(Module_six_moves_urllib_robotparser, attr.name, attr) -del attr - -Module_six_moves_urllib_robotparser._moved_attributes = ( - _urllib_robotparser_moved_attributes -) - -_importer._add_module( - Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), - "moves.urllib_robotparser", - "moves.urllib.robotparser", -) - - -class Module_six_moves_urllib(types.ModuleType): - - """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" - - __path__ = [] # mark as package - parse = _importer._get_module("moves.urllib_parse") - error = _importer._get_module("moves.urllib_error") - request = _importer._get_module("moves.urllib_request") - response = _importer._get_module("moves.urllib_response") - robotparser = _importer._get_module("moves.urllib_robotparser") - - def __dir__(self): - return ["parse", "error", "request", "response", "robotparser"] - - -_importer._add_module( - Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib" -) - - -def add_move(move): - """Add an item to six.moves.""" - setattr(_MovedItems, move.name, move) - - -def remove_move(name): - """Remove item from six.moves.""" - try: - delattr(_MovedItems, name) - except AttributeError: - try: - del moves.__dict__[name] - except KeyError: - raise AttributeError("no such move, %r" % (name,)) - - -if PY3: - _meth_func = "__func__" - _meth_self = "__self__" - - _func_closure = "__closure__" - _func_code = "__code__" - _func_defaults = "__defaults__" - _func_globals = "__globals__" -else: - _meth_func = "im_func" - _meth_self = "im_self" - - _func_closure = "func_closure" - _func_code = "func_code" - _func_defaults = "func_defaults" - _func_globals = "func_globals" - - -try: - advance_iterator = next -except NameError: - - def advance_iterator(it): - return it.next() - - -next = advance_iterator - - -try: - callable = callable -except NameError: - - def callable(obj): - return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) - - -if PY3: - - def get_unbound_function(unbound): - return unbound - - create_bound_method = types.MethodType - - def create_unbound_method(func, cls): - return func - - Iterator = object -else: - - def get_unbound_function(unbound): - return unbound.im_func - - def create_bound_method(func, obj): - return types.MethodType(func, obj, obj.__class__) - - def create_unbound_method(func, cls): - return types.MethodType(func, None, cls) - - class Iterator(object): - def next(self): - return type(self).__next__(self) - - callable = callable -_add_doc( - get_unbound_function, """Get the function out of a possibly unbound function""" -) - - -get_method_function = operator.attrgetter(_meth_func) -get_method_self = operator.attrgetter(_meth_self) -get_function_closure = operator.attrgetter(_func_closure) -get_function_code = operator.attrgetter(_func_code) -get_function_defaults = operator.attrgetter(_func_defaults) -get_function_globals = operator.attrgetter(_func_globals) - - -if PY3: - - def iterkeys(d, **kw): - return iter(d.keys(**kw)) - - def itervalues(d, **kw): - return iter(d.values(**kw)) - - def iteritems(d, **kw): - return iter(d.items(**kw)) - - def iterlists(d, **kw): - return iter(d.lists(**kw)) - - viewkeys = operator.methodcaller("keys") - - viewvalues = operator.methodcaller("values") - - viewitems = operator.methodcaller("items") -else: - - def iterkeys(d, **kw): - return d.iterkeys(**kw) - - def itervalues(d, **kw): - return d.itervalues(**kw) - - def iteritems(d, **kw): - return d.iteritems(**kw) - - def iterlists(d, **kw): - return d.iterlists(**kw) - - viewkeys = operator.methodcaller("viewkeys") - - viewvalues = operator.methodcaller("viewvalues") - - viewitems = operator.methodcaller("viewitems") - -_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") -_add_doc(itervalues, "Return an iterator over the values of a dictionary.") -_add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc( - iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary." -) - - -if PY3: - - def b(s): - return s.encode("latin-1") - - def u(s): - return s - - unichr = chr - import struct - - int2byte = struct.Struct(">B").pack - del struct - byte2int = operator.itemgetter(0) - indexbytes = operator.getitem - iterbytes = iter - import io - - StringIO = io.StringIO - BytesIO = io.BytesIO - del io - _assertCountEqual = "assertCountEqual" - if sys.version_info[1] <= 1: - _assertRaisesRegex = "assertRaisesRegexp" - _assertRegex = "assertRegexpMatches" - _assertNotRegex = "assertNotRegexpMatches" - else: - _assertRaisesRegex = "assertRaisesRegex" - _assertRegex = "assertRegex" - _assertNotRegex = "assertNotRegex" -else: - - def b(s): - return s - - # Workaround for standalone backslash - - def u(s): - return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape") - - unichr = unichr - int2byte = chr - - def byte2int(bs): - return ord(bs[0]) - - def indexbytes(buf, i): - return ord(buf[i]) - - iterbytes = functools.partial(itertools.imap, ord) - import StringIO - - StringIO = BytesIO = StringIO.StringIO - _assertCountEqual = "assertItemsEqual" - _assertRaisesRegex = "assertRaisesRegexp" - _assertRegex = "assertRegexpMatches" - _assertNotRegex = "assertNotRegexpMatches" -_add_doc(b, """Byte literal""") -_add_doc(u, """Text literal""") - - -def assertCountEqual(self, *args, **kwargs): - return getattr(self, _assertCountEqual)(*args, **kwargs) - - -def assertRaisesRegex(self, *args, **kwargs): - return getattr(self, _assertRaisesRegex)(*args, **kwargs) - - -def assertRegex(self, *args, **kwargs): - return getattr(self, _assertRegex)(*args, **kwargs) - - -def assertNotRegex(self, *args, **kwargs): - return getattr(self, _assertNotRegex)(*args, **kwargs) - - -if PY3: - exec_ = getattr(moves.builtins, "exec") - - def reraise(tp, value, tb=None): - try: - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - finally: - value = None - tb = None - -else: - - def exec_(_code_, _globs_=None, _locs_=None): - """Execute code in a namespace.""" - if _globs_ is None: - frame = sys._getframe(1) - _globs_ = frame.f_globals - if _locs_ is None: - _locs_ = frame.f_locals - del frame - elif _locs_ is None: - _locs_ = _globs_ - exec ("""exec _code_ in _globs_, _locs_""") - - exec_( - """def reraise(tp, value, tb=None): - try: - raise tp, value, tb - finally: - tb = None -""" - ) - - -if sys.version_info[:2] > (3,): - exec_( - """def raise_from(value, from_value): - try: - raise value from from_value - finally: - value = None -""" - ) -else: - - def raise_from(value, from_value): - raise value - - -print_ = getattr(moves.builtins, "print", None) -if print_ is None: - - def print_(*args, **kwargs): - """The new-style print function for Python 2.4 and 2.5.""" - fp = kwargs.pop("file", sys.stdout) - if fp is None: - return - - def write(data): - if not isinstance(data, basestring): - data = str(data) - # If the file has an encoding, encode unicode with it. - if ( - isinstance(fp, file) - and isinstance(data, unicode) - and fp.encoding is not None - ): - errors = getattr(fp, "errors", None) - if errors is None: - errors = "strict" - data = data.encode(fp.encoding, errors) - fp.write(data) - - want_unicode = False - sep = kwargs.pop("sep", None) - if sep is not None: - if isinstance(sep, unicode): - want_unicode = True - elif not isinstance(sep, str): - raise TypeError("sep must be None or a string") - end = kwargs.pop("end", None) - if end is not None: - if isinstance(end, unicode): - want_unicode = True - elif not isinstance(end, str): - raise TypeError("end must be None or a string") - if kwargs: - raise TypeError("invalid keyword arguments to print()") - if not want_unicode: - for arg in args: - if isinstance(arg, unicode): - want_unicode = True - break - if want_unicode: - newline = unicode("\n") - space = unicode(" ") - else: - newline = "\n" - space = " " - if sep is None: - sep = space - if end is None: - end = newline - for i, arg in enumerate(args): - if i: - write(sep) - write(arg) - write(end) - - -if sys.version_info[:2] < (3, 3): - _print = print_ - - def print_(*args, **kwargs): - fp = kwargs.get("file", sys.stdout) - flush = kwargs.pop("flush", False) - _print(*args, **kwargs) - if flush and fp is not None: - fp.flush() - - -_add_doc(reraise, """Reraise an exception.""") - -if sys.version_info[0:2] < (3, 4): - # This does exactly the same what the :func:`py3:functools.update_wrapper` - # function does on Python versions after 3.2. It sets the ``__wrapped__`` - # attribute on ``wrapper`` object and it doesn't raise an error if any of - # the attributes mentioned in ``assigned`` and ``updated`` are missing on - # ``wrapped`` object. - def _update_wrapper( - wrapper, - wrapped, - assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES, - ): - for attr in assigned: - try: - value = getattr(wrapped, attr) - except AttributeError: - continue - else: - setattr(wrapper, attr, value) - for attr in updated: - getattr(wrapper, attr).update(getattr(wrapped, attr, {})) - wrapper.__wrapped__ = wrapped - return wrapper - - _update_wrapper.__doc__ = functools.update_wrapper.__doc__ - - def wraps( - wrapped, - assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES, - ): - return functools.partial( - _update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated - ) - - wraps.__doc__ = functools.wraps.__doc__ - -else: - wraps = functools.wraps - - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - # This requires a bit of explanation: the basic idea is to make a dummy - # metaclass for one level of class instantiation that replaces itself with - # the actual metaclass. - class metaclass(type): - def __new__(cls, name, this_bases, d): - if sys.version_info[:2] >= (3, 7): - # This version introduced PEP 560 that requires a bit - # of extra care (we mimic what is done by __build_class__). - resolved_bases = types.resolve_bases(bases) - if resolved_bases is not bases: - d["__orig_bases__"] = bases - else: - resolved_bases = bases - return meta(name, resolved_bases, d) - - @classmethod - def __prepare__(cls, name, this_bases): - return meta.__prepare__(name, bases) - - return type.__new__(metaclass, "temporary_class", (), {}) - - -def add_metaclass(metaclass): - """Class decorator for creating a class with a metaclass.""" - - def wrapper(cls): - orig_vars = cls.__dict__.copy() - slots = orig_vars.get("__slots__") - if slots is not None: - if isinstance(slots, str): - slots = [slots] - for slots_var in slots: - orig_vars.pop(slots_var) - orig_vars.pop("__dict__", None) - orig_vars.pop("__weakref__", None) - if hasattr(cls, "__qualname__"): - orig_vars["__qualname__"] = cls.__qualname__ - return metaclass(cls.__name__, cls.__bases__, orig_vars) - - return wrapper - - -def ensure_binary(s, encoding="utf-8", errors="strict"): - """Coerce **s** to six.binary_type. - - For Python 2: - - `unicode` -> encoded to `str` - - `str` -> `str` - - For Python 3: - - `str` -> encoded to `bytes` - - `bytes` -> `bytes` - """ - if isinstance(s, binary_type): - return s - if isinstance(s, text_type): - return s.encode(encoding, errors) - raise TypeError("not expecting type '%s'" % type(s)) - - -def ensure_str(s, encoding="utf-8", errors="strict"): - """Coerce *s* to `str`. - - For Python 2: - - `unicode` -> encoded to `str` - - `str` -> `str` - - For Python 3: - - `str` -> `str` - - `bytes` -> decoded to `str` - """ - # Optimization: Fast return for the common case. - if type(s) is str: - return s - if PY2 and isinstance(s, text_type): - return s.encode(encoding, errors) - elif PY3 and isinstance(s, binary_type): - return s.decode(encoding, errors) - elif not isinstance(s, (text_type, binary_type)): - raise TypeError("not expecting type '%s'" % type(s)) - return s - - -def ensure_text(s, encoding="utf-8", errors="strict"): - """Coerce *s* to six.text_type. - - For Python 2: - - `unicode` -> `unicode` - - `str` -> `unicode` - - For Python 3: - - `str` -> `str` - - `bytes` -> decoded to `str` - """ - if isinstance(s, binary_type): - return s.decode(encoding, errors) - elif isinstance(s, text_type): - return s - else: - raise TypeError("not expecting type '%s'" % type(s)) - - -def python_2_unicode_compatible(klass): - """ - A class decorator that defines __unicode__ and __str__ methods under Python 2. - Under Python 3 it does nothing. - - To support Python 2 and 3 with a single code base, define a __str__ method - returning text and apply this decorator to the class. - """ - if PY2: - if "__str__" not in klass.__dict__: - raise ValueError( - "@python_2_unicode_compatible cannot be applied " - "to %s because it doesn't define __str__()." % klass.__name__ - ) - klass.__unicode__ = klass.__str__ - klass.__str__ = lambda self: self.__unicode__().encode("utf-8") - return klass - - -# Complete the moves implementation. -# This code is at the end of this module to speed up module loading. -# Turn this module into a package. -__path__ = [] # required for PEP 302 and PEP 451 -__package__ = __name__ # see PEP 366 @ReservedAssignment -if globals().get("__spec__") is not None: - __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable -# Remove other six meta path importers, since they cause problems. This can -# happen if six is removed from sys.modules and then reloaded. (Setuptools does -# this for some reason.) -if sys.meta_path: - for i, importer in enumerate(sys.meta_path): - # Here's some real nastiness: Another "instance" of the six module might - # be floating around. Therefore, we can't use isinstance() to check for - # the six meta path importer, since the other six instance will have - # inserted an importer with different class. - if ( - type(importer).__name__ == "_SixMetaPathImporter" - and importer.name == __name__ - ): - del sys.meta_path[i] - break - del i, importer -# Finally, add the importer to the meta path import hook. -sys.meta_path.append(_importer) diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index fb51bf7..085d1db 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -1,24 +1,33 @@ -from __future__ import absolute_import +from __future__ import annotations -import collections import functools import logging +import typing +import warnings +from types import TracebackType +from urllib.parse import urljoin from ._collections import HTTPHeaderDict, RecentlyUsedContainer +from ._request_methods import RequestMethods +from .connection import ProxyConfig from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme from .exceptions import ( LocationValueError, MaxRetryError, ProxySchemeUnknown, - ProxySchemeUnsupported, URLSchemeUnknown, ) -from .packages import six -from .packages.six.moves.urllib.parse import urljoin -from .request import RequestMethods +from .response import BaseHTTPResponse +from .util.connection import _TYPE_SOCKET_OPTIONS from .util.proxy import connection_requires_http_tunnel from .util.retry import Retry -from .util.url import parse_url +from .util.timeout import Timeout +from .util.url import Url, parse_url + +if typing.TYPE_CHECKING: + import ssl + + from typing_extensions import Self __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] @@ -30,53 +39,62 @@ "cert_file", "cert_reqs", "ca_certs", + "ca_cert_data", "ssl_version", + "ssl_minimum_version", + "ssl_maximum_version", "ca_cert_dir", "ssl_context", "key_password", "server_hostname", ) +# Default value for `blocksize` - a new parameter introduced to +# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7 +_DEFAULT_BLOCKSIZE = 16384 -# All known keyword arguments that could be provided to the pool manager, its -# pools, or the underlying connections. This is used to construct a pool key. -_key_fields = ( - "key_scheme", # str - "key_host", # str - "key_port", # int - "key_timeout", # int or float or Timeout - "key_retries", # int or Retry - "key_strict", # bool - "key_block", # bool - "key_source_address", # str - "key_key_file", # str - "key_key_password", # str - "key_cert_file", # str - "key_cert_reqs", # str - "key_ca_certs", # str - "key_ssl_version", # str - "key_ca_cert_dir", # str - "key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext - "key_maxsize", # int - "key_headers", # dict - "key__proxy", # parsed proxy url - "key__proxy_headers", # dict - "key__proxy_config", # class - "key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples - "key__socks_options", # dict - "key_assert_hostname", # bool or string - "key_assert_fingerprint", # str - "key_server_hostname", # str -) - -#: The namedtuple class used to construct keys for the connection pool. -#: All custom key schemes should include the fields in this key at a minimum. -PoolKey = collections.namedtuple("PoolKey", _key_fields) -_proxy_config_fields = ("ssl_context", "use_forwarding_for_https") -ProxyConfig = collections.namedtuple("ProxyConfig", _proxy_config_fields) +class PoolKey(typing.NamedTuple): + """ + All known keyword arguments that could be provided to the pool manager, its + pools, or the underlying connections. + All custom key schemes should include the fields in this key at a minimum. + """ -def _default_key_normalizer(key_class, request_context): + key_scheme: str + key_host: str + key_port: int | None + key_timeout: Timeout | float | int | None + key_retries: Retry | bool | int | None + key_block: bool | None + key_source_address: tuple[str, int] | None + key_key_file: str | None + key_key_password: str | None + key_cert_file: str | None + key_cert_reqs: str | None + key_ca_certs: str | None + key_ca_cert_data: str | bytes | None + key_ssl_version: int | str | None + key_ssl_minimum_version: ssl.TLSVersion | None + key_ssl_maximum_version: ssl.TLSVersion | None + key_ca_cert_dir: str | None + key_ssl_context: ssl.SSLContext | None + key_maxsize: int | None + key_headers: frozenset[tuple[str, str]] | None + key__proxy: Url | None + key__proxy_headers: frozenset[tuple[str, str]] | None + key__proxy_config: ProxyConfig | None + key_socket_options: _TYPE_SOCKET_OPTIONS | None + key__socks_options: frozenset[tuple[str, str]] | None + key_assert_hostname: bool | str | None + key_assert_fingerprint: str | None + key_server_hostname: str | None + key_blocksize: int | None + + +def _default_key_normalizer( + key_class: type[PoolKey], request_context: dict[str, typing.Any] +) -> PoolKey: """ Create a pool key out of a request context dictionary. @@ -122,6 +140,10 @@ def _default_key_normalizer(key_class, request_context): if field not in context: context[field] = None + # Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context + if context.get("key_blocksize") is None: + context["key_blocksize"] = _DEFAULT_BLOCKSIZE + return key_class(**context) @@ -154,23 +176,36 @@ class PoolManager(RequestMethods): Additional parameters are used to create fresh :class:`urllib3.connectionpool.ConnectionPool` instances. - Example:: + Example: + + .. code-block:: python + + import urllib3 + + http = urllib3.PoolManager(num_pools=2) - >>> manager = PoolManager(num_pools=2) - >>> r = manager.request('GET', 'http://google.com/') - >>> r = manager.request('GET', 'http://google.com/mail') - >>> r = manager.request('GET', 'http://yahoo.com/') - >>> len(manager.pools) - 2 + resp1 = http.request("GET", "https://google.com/") + resp2 = http.request("GET", "https://google.com/mail") + resp3 = http.request("GET", "https://yahoo.com/") + + print(len(http.pools)) + # 2 """ - proxy = None - proxy_config = None + proxy: Url | None = None + proxy_config: ProxyConfig | None = None - def __init__(self, num_pools=10, headers=None, **connection_pool_kw): - RequestMethods.__init__(self, headers) + def __init__( + self, + num_pools: int = 10, + headers: typing.Mapping[str, str] | None = None, + **connection_pool_kw: typing.Any, + ) -> None: + super().__init__(headers) self.connection_pool_kw = connection_pool_kw + + self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool] self.pools = RecentlyUsedContainer(num_pools) # Locally set the pool classes and keys so other PoolManagers can @@ -178,15 +213,26 @@ def __init__(self, num_pools=10, headers=None, **connection_pool_kw): self.pool_classes_by_scheme = pool_classes_by_scheme self.key_fn_by_scheme = key_fn_by_scheme.copy() - def __enter__(self): + def __enter__(self) -> Self: return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> typing.Literal[False]: self.clear() # Return False to re-raise any potential exceptions return False - def _new_pool(self, scheme, host, port, request_context=None): + def _new_pool( + self, + scheme: str, + host: str, + port: int, + request_context: dict[str, typing.Any] | None = None, + ) -> HTTPConnectionPool: """ Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and any additional pool keyword arguments. @@ -196,10 +242,15 @@ def _new_pool(self, scheme, host, port, request_context=None): connection pools handed out by :meth:`connection_from_url` and companion methods. It is intended to be overridden for customization. """ - pool_cls = self.pool_classes_by_scheme[scheme] + pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme] if request_context is None: request_context = self.connection_pool_kw.copy() + # Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly + # set to 'None' in the request_context. + if request_context.get("blocksize") is None: + request_context["blocksize"] = _DEFAULT_BLOCKSIZE + # Although the context has everything necessary to create the pool, # this function has historically only used the scheme, host, and port # in the positional args. When an API change is acceptable these can @@ -213,7 +264,7 @@ def _new_pool(self, scheme, host, port, request_context=None): return pool_cls(host, port, **request_context) - def clear(self): + def clear(self) -> None: """ Empty our store of pools and direct them all to close. @@ -222,7 +273,13 @@ def clear(self): """ self.pools.clear() - def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): + def connection_from_host( + self, + host: str | None, + port: int | None = None, + scheme: str | None = "http", + pool_kwargs: dict[str, typing.Any] | None = None, + ) -> HTTPConnectionPool: """ Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme. @@ -245,13 +302,23 @@ def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None) return self.connection_from_context(request_context) - def connection_from_context(self, request_context): + def connection_from_context( + self, request_context: dict[str, typing.Any] + ) -> HTTPConnectionPool: """ Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context. ``request_context`` must at least contain the ``scheme`` key and its value must be a key in ``key_fn_by_scheme`` instance variable. """ + if "strict" in request_context: + warnings.warn( + "The 'strict' parameter is no longer needed on Python 3+. " + "This will raise an error in urllib3 v2.1.0.", + DeprecationWarning, + ) + request_context.pop("strict") + scheme = request_context["scheme"].lower() pool_key_constructor = self.key_fn_by_scheme.get(scheme) if not pool_key_constructor: @@ -260,7 +327,9 @@ def connection_from_context(self, request_context): return self.connection_from_pool_key(pool_key, request_context=request_context) - def connection_from_pool_key(self, pool_key, request_context=None): + def connection_from_pool_key( + self, pool_key: PoolKey, request_context: dict[str, typing.Any] + ) -> HTTPConnectionPool: """ Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key. @@ -284,7 +353,9 @@ def connection_from_pool_key(self, pool_key, request_context=None): return pool - def connection_from_url(self, url, pool_kwargs=None): + def connection_from_url( + self, url: str, pool_kwargs: dict[str, typing.Any] | None = None + ) -> HTTPConnectionPool: """ Similar to :func:`urllib3.connectionpool.connection_from_url`. @@ -300,7 +371,9 @@ def connection_from_url(self, url, pool_kwargs=None): u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs ) - def _merge_pool_kwargs(self, override): + def _merge_pool_kwargs( + self, override: dict[str, typing.Any] | None + ) -> dict[str, typing.Any]: """ Merge a dictionary of override values for self.connection_pool_kw. @@ -320,7 +393,7 @@ def _merge_pool_kwargs(self, override): base_pool_kwargs[key] = value return base_pool_kwargs - def _proxy_requires_url_absolute_form(self, parsed_url): + def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool: """ Indicates if the proxy requires the complete destination URL in the request. Normally this is only needed when not using an HTTP CONNECT @@ -333,24 +406,9 @@ def _proxy_requires_url_absolute_form(self, parsed_url): self.proxy, self.proxy_config, parsed_url.scheme ) - def _validate_proxy_scheme_url_selection(self, url_scheme): - """ - Validates that were not attempting to do TLS in TLS connections on - Python2 or with unsupported SSL implementations. - """ - if self.proxy is None or url_scheme != "https": - return - - if self.proxy.scheme != "https": - return - - if six.PY2 and not self.proxy_config.use_forwarding_for_https: - raise ProxySchemeUnsupported( - "Contacting HTTPS destinations through HTTPS proxies " - "'via CONNECT tunnels' is not supported in Python 2" - ) - - def urlopen(self, method, url, redirect=True, **kw): + def urlopen( # type: ignore[override] + self, method: str, url: str, redirect: bool = True, **kw: typing.Any + ) -> BaseHTTPResponse: """ Same as :meth:`urllib3.HTTPConnectionPool.urlopen` with custom cross-host redirect logic and only sends the request-uri @@ -360,7 +418,16 @@ def urlopen(self, method, url, redirect=True, **kw): :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. """ u = parse_url(url) - self._validate_proxy_scheme_url_selection(u.scheme) + + if u.scheme is None: + warnings.warn( + "URLs without a scheme (ie 'https://') are deprecated and will raise an error " + "in a future version of urllib3. To avoid this DeprecationWarning ensure all URLs " + "start with 'https://' or 'http://'. Read more in this issue: " + "https://github.com/urllib3/urllib3/issues/2920", + category=DeprecationWarning, + stacklevel=2, + ) conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) @@ -368,7 +435,7 @@ def urlopen(self, method, url, redirect=True, **kw): kw["redirect"] = False if "headers" not in kw: - kw["headers"] = self.headers.copy() + kw["headers"] = self.headers if self._proxy_requires_url_absolute_form(u): response = conn.urlopen(method, url, **kw) @@ -399,10 +466,11 @@ def urlopen(self, method, url, redirect=True, **kw): if retries.remove_headers_on_redirect and not conn.is_same_host( redirect_location ): - headers = list(six.iterkeys(kw["headers"])) - for header in headers: + new_headers = kw["headers"].copy() + for header in kw["headers"]: if header.lower() in retries.remove_headers_on_redirect: - kw["headers"].pop(header, None) + new_headers.pop(header, None) + kw["headers"] = new_headers try: retries = retries.increment(method, url, response=response, _pool=conn) @@ -448,37 +516,51 @@ class ProxyManager(PoolManager): private. IP address, target hostname, SNI, and port are always visible to an HTTPS proxy even when this flag is disabled. + :param proxy_assert_hostname: + The hostname of the certificate to verify against. + + :param proxy_assert_fingerprint: + The fingerprint of the certificate to verify against. + Example: - >>> proxy = urllib3.ProxyManager('http://localhost:3128/') - >>> r1 = proxy.request('GET', 'http://google.com/') - >>> r2 = proxy.request('GET', 'http://httpbin.org/') - >>> len(proxy.pools) - 1 - >>> r3 = proxy.request('GET', 'https://httpbin.org/') - >>> r4 = proxy.request('GET', 'https://twitter.com/') - >>> len(proxy.pools) - 3 + + .. code-block:: python + + import urllib3 + + proxy = urllib3.ProxyManager("https://localhost:3128/") + + resp1 = proxy.request("GET", "https://google.com/") + resp2 = proxy.request("GET", "https://httpbin.org/") + + print(len(proxy.pools)) + # 1 + + resp3 = proxy.request("GET", "https://httpbin.org/") + resp4 = proxy.request("GET", "https://twitter.com/") + + print(len(proxy.pools)) + # 3 """ def __init__( self, - proxy_url, - num_pools=10, - headers=None, - proxy_headers=None, - proxy_ssl_context=None, - use_forwarding_for_https=False, - **connection_pool_kw - ): - + proxy_url: str, + num_pools: int = 10, + headers: typing.Mapping[str, str] | None = None, + proxy_headers: typing.Mapping[str, str] | None = None, + proxy_ssl_context: ssl.SSLContext | None = None, + use_forwarding_for_https: bool = False, + proxy_assert_hostname: None | str | typing.Literal[False] = None, + proxy_assert_fingerprint: str | None = None, + **connection_pool_kw: typing.Any, + ) -> None: if isinstance(proxy_url, HTTPConnectionPool): - proxy_url = "%s://%s:%i" % ( - proxy_url.scheme, - proxy_url.host, - proxy_url.port, - ) - proxy = parse_url(proxy_url) + str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}" + else: + str_proxy_url = proxy_url + proxy = parse_url(str_proxy_url) if proxy.scheme not in ("http", "https"): raise ProxySchemeUnknown(proxy.scheme) @@ -490,25 +572,38 @@ def __init__( self.proxy = proxy self.proxy_headers = proxy_headers or {} self.proxy_ssl_context = proxy_ssl_context - self.proxy_config = ProxyConfig(proxy_ssl_context, use_forwarding_for_https) + self.proxy_config = ProxyConfig( + proxy_ssl_context, + use_forwarding_for_https, + proxy_assert_hostname, + proxy_assert_fingerprint, + ) connection_pool_kw["_proxy"] = self.proxy connection_pool_kw["_proxy_headers"] = self.proxy_headers connection_pool_kw["_proxy_config"] = self.proxy_config - super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw) + super().__init__(num_pools, headers, **connection_pool_kw) - def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): + def connection_from_host( + self, + host: str | None, + port: int | None = None, + scheme: str | None = "http", + pool_kwargs: dict[str, typing.Any] | None = None, + ) -> HTTPConnectionPool: if scheme == "https": - return super(ProxyManager, self).connection_from_host( + return super().connection_from_host( host, port, scheme, pool_kwargs=pool_kwargs ) - return super(ProxyManager, self).connection_from_host( - self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs + return super().connection_from_host( + self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr] ) - def _set_proxy_headers(self, url, headers=None): + def _set_proxy_headers( + self, url: str, headers: typing.Mapping[str, str] | None = None + ) -> typing.Mapping[str, str]: """ Sets headers needed by proxies: specifically, the Accept and Host headers. Only sets headers not provided by the user. @@ -523,7 +618,9 @@ def _set_proxy_headers(self, url, headers=None): headers_.update(headers) return headers_ - def urlopen(self, method, url, redirect=True, **kw): + def urlopen( # type: ignore[override] + self, method: str, url: str, redirect: bool = True, **kw: typing.Any + ) -> BaseHTTPResponse: "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." u = parse_url(url) if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme): @@ -533,8 +630,8 @@ def urlopen(self, method, url, redirect=True, **kw): headers = kw.get("headers", self.headers) kw["headers"] = self._set_proxy_headers(url, headers) - return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw) + return super().urlopen(method, url, redirect=redirect, **kw) -def proxy_from_url(url, **kw): +def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager: return ProxyManager(proxy_url=url, **kw) diff --git a/src/urllib3/py.typed b/src/urllib3/py.typed new file mode 100644 index 0000000..5f3ea3d --- /dev/null +++ b/src/urllib3/py.typed @@ -0,0 +1,2 @@ +# Instruct type checkers to look for inline type annotations in this package. +# See PEP 561. diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 0bd13d4..66c6a68 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -1,25 +1,52 @@ -from __future__ import absolute_import +from __future__ import annotations +import collections import io +import json as _json import logging +import re +import socket import sys +import typing import warnings import zlib from contextlib import contextmanager -from socket import error as SocketError +from http.client import HTTPMessage as _HttplibHTTPMessage +from http.client import HTTPResponse as _HttplibHTTPResponse from socket import timeout as SocketTimeout +if typing.TYPE_CHECKING: + from ._base_connection import BaseHTTPConnection + try: try: - import brotlicffi as brotli + import brotlicffi as brotli # type: ignore[import-not-found] except ImportError: - import brotli + import brotli # type: ignore[import-not-found] except ImportError: brotli = None +try: + import zstandard as zstd +except (AttributeError, ImportError, ValueError): # Defensive: + HAS_ZSTD = False +else: + # The package 'zstandard' added the 'eof' property starting + # in v0.18.0 which we require to ensure a complete and + # valid zstd stream was fed into the ZstdDecoder. + # See: https://github.com/urllib3/urllib3/pull/2624 + _zstd_version = tuple( + map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr] + ) + if _zstd_version < (0, 18): # Defensive: + HAS_ZSTD = False + else: + HAS_ZSTD = True + from . import util +from ._base_connection import _TYPE_BODY from ._collections import HTTPHeaderDict -from .connection import BaseSSLError, HTTPException +from .connection import BaseSSLError, HTTPConnection, HTTPException from .exceptions import ( BodyNotHttplibCompatible, DecodeError, @@ -32,22 +59,30 @@ ResponseNotChunked, SSLError, ) -from .packages import six from .util.response import is_fp_closed, is_response_to_head +from .util.retry import Retry + +if typing.TYPE_CHECKING: + from .connectionpool import HTTPConnectionPool log = logging.getLogger(__name__) -class DeflateDecoder(object): - def __init__(self): +class ContentDecoder: + def decompress(self, data: bytes) -> bytes: + raise NotImplementedError() + + def flush(self) -> bytes: + raise NotImplementedError() + + +class DeflateDecoder(ContentDecoder): + def __init__(self) -> None: self._first_try = True self._data = b"" self._obj = zlib.decompressobj() - def __getattr__(self, name): - return getattr(self._obj, name) - - def decompress(self, data): + def decompress(self, data: bytes) -> bytes: if not data: return data @@ -59,7 +94,7 @@ def decompress(self, data): decompressed = self._obj.decompress(data) if decompressed: self._first_try = False - self._data = None + self._data = None # type: ignore[assignment] return decompressed except zlib.error: self._first_try = False @@ -67,25 +102,24 @@ def decompress(self, data): try: return self.decompress(self._data) finally: - self._data = None + self._data = None # type: ignore[assignment] + def flush(self) -> bytes: + return self._obj.flush() -class GzipDecoderState(object): +class GzipDecoderState: FIRST_MEMBER = 0 OTHER_MEMBERS = 1 SWALLOW_DATA = 2 -class GzipDecoder(object): - def __init__(self): +class GzipDecoder(ContentDecoder): + def __init__(self) -> None: self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) self._state = GzipDecoderState.FIRST_MEMBER - def __getattr__(self, name): - return getattr(self._obj, name) - - def decompress(self, data): + def decompress(self, data: bytes) -> bytes: ret = bytearray() if self._state == GzipDecoderState.SWALLOW_DATA or not data: return bytes(ret) @@ -106,27 +140,53 @@ def decompress(self, data): self._state = GzipDecoderState.OTHER_MEMBERS self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) + def flush(self) -> bytes: + return self._obj.flush() + if brotli is not None: - class BrotliDecoder(object): + class BrotliDecoder(ContentDecoder): # Supports both 'brotlipy' and 'Brotli' packages # since they share an import name. The top branches # are for 'brotlipy' and bottom branches for 'Brotli' - def __init__(self): + def __init__(self) -> None: self._obj = brotli.Decompressor() if hasattr(self._obj, "decompress"): - self.decompress = self._obj.decompress + setattr(self, "decompress", self._obj.decompress) else: - self.decompress = self._obj.process + setattr(self, "decompress", self._obj.process) - def flush(self): + def flush(self) -> bytes: if hasattr(self._obj, "flush"): - return self._obj.flush() + return self._obj.flush() # type: ignore[no-any-return] return b"" -class MultiDecoder(object): +if HAS_ZSTD: + + class ZstdDecoder(ContentDecoder): + def __init__(self) -> None: + self._obj = zstd.ZstdDecompressor().decompressobj() + + def decompress(self, data: bytes) -> bytes: + if not data: + return b"" + data_parts = [self._obj.decompress(data)] + while self._obj.eof and self._obj.unused_data: + unused_data = self._obj.unused_data + self._obj = zstd.ZstdDecompressor().decompressobj() + data_parts.append(self._obj.decompress(unused_data)) + return b"".join(data_parts) + + def flush(self) -> bytes: + ret = self._obj.flush() # note: this is a no-op + if not self._obj.eof: + raise DecodeError("Zstandard data is incomplete") + return ret + + +class MultiDecoder(ContentDecoder): """ From RFC7231: If one or more encodings have been applied to a representation, the @@ -135,32 +195,353 @@ class MultiDecoder(object): they were applied. """ - def __init__(self, modes): + def __init__(self, modes: str) -> None: self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")] - def flush(self): + def flush(self) -> bytes: return self._decoders[0].flush() - def decompress(self, data): + def decompress(self, data: bytes) -> bytes: for d in reversed(self._decoders): data = d.decompress(data) return data -def _get_decoder(mode): +def _get_decoder(mode: str) -> ContentDecoder: if "," in mode: return MultiDecoder(mode) - if mode == "gzip": + # According to RFC 9110 section 8.4.1.3, recipients should + # consider x-gzip equivalent to gzip + if mode in ("gzip", "x-gzip"): return GzipDecoder() if brotli is not None and mode == "br": return BrotliDecoder() + if HAS_ZSTD and mode == "zstd": + return ZstdDecoder() + return DeflateDecoder() -class HTTPResponse(io.IOBase): +class BytesQueueBuffer: + """Memory-efficient bytes buffer + + To return decoded data in read() and still follow the BufferedIOBase API, we need a + buffer to always return the correct amount of bytes. + + This buffer should be filled using calls to put() + + Our maximum memory usage is determined by the sum of the size of: + + * self.buffer, which contains the full data + * the largest chunk that we will copy in get() + + The worst case scenario is a single chunk, in which case we'll make a full copy of + the data inside get(). + """ + + def __init__(self) -> None: + self.buffer: typing.Deque[bytes] = collections.deque() + self._size: int = 0 + + def __len__(self) -> int: + return self._size + + def put(self, data: bytes) -> None: + self.buffer.append(data) + self._size += len(data) + + def get(self, n: int) -> bytes: + if n == 0: + return b"" + elif not self.buffer: + raise RuntimeError("buffer is empty") + elif n < 0: + raise ValueError("n should be > 0") + + fetched = 0 + ret = io.BytesIO() + while fetched < n: + remaining = n - fetched + chunk = self.buffer.popleft() + chunk_length = len(chunk) + if remaining < chunk_length: + left_chunk, right_chunk = chunk[:remaining], chunk[remaining:] + ret.write(left_chunk) + self.buffer.appendleft(right_chunk) + self._size -= remaining + break + else: + ret.write(chunk) + self._size -= chunk_length + fetched += chunk_length + + if not self.buffer: + break + + return ret.getvalue() + + def get_all(self) -> bytes: + buffer = self.buffer + if not buffer: + assert self._size == 0 + return b"" + if len(buffer) == 1: + result = buffer.pop() + else: + ret = io.BytesIO() + ret.writelines(buffer.popleft() for _ in range(len(buffer))) + result = ret.getvalue() + self._size = 0 + return result + + +class BaseHTTPResponse(io.IOBase): + CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"] + if brotli is not None: + CONTENT_DECODERS += ["br"] + if HAS_ZSTD: + CONTENT_DECODERS += ["zstd"] + REDIRECT_STATUSES = [301, 302, 303, 307, 308] + + DECODER_ERROR_CLASSES: tuple[type[Exception], ...] = (IOError, zlib.error) + if brotli is not None: + DECODER_ERROR_CLASSES += (brotli.error,) + + if HAS_ZSTD: + DECODER_ERROR_CLASSES += (zstd.ZstdError,) + + def __init__( + self, + *, + headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, + status: int, + version: int, + version_string: str, + reason: str | None, + decode_content: bool, + request_url: str | None, + retries: Retry | None = None, + ) -> None: + if isinstance(headers, HTTPHeaderDict): + self.headers = headers + else: + self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type] + self.status = status + self.version = version + self.version_string = version_string + self.reason = reason + self.decode_content = decode_content + self._has_decoded_content = False + self._request_url: str | None = request_url + self.retries = retries + + self.chunked = False + tr_enc = self.headers.get("transfer-encoding", "").lower() + # Don't incur the penalty of creating a list and then discarding it + encodings = (enc.strip() for enc in tr_enc.split(",")) + if "chunked" in encodings: + self.chunked = True + + self._decoder: ContentDecoder | None = None + self.length_remaining: int | None + + def get_redirect_location(self) -> str | None | typing.Literal[False]: + """ + Should we redirect and where to? + + :returns: Truthy redirect location string if we got a redirect status + code and valid location. ``None`` if redirect status and no + location. ``False`` if not a redirect status code. + """ + if self.status in self.REDIRECT_STATUSES: + return self.headers.get("location") + return False + + @property + def data(self) -> bytes: + raise NotImplementedError() + + def json(self) -> typing.Any: + """ + Deserializes the body of the HTTP response as a Python object. + + The body of the HTTP response must be encoded using UTF-8, as per + `RFC 8529 Section 8.1 `_. + + To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to + your custom decoder instead. + + If the body of the HTTP response is not decodable to UTF-8, a + `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a + valid JSON document, a `json.JSONDecodeError` will be raised. + + Read more :ref:`here `. + + :returns: The body of the HTTP response as a Python object. + """ + data = self.data.decode("utf-8") + return _json.loads(data) + + @property + def url(self) -> str | None: + raise NotImplementedError() + + @url.setter + def url(self, url: str | None) -> None: + raise NotImplementedError() + + @property + def connection(self) -> BaseHTTPConnection | None: + raise NotImplementedError() + + @property + def retries(self) -> Retry | None: + return self._retries + + @retries.setter + def retries(self, retries: Retry | None) -> None: + # Override the request_url if retries has a redirect location. + if retries is not None and retries.history: + self.url = retries.history[-1].redirect_location + self._retries = retries + + def stream( + self, amt: int | None = 2**16, decode_content: bool | None = None + ) -> typing.Iterator[bytes]: + raise NotImplementedError() + + def read( + self, + amt: int | None = None, + decode_content: bool | None = None, + cache_content: bool = False, + ) -> bytes: + raise NotImplementedError() + + def read1( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> bytes: + raise NotImplementedError() + + def read_chunked( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> typing.Iterator[bytes]: + raise NotImplementedError() + + def release_conn(self) -> None: + raise NotImplementedError() + + def drain_conn(self) -> None: + raise NotImplementedError() + + def shutdown(self) -> None: + raise NotImplementedError() + + def close(self) -> None: + raise NotImplementedError() + + def _init_decoder(self) -> None: + """ + Set-up the _decoder attribute if necessary. + """ + # Note: content-encoding value should be case-insensitive, per RFC 7230 + # Section 3.2 + content_encoding = self.headers.get("content-encoding", "").lower() + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + elif "," in content_encoding: + encodings = [ + e.strip() + for e in content_encoding.split(",") + if e.strip() in self.CONTENT_DECODERS + ] + if encodings: + self._decoder = _get_decoder(content_encoding) + + def _decode( + self, data: bytes, decode_content: bool | None, flush_decoder: bool + ) -> bytes: + """ + Decode the data passed in and potentially flush the decoder. + """ + if not decode_content: + if self._has_decoded_content: + raise RuntimeError( + "Calling read(decode_content=False) is not supported after " + "read(decode_content=True) was called." + ) + return data + + try: + if self._decoder: + data = self._decoder.decompress(data) + self._has_decoded_content = True + except self.DECODER_ERROR_CLASSES as e: + content_encoding = self.headers.get("content-encoding", "").lower() + raise DecodeError( + "Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding, + e, + ) from e + if flush_decoder: + data += self._flush_decoder() + + return data + + def _flush_decoder(self) -> bytes: + """ + Flushes the decoder. Should only be called if the decoder is actually + being used. + """ + if self._decoder: + return self._decoder.decompress(b"") + self._decoder.flush() + return b"" + + # Compatibility methods for `io` module + def readinto(self, b: bytearray) -> int: + temp = self.read(len(b)) + if len(temp) == 0: + return 0 + else: + b[: len(temp)] = temp + return len(temp) + + # Compatibility methods for http.client.HTTPResponse + def getheaders(self) -> HTTPHeaderDict: + warnings.warn( + "HTTPResponse.getheaders() is deprecated and will be removed " + "in urllib3 v2.1.0. Instead access HTTPResponse.headers directly.", + category=DeprecationWarning, + stacklevel=2, + ) + return self.headers + + def getheader(self, name: str, default: str | None = None) -> str | None: + warnings.warn( + "HTTPResponse.getheader() is deprecated and will be removed " + "in urllib3 v2.1.0. Instead use HTTPResponse.headers.get(name, default).", + category=DeprecationWarning, + stacklevel=2, + ) + return self.headers.get(name, default) + + # Compatibility method for http.cookiejar + def info(self) -> HTTPHeaderDict: + return self.headers + + def geturl(self) -> str | None: + return self.url + + +class HTTPResponse(BaseHTTPResponse): """ HTTP Response container. @@ -193,99 +574,78 @@ class is also compatible with the Python standard library's :mod:`io` value of Content-Length header, if present. Otherwise, raise error. """ - CONTENT_DECODERS = ["gzip", "deflate"] - if brotli is not None: - CONTENT_DECODERS += ["br"] - REDIRECT_STATUSES = [301, 302, 303, 307, 308] - def __init__( self, - body="", - headers=None, - status=0, - version=0, - reason=None, - strict=0, - preload_content=True, - decode_content=True, - original_response=None, - pool=None, - connection=None, - msg=None, - retries=None, - enforce_content_length=False, - request_method=None, - request_url=None, - auto_close=True, - ): + body: _TYPE_BODY = "", + headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, + status: int = 0, + version: int = 0, + version_string: str = "HTTP/?", + reason: str | None = None, + preload_content: bool = True, + decode_content: bool = True, + original_response: _HttplibHTTPResponse | None = None, + pool: HTTPConnectionPool | None = None, + connection: HTTPConnection | None = None, + msg: _HttplibHTTPMessage | None = None, + retries: Retry | None = None, + enforce_content_length: bool = True, + request_method: str | None = None, + request_url: str | None = None, + auto_close: bool = True, + sock_shutdown: typing.Callable[[int], None] | None = None, + ) -> None: + super().__init__( + headers=headers, + status=status, + version=version, + version_string=version_string, + reason=reason, + decode_content=decode_content, + request_url=request_url, + retries=retries, + ) - if isinstance(headers, HTTPHeaderDict): - self.headers = headers - else: - self.headers = HTTPHeaderDict(headers) - self.status = status - self.version = version - self.reason = reason - self.strict = strict - self.decode_content = decode_content - self.retries = retries self.enforce_content_length = enforce_content_length self.auto_close = auto_close - self._decoder = None self._body = None - self._fp = None + self._fp: _HttplibHTTPResponse | None = None self._original_response = original_response self._fp_bytes_read = 0 self.msg = msg - self._request_url = request_url - if body and isinstance(body, (six.string_types, bytes)): + if body and isinstance(body, (str, bytes)): self._body = body self._pool = pool self._connection = connection if hasattr(body, "read"): - self._fp = body + self._fp = body # type: ignore[assignment] + self._sock_shutdown = sock_shutdown # Are we using the chunked-style of transfer encoding? - self.chunked = False - self.chunk_left = None - tr_enc = self.headers.get("transfer-encoding", "").lower() - # Don't incur the penalty of creating a list and then discarding it - encodings = (enc.strip() for enc in tr_enc.split(",")) - if "chunked" in encodings: - self.chunked = True + self.chunk_left: int | None = None # Determine length of response self.length_remaining = self._init_length(request_method) + # Used to return the correct amount of bytes for partial read()s + self._decoded_buffer = BytesQueueBuffer() + # If requested, preload the body. if preload_content and not self._body: self._body = self.read(decode_content=decode_content) - def get_redirect_location(self): - """ - Should we redirect and where to? - - :returns: Truthy redirect location string if we got a redirect status - code and valid location. ``None`` if redirect status and no - location. ``False`` if not a redirect status code. - """ - if self.status in self.REDIRECT_STATUSES: - return self.headers.get("location") - - return False - - def release_conn(self): + def release_conn(self) -> None: if not self._pool or not self._connection: - return + return None self._pool._put_conn(self._connection) self._connection = None - def drain_conn(self): + def drain_conn(self) -> None: """ Read and discard any remaining HTTP response data in the response connection. @@ -293,26 +653,28 @@ def drain_conn(self): """ try: self.read() - except (HTTPError, SocketError, BaseSSLError, HTTPException): + except (HTTPError, OSError, BaseSSLError, HTTPException): pass @property - def data(self): + def data(self) -> bytes: # For backwards-compat with earlier urllib3 0.4 and earlier. if self._body: - return self._body + return self._body # type: ignore[return-value] if self._fp: return self.read(cache_content=True) + return None # type: ignore[return-value] + @property - def connection(self): + def connection(self) -> HTTPConnection | None: return self._connection - def isclosed(self): + def isclosed(self) -> bool: return is_fp_closed(self._fp) - def tell(self): + def tell(self) -> int: """ Obtain the number of bytes pulled over the wire so far. May differ from the amount of content returned by :meth:``urllib3.response.HTTPResponse.read`` @@ -320,13 +682,14 @@ def tell(self): """ return self._fp_bytes_read - def _init_length(self, request_method): + def _init_length(self, request_method: str | None) -> int | None: """ Set initial length value for Response content if available. """ - length = self.headers.get("content-length") + length: int | None + content_length: str | None = self.headers.get("content-length") - if length is not None: + if content_length is not None: if self.chunked: # This Response will fail with an IncompleteRead if it can't be # received as chunked. This method falls back to attempt reading @@ -346,11 +709,11 @@ def _init_length(self, request_method): # (e.g. Content-Length: 42, 42). This line ensures the values # are all valid ints and that as long as the `set` length is 1, # all values are the same. Otherwise, the header is invalid. - lengths = set([int(val) for val in length.split(",")]) + lengths = {int(val) for val in content_length.split(",")} if len(lengths) > 1: raise InvalidHeader( "Content-Length contained multiple " - "unmatching values (%s)" % length + "unmatching values (%s)" % content_length ) length = lengths.pop() except ValueError: @@ -359,6 +722,9 @@ def _init_length(self, request_method): if length < 0: length = None + else: # if content_length is None + length = None + # Convert status to int for comparison # In some cases, httplib returns a status of "_UNKNOWN" try: @@ -372,64 +738,8 @@ def _init_length(self, request_method): return length - def _init_decoder(self): - """ - Set-up the _decoder attribute if necessary. - """ - # Note: content-encoding value should be case-insensitive, per RFC 7230 - # Section 3.2 - content_encoding = self.headers.get("content-encoding", "").lower() - if self._decoder is None: - if content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) - elif "," in content_encoding: - encodings = [ - e.strip() - for e in content_encoding.split(",") - if e.strip() in self.CONTENT_DECODERS - ] - if len(encodings): - self._decoder = _get_decoder(content_encoding) - - DECODER_ERROR_CLASSES = (IOError, zlib.error) - if brotli is not None: - DECODER_ERROR_CLASSES += (brotli.error,) - - def _decode(self, data, decode_content, flush_decoder): - """ - Decode the data passed in and potentially flush the decoder. - """ - if not decode_content: - return data - - try: - if self._decoder: - data = self._decoder.decompress(data) - except self.DECODER_ERROR_CLASSES as e: - content_encoding = self.headers.get("content-encoding", "").lower() - raise DecodeError( - "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, - e, - ) - if flush_decoder: - data += self._flush_decoder() - - return data - - def _flush_decoder(self): - """ - Flushes the decoder. Should only be called if the decoder is actually - being used. - """ - if self._decoder: - buf = self._decoder.decompress(b"") - return buf + self._decoder.flush() - - return b"" - @contextmanager - def _error_catcher(self): + def _error_catcher(self) -> typing.Generator[None]: """ Catch low-level python exceptions, instead re-raising urllib3 variants, so that low-level exceptions are not leaked in the @@ -443,22 +753,32 @@ def _error_catcher(self): try: yield - except SocketTimeout: + except SocketTimeout as e: # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but # there is yet no clean way to get at it from this context. - raise ReadTimeoutError(self._pool, None, "Read timed out.") + raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] except BaseSSLError as e: # FIXME: Is there a better way to differentiate between SSLErrors? if "read operation timed out" not in str(e): # SSL errors related to framing/MAC get wrapped and reraised here - raise SSLError(e) + raise SSLError(e) from e - raise ReadTimeoutError(self._pool, None, "Read timed out.") + raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] - except (HTTPException, SocketError) as e: - # This includes IncompleteRead. - raise ProtocolError("Connection broken: %r" % e, e) + except IncompleteRead as e: + if ( + e.expected is not None + and e.partial is not None + and e.expected == -e.partial + ): + arg = "Response may not contain content." + else: + arg = f"Connection broken: {e!r}" + raise ProtocolError(arg, e) from e + + except (HTTPException, OSError) as e: + raise ProtocolError(f"Connection broken: {e!r}", e) from e # If no exception is thrown, we should avoid cleaning up # unnecessarily. @@ -484,7 +804,12 @@ def _error_catcher(self): if self._original_response and self._original_response.isclosed(): self.release_conn() - def _fp_read(self, amt): + def _fp_read( + self, + amt: int | None = None, + *, + read1: bool = False, + ) -> bytes: """ Read a response with the thought that reading the number of bytes larger than can fit in a 32-bit int at a time via SSL in some @@ -493,21 +818,23 @@ def _fp_read(self, amt): happen. The known cases: - * 3.8 <= CPython < 3.9.7 because of a bug + * CPython < 3.9.7 because of a bug https://github.com/urllib3/urllib3/issues/2513#issuecomment-1152559900. * urllib3 injected with pyOpenSSL-backed SSL-support. * CPython < 3.10 only when `amt` does not fit 32-bit int. """ assert self._fp - c_int_max = 2 ** 31 - 1 + c_int_max = 2**31 - 1 if ( - ( - (amt and amt > c_int_max) - or (self.length_remaining and self.length_remaining > c_int_max) + (amt and amt > c_int_max) + or ( + amt is None + and self.length_remaining + and self.length_remaining > c_int_max ) - and not util.IS_SECURETRANSPORT - and (util.IS_PYOPENSSL or sys.version_info < (3, 10)) - ): + ) and (util.IS_PYOPENSSL or sys.version_info < (3, 10)): + if read1: + return self._fp.read1(c_int_max) buffer = io.BytesIO() # Besides `max_chunk_amt` being a maximum chunk size, it # affects memory overhead of reading a response by this @@ -515,7 +842,7 @@ def _fp_read(self, amt): # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum # chunk size that does not lead to an overflow error, but # 256 MiB is a compromise. - max_chunk_amt = 2 ** 28 + max_chunk_amt = 2**28 while amt is None or amt != 0: if amt is not None: chunk_amt = min(amt, max_chunk_amt) @@ -528,11 +855,70 @@ def _fp_read(self, amt): buffer.write(data) del data # to reduce peak memory usage by `max_chunk_amt`. return buffer.getvalue() + elif read1: + return self._fp.read1(amt) if amt is not None else self._fp.read1() else: # StringIO doesn't like amt=None return self._fp.read(amt) if amt is not None else self._fp.read() - def read(self, amt=None, decode_content=None, cache_content=False): + def _raw_read( + self, + amt: int | None = None, + *, + read1: bool = False, + ) -> bytes: + """ + Reads `amt` of bytes from the socket. + """ + if self._fp is None: + return None # type: ignore[return-value] + + fp_closed = getattr(self._fp, "closed", False) + + with self._error_catcher(): + data = self._fp_read(amt, read1=read1) if not fp_closed else b"" + if amt is not None and amt != 0 and not data: + # Platform-specific: Buggy versions of Python. + # Close the connection when no data is returned + # + # This is redundant to what httplib/http.client _should_ + # already do. However, versions of python released before + # December 15, 2012 (http://bugs.python.org/issue16298) do + # not properly close the connection in all cases. There is + # no harm in redundantly calling close. + self._fp.close() + if ( + self.enforce_content_length + and self.length_remaining is not None + and self.length_remaining != 0 + ): + # This is an edge case that httplib failed to cover due + # to concerns of backward compatibility. We're + # addressing it here to make sure IncompleteRead is + # raised during streaming, so all calls with incorrect + # Content-Length are caught. + raise IncompleteRead(self._fp_bytes_read, self.length_remaining) + elif read1 and ( + (amt != 0 and not data) or self.length_remaining == len(data) + ): + # All data has been read, but `self._fp.read1` in + # CPython 3.12 and older doesn't always close + # `http.client.HTTPResponse`, so we close it here. + # See https://github.com/python/cpython/issues/113199 + self._fp.close() + + if data: + self._fp_bytes_read += len(data) + if self.length_remaining is not None: + self.length_remaining -= len(data) + return data + + def read( + self, + amt: int | None = None, + decode_content: bool | None = None, + cache_content: bool = False, + ) -> bytes: """ Similar to :meth:`http.client.HTTPResponse.read`, but with two additional parameters: ``decode_content`` and ``cache_content``. @@ -557,54 +943,107 @@ def read(self, amt=None, decode_content=None, cache_content=False): if decode_content is None: decode_content = self.decode_content - if self._fp is None: - return + if amt and amt < 0: + # Negative numbers and `None` should be treated the same. + amt = None + elif amt is not None: + cache_content = False - flush_decoder = False - fp_closed = getattr(self._fp, "closed", False) + if len(self._decoded_buffer) >= amt: + return self._decoded_buffer.get(amt) - with self._error_catcher(): - data = self._fp_read(amt) if not fp_closed else b"" - if amt is None: - flush_decoder = True - else: - cache_content = False - if ( - amt != 0 and not data - ): # Platform-specific: Buggy versions of Python. - # Close the connection when no data is returned - # - # This is redundant to what httplib/http.client _should_ - # already do. However, versions of python released before - # December 15, 2012 (http://bugs.python.org/issue16298) do - # not properly close the connection in all cases. There is - # no harm in redundantly calling close. - self._fp.close() - flush_decoder = True - if self.enforce_content_length and self.length_remaining not in ( - 0, - None, - ): - # This is an edge case that httplib failed to cover due - # to concerns of backward compatibility. We're - # addressing it here to make sure IncompleteRead is - # raised during streaming, so all calls with incorrect - # Content-Length are caught. - raise IncompleteRead(self._fp_bytes_read, self.length_remaining) + data = self._raw_read(amt) - if data: - self._fp_bytes_read += len(data) - if self.length_remaining is not None: - self.length_remaining -= len(data) + flush_decoder = amt is None or (amt != 0 and not data) - data = self._decode(data, decode_content, flush_decoder) + if not data and len(self._decoded_buffer) == 0: + return data + if amt is None: + data = self._decode(data, decode_content, flush_decoder) if cache_content: self._body = data + else: + # do not waste memory on buffer when not decoding + if not decode_content: + if self._has_decoded_content: + raise RuntimeError( + "Calling read(decode_content=False) is not supported after " + "read(decode_content=True) was called." + ) + return data + + decoded_data = self._decode(data, decode_content, flush_decoder) + self._decoded_buffer.put(decoded_data) + + while len(self._decoded_buffer) < amt and data: + # TODO make sure to initially read enough data to get past the headers + # For example, the GZ file header takes 10 bytes, we don't want to read + # it one byte at a time + data = self._raw_read(amt) + decoded_data = self._decode(data, decode_content, flush_decoder) + self._decoded_buffer.put(decoded_data) + data = self._decoded_buffer.get(amt) return data - def stream(self, amt=2 ** 16, decode_content=None): + def read1( + self, + amt: int | None = None, + decode_content: bool | None = None, + ) -> bytes: + """ + Similar to ``http.client.HTTPResponse.read1`` and documented + in :meth:`io.BufferedReader.read1`, but with an additional parameter: + ``decode_content``. + + :param amt: + How much of the content to read. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + if decode_content is None: + decode_content = self.decode_content + if amt and amt < 0: + # Negative numbers and `None` should be treated the same. + amt = None + # try and respond without going to the network + if self._has_decoded_content: + if not decode_content: + raise RuntimeError( + "Calling read1(decode_content=False) is not supported after " + "read1(decode_content=True) was called." + ) + if len(self._decoded_buffer) > 0: + if amt is None: + return self._decoded_buffer.get_all() + return self._decoded_buffer.get(amt) + if amt == 0: + return b"" + + # FIXME, this method's type doesn't say returning None is possible + data = self._raw_read(amt, read1=True) + if not decode_content or data is None: + return data + + self._init_decoder() + while True: + flush_decoder = not data + decoded_data = self._decode(data, decode_content, flush_decoder) + self._decoded_buffer.put(decoded_data) + if decoded_data or flush_decoder: + break + data = self._raw_read(8192, read1=True) + + if amt is None: + return self._decoded_buffer.get_all() + return self._decoded_buffer.get(amt) + + def stream( + self, amt: int | None = 2**16, decode_content: bool | None = None + ) -> typing.Generator[bytes]: """ A generator wrapper for the read() method. A call will block until ``amt`` bytes have been read from the connection or until the @@ -621,73 +1060,27 @@ def stream(self, amt=2 ** 16, decode_content=None): 'content-encoding' header. """ if self.chunked and self.supports_chunked_reads(): - for line in self.read_chunked(amt, decode_content=decode_content): - yield line + yield from self.read_chunked(amt, decode_content=decode_content) else: - while not is_fp_closed(self._fp): + while not is_fp_closed(self._fp) or len(self._decoded_buffer) > 0: data = self.read(amt=amt, decode_content=decode_content) if data: yield data - @classmethod - def from_httplib(ResponseCls, r, **response_kw): - """ - Given an :class:`http.client.HTTPResponse` instance ``r``, return a - corresponding :class:`urllib3.response.HTTPResponse` object. - - Remaining parameters are passed to the HTTPResponse constructor, along - with ``original_response=r``. - """ - headers = r.msg - - if not isinstance(headers, HTTPHeaderDict): - if six.PY2: - # Python 2.7 - headers = HTTPHeaderDict.from_httplib(headers) - else: - headers = HTTPHeaderDict(headers.items()) - - # HTTPResponse objects in Python 3 don't have a .strict attribute - strict = getattr(r, "strict", 0) - resp = ResponseCls( - body=r, - headers=headers, - status=r.status, - version=r.version, - reason=r.reason, - strict=strict, - original_response=r, - **response_kw - ) - return resp - - # Backwards-compatibility methods for http.client.HTTPResponse - def getheaders(self): - warnings.warn( - "HTTPResponse.getheaders() is deprecated and will be removed " - "in urllib3 v2.1.0. Instead access HTTPResponse.headers directly.", - category=DeprecationWarning, - stacklevel=2, - ) - return self.headers + # Overrides from io.IOBase + def readable(self) -> bool: + return True - def getheader(self, name, default=None): - warnings.warn( - "HTTPResponse.getheader() is deprecated and will be removed " - "in urllib3 v2.1.0. Instead use HTTPResponse.headers.get(name, default).", - category=DeprecationWarning, - stacklevel=2, - ) - return self.headers.get(name, default) + def shutdown(self) -> None: + if not self._sock_shutdown: + raise ValueError("Cannot shutdown socket as self._sock_shutdown is not set") + self._sock_shutdown(socket.SHUT_RD) - # Backwards compatibility for http.cookiejar - def info(self): - return self.headers + def close(self) -> None: + self._sock_shutdown = None - # Overrides from io.IOBase - def close(self): - if not self.closed: + if not self.closed and self._fp: self._fp.close() if self._connection: @@ -697,9 +1090,9 @@ def close(self): io.IOBase.close(self) @property - def closed(self): + def closed(self) -> bool: if not self.auto_close: - return io.IOBase.closed.__get__(self) + return io.IOBase.closed.__get__(self) # type: ignore[no-any-return] elif self._fp is None: return True elif hasattr(self._fp, "isclosed"): @@ -709,18 +1102,18 @@ def closed(self): else: return True - def fileno(self): + def fileno(self) -> int: if self._fp is None: - raise IOError("HTTPResponse has no file to get a fileno from") + raise OSError("HTTPResponse has no file to get a fileno from") elif hasattr(self._fp, "fileno"): return self._fp.fileno() else: - raise IOError( + raise OSError( "The file-like object this HTTPResponse is wrapped " "around has no file descriptor" ) - def flush(self): + def flush(self) -> None: if ( self._fp is not None and hasattr(self._fp, "flush") @@ -728,20 +1121,7 @@ def flush(self): ): return self._fp.flush() - def readable(self): - # This method is required for `io` module compatibility. - return True - - def readinto(self, b): - # This method is required for `io` module compatibility. - temp = self.read(len(b)) - if len(temp) == 0: - return 0 - else: - b[: len(temp)] = temp - return len(temp) - - def supports_chunked_reads(self): + def supports_chunked_reads(self) -> bool: """ Checks if the underlying file-like object looks like a :class:`http.client.HTTPResponse` object. We do this by testing for @@ -750,43 +1130,49 @@ def supports_chunked_reads(self): """ return hasattr(self._fp, "fp") - def _update_chunk_length(self): + def _update_chunk_length(self) -> None: # First, we'll figure out length of a chunk and then # we'll try to read it from socket. if self.chunk_left is not None: - return - line = self._fp.fp.readline() + return None + line = self._fp.fp.readline() # type: ignore[union-attr] line = line.split(b";", 1)[0] try: self.chunk_left = int(line, 16) except ValueError: - # Invalid chunked protocol response, abort. self.close() - raise InvalidChunkLength(self, line) + if line: + # Invalid chunked protocol response, abort. + raise InvalidChunkLength(self, line) from None + else: + # Truncated at start of next chunk + raise ProtocolError("Response ended prematurely") from None - def _handle_chunk(self, amt): + def _handle_chunk(self, amt: int | None) -> bytes: returned_chunk = None if amt is None: - chunk = self._fp._safe_read(self.chunk_left) + chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr] returned_chunk = chunk - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk. self.chunk_left = None - elif amt < self.chunk_left: - value = self._fp._safe_read(amt) + elif self.chunk_left is not None and amt < self.chunk_left: + value = self._fp._safe_read(amt) # type: ignore[union-attr] self.chunk_left = self.chunk_left - amt returned_chunk = value elif amt == self.chunk_left: - value = self._fp._safe_read(amt) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + value = self._fp._safe_read(amt) # type: ignore[union-attr] + self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk. self.chunk_left = None returned_chunk = value else: # amt > self.chunk_left - returned_chunk = self._fp._safe_read(self.chunk_left) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + returned_chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr] + self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk. self.chunk_left = None - return returned_chunk + return returned_chunk # type: ignore[no-any-return] - def read_chunked(self, amt=None, decode_content=None): + def read_chunked( + self, amt: int | None = None, decode_content: bool | None = None + ) -> typing.Generator[bytes]: """ Similar to :meth:`HTTPResponse.read`, but with an additional parameter: ``decode_content``. @@ -817,12 +1203,17 @@ def read_chunked(self, amt=None, decode_content=None): # Don't bother reading the body of a HEAD request. if self._original_response and is_response_to_head(self._original_response): self._original_response.close() - return + return None # If a response is already read and closed # then return immediately. - if self._fp.fp is None: - return + if self._fp.fp is None: # type: ignore[union-attr] + return None + + if amt and amt < 0: + # Negative numbers and `None` should be treated the same, + # but httplib handles only `None` correctly. + amt = None while True: self._update_chunk_length() @@ -844,7 +1235,7 @@ def read_chunked(self, amt=None, decode_content=None): yield decoded # Chunk content ends with \r\n: discard it. - while True: + while self._fp is not None: line = self._fp.fp.readline() if not line: # Some sites may not end with '\r\n'. @@ -856,27 +1247,29 @@ def read_chunked(self, amt=None, decode_content=None): if self._original_response: self._original_response.close() - def geturl(self): + @property + def url(self) -> str | None: """ Returns the URL that was the source of this response. If the request that generated this response redirected, this method will return the final redirect location. """ - if self.retries is not None and len(self.retries.history): - return self.retries.history[-1].redirect_location - else: - return self._request_url + return self._request_url + + @url.setter + def url(self, url: str) -> None: + self._request_url = url - def __iter__(self): - buffer = [] + def __iter__(self) -> typing.Iterator[bytes]: + buffer: list[bytes] = [] for chunk in self.stream(decode_content=True): if b"\n" in chunk: - chunk = chunk.split(b"\n") - yield b"".join(buffer) + chunk[0] + b"\n" - for x in chunk[1:-1]: + chunks = chunk.split(b"\n") + yield b"".join(buffer) + chunks[0] + b"\n" + for x in chunks[1:-1]: yield x + b"\n" - if chunk[-1]: - buffer = [chunk[-1]] + if chunks[-1]: + buffer = [chunks[-1]] else: buffer = [] else: diff --git a/src/urllib3/util/__init__.py b/src/urllib3/util/__init__.py index 4547fc5..5341260 100644 --- a/src/urllib3/util/__init__.py +++ b/src/urllib3/util/__init__.py @@ -1,46 +1,39 @@ -from __future__ import absolute_import - # For backwards compatibility, provide imports that used to be here. +from __future__ import annotations + from .connection import is_connection_dropped from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers from .response import is_fp_closed from .retry import Retry from .ssl_ import ( ALPN_PROTOCOLS, - HAS_SNI, IS_PYOPENSSL, - IS_SECURETRANSPORT, - PROTOCOL_TLS, SSLContext, assert_fingerprint, + create_urllib3_context, resolve_cert_reqs, resolve_ssl_version, ssl_wrap_socket, ) -from .timeout import Timeout, current_time -from .url import Url, get_host, parse_url, split_first +from .timeout import Timeout +from .url import Url, parse_url from .wait import wait_for_read, wait_for_write __all__ = ( - "HAS_SNI", "IS_PYOPENSSL", - "IS_SECURETRANSPORT", "SSLContext", - "PROTOCOL_TLS", "ALPN_PROTOCOLS", "Retry", "Timeout", "Url", "assert_fingerprint", - "current_time", + "create_urllib3_context", "is_connection_dropped", "is_fp_closed", - "get_host", "parse_url", "make_headers", "resolve_cert_reqs", "resolve_ssl_version", - "split_first", "ssl_wrap_socket", "wait_for_read", "wait_for_write", diff --git a/src/urllib3/util/connection.py b/src/urllib3/util/connection.py index 6af1138..f92519e 100644 --- a/src/urllib3/util/connection.py +++ b/src/urllib3/util/connection.py @@ -1,33 +1,23 @@ -from __future__ import absolute_import +from __future__ import annotations import socket +import typing -from ..contrib import _appengine_environ from ..exceptions import LocationParseError -from ..packages import six -from .wait import NoWayToWaitForSocketError, wait_for_read +from .timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT +_TYPE_SOCKET_OPTIONS = list[tuple[int, int, typing.Union[int, bytes]]] -def is_connection_dropped(conn): # Platform-specific - """ - Returns True if the connection is dropped and should be closed. +if typing.TYPE_CHECKING: + from .._base_connection import BaseHTTPConnection - :param conn: - :class:`http.client.HTTPConnection` object. - Note: For platforms like AppEngine, this will always return ``False`` to - let the platform handle connection recycling transparently for us. +def is_connection_dropped(conn: BaseHTTPConnection) -> bool: # Platform-specific """ - sock = getattr(conn, "sock", False) - if sock is False: # Platform-specific: AppEngine - return False - if sock is None: # Connection already closed (such as by httplib). - return True - try: - # Returns True if readable, which here means it's been dropped - return wait_for_read(sock, timeout=0.0) - except NoWayToWaitForSocketError: # Platform-specific: AppEngine - return False + Returns True if the connection is dropped and should be closed. + :param conn: :class:`urllib3.connection.HTTPConnection` object. + """ + return not conn.is_connected # This function is copied from socket.py in the Python 2.7 standard @@ -35,11 +25,11 @@ def is_connection_dropped(conn): # Platform-specific # One additional modification is that we avoid binding to IPv6 servers # discovered in DNS if the system doesn't have IPv6 functionality. def create_connection( - address, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, - socket_options=None, -): + address: tuple[str, int], + timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + source_address: tuple[str, int] | None = None, + socket_options: _TYPE_SOCKET_OPTIONS | None = None, +) -> socket.socket: """Connect to *address* and return the socket object. Convenience function. Connect to *address* (a 2-tuple ``(host, @@ -65,9 +55,7 @@ def create_connection( try: host.encode("idna") except UnicodeError: - return six.raise_from( - LocationParseError(u"'%s', label empty or too long" % host), None - ) + raise LocationParseError(f"'{host}', label empty or too long") from None for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res @@ -78,26 +66,33 @@ def create_connection( # If provided, set socket level options before connecting. _set_socket_options(sock, socket_options) - if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + if timeout is not _DEFAULT_TIMEOUT: sock.settimeout(timeout) if source_address: sock.bind(source_address) sock.connect(sa) + # Break explicitly a reference cycle + err = None return sock - except socket.error as e: - err = e + except OSError as _: + err = _ if sock is not None: sock.close() - sock = None if err is not None: - raise err - - raise socket.error("getaddrinfo returns an empty list") + try: + raise err + finally: + # Break explicitly a reference cycle + err = None + else: + raise OSError("getaddrinfo returns an empty list") -def _set_socket_options(sock, options): +def _set_socket_options( + sock: socket.socket, options: _TYPE_SOCKET_OPTIONS | None +) -> None: if options is None: return @@ -105,7 +100,7 @@ def _set_socket_options(sock, options): sock.setsockopt(*opt) -def allowed_gai_family(): +def allowed_gai_family() -> socket.AddressFamily: """This function is designed to work in the context of getaddrinfo, where family=socket.AF_UNSPEC is the default and will perform a DNS search for both IPv6 and IPv4 records.""" @@ -116,18 +111,11 @@ def allowed_gai_family(): return family -def _has_ipv6(host): +def _has_ipv6(host: str) -> bool: """Returns True if the system can bind an IPv6 address.""" sock = None has_ipv6 = False - # App Engine doesn't support IPV6 sockets and actually has a quota on the - # number of sockets that can be used, so just early out here instead of - # creating a socket needlessly. - # See https://github.com/urllib3/urllib3/issues/1446 - if _appengine_environ.is_appengine_sandbox(): - return False - if socket.has_ipv6: # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To diff --git a/src/urllib3/util/proxy.py b/src/urllib3/util/proxy.py index 2199cc7..908fc66 100644 --- a/src/urllib3/util/proxy.py +++ b/src/urllib3/util/proxy.py @@ -1,9 +1,18 @@ -from .ssl_ import create_urllib3_context, resolve_cert_reqs, resolve_ssl_version +from __future__ import annotations + +import typing + +from .url import Url + +if typing.TYPE_CHECKING: + from ..connection import ProxyConfig def connection_requires_http_tunnel( - proxy_url=None, proxy_config=None, destination_scheme=None -): + proxy_url: Url | None = None, + proxy_config: ProxyConfig | None = None, + destination_scheme: str | None = None, +) -> bool: """ Returns True if the connection requires an HTTP CONNECT through the proxy. @@ -32,26 +41,3 @@ def connection_requires_http_tunnel( # Otherwise always use a tunnel. return True - - -def create_proxy_ssl_context( - ssl_version, cert_reqs, ca_certs=None, ca_cert_dir=None, ca_cert_data=None -): - """ - Generates a default proxy ssl context if one hasn't been provided by the - user. - """ - ssl_context = create_urllib3_context( - ssl_version=resolve_ssl_version(ssl_version), - cert_reqs=resolve_cert_reqs(cert_reqs), - ) - - if ( - not ca_certs - and not ca_cert_dir - and not ca_cert_data - and hasattr(ssl_context, "load_default_certs") - ): - ssl_context.load_default_certs() - - return ssl_context diff --git a/src/urllib3/util/queue.py b/src/urllib3/util/queue.py deleted file mode 100644 index 4178410..0000000 --- a/src/urllib3/util/queue.py +++ /dev/null @@ -1,22 +0,0 @@ -import collections - -from ..packages import six -from ..packages.six.moves import queue - -if six.PY2: - # Queue is imported for side effects on MS Windows. See issue #229. - import Queue as _unused_module_Queue # noqa: F401 - - -class LifoQueue(queue.Queue): - def _init(self, _): - self.queue = collections.deque() - - def _qsize(self, len=len): - return len(self.queue) - - def _put(self, item): - self.queue.append(item) - - def _get(self): - return self.queue.pop() diff --git a/src/urllib3/util/request.py b/src/urllib3/util/request.py index b574b08..94392a1 100644 --- a/src/urllib3/util/request.py +++ b/src/urllib3/util/request.py @@ -1,9 +1,15 @@ -from __future__ import absolute_import +from __future__ import annotations +import io +import typing from base64 import b64encode +from enum import Enum from ..exceptions import UnrewindableBodyError -from ..packages.six import b, integer_types +from .util import to_bytes + +if typing.TYPE_CHECKING: + from typing import Final # Pass as a value within ``headers`` to skip # emitting some HTTP headers that are added automatically. @@ -15,25 +21,45 @@ ACCEPT_ENCODING = "gzip,deflate" try: try: - import brotlicffi as _unused_module_brotli # noqa: F401 + import brotlicffi as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401 except ImportError: - import brotli as _unused_module_brotli # noqa: F401 + import brotli as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401 except ImportError: pass else: ACCEPT_ENCODING += ",br" +try: + import zstandard as _unused_module_zstd # noqa: F401 +except ImportError: + pass +else: + ACCEPT_ENCODING += ",zstd" + + +class _TYPE_FAILEDTELL(Enum): + token = 0 + -_FAILEDTELL = object() +_FAILEDTELL: Final[_TYPE_FAILEDTELL] = _TYPE_FAILEDTELL.token + +_TYPE_BODY_POSITION = typing.Union[int, _TYPE_FAILEDTELL] + +# When sending a request with these methods we aren't expecting +# a body so don't need to set an explicit 'Content-Length: 0' +# The reason we do this in the negative instead of tracking methods +# which 'should' have a body is because unknown methods should be +# treated as if they were 'POST' which *does* expect a body. +_METHODS_NOT_EXPECTING_BODY = {"GET", "HEAD", "DELETE", "TRACE", "OPTIONS", "CONNECT"} def make_headers( - keep_alive=None, - accept_encoding=None, - user_agent=None, - basic_auth=None, - proxy_basic_auth=None, - disable_cache=None, -): + keep_alive: bool | None = None, + accept_encoding: bool | list[str] | str | None = None, + user_agent: str | None = None, + basic_auth: str | None = None, + proxy_basic_auth: str | None = None, + disable_cache: bool | None = None, +) -> dict[str, str]: """ Shortcuts for generating request headers. @@ -42,7 +68,10 @@ def make_headers( :param accept_encoding: Can be a boolean, list, or string. - ``True`` translates to 'gzip,deflate'. + ``True`` translates to 'gzip,deflate'. If the dependencies for + Brotli (either the ``brotli`` or ``brotlicffi`` package) and/or Zstandard + (the ``zstandard`` package) algorithms are installed, then their encodings are + included in the string ('br' and 'zstd', respectively). List will get joined by comma. String will be used as provided. @@ -61,14 +90,18 @@ def make_headers( :param disable_cache: If ``True``, adds 'cache-control: no-cache' header. - Example:: + Example: + + .. code-block:: python + + import urllib3 - >>> make_headers(keep_alive=True, user_agent="Batman/1.0") - {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} - >>> make_headers(accept_encoding=True) - {'accept-encoding': 'gzip,deflate'} + print(urllib3.util.make_headers(keep_alive=True, user_agent="Batman/1.0")) + # {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + print(urllib3.util.make_headers(accept_encoding=True)) + # {'accept-encoding': 'gzip,deflate'} """ - headers = {} + headers: dict[str, str] = {} if accept_encoding: if isinstance(accept_encoding, str): pass @@ -85,12 +118,14 @@ def make_headers( headers["connection"] = "keep-alive" if basic_auth: - headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8") + headers["authorization"] = ( + f"Basic {b64encode(basic_auth.encode('latin-1')).decode()}" + ) if proxy_basic_auth: - headers["proxy-authorization"] = "Basic " + b64encode( - b(proxy_basic_auth) - ).decode("utf-8") + headers["proxy-authorization"] = ( + f"Basic {b64encode(proxy_basic_auth.encode('latin-1')).decode()}" + ) if disable_cache: headers["cache-control"] = "no-cache" @@ -98,7 +133,9 @@ def make_headers( return headers -def set_file_position(body, pos): +def set_file_position( + body: typing.Any, pos: _TYPE_BODY_POSITION | None +) -> _TYPE_BODY_POSITION | None: """ If a position is provided, move file to that point. Otherwise, we'll attempt to record a position for future use. @@ -108,7 +145,7 @@ def set_file_position(body, pos): elif getattr(body, "tell", None) is not None: try: pos = body.tell() - except (IOError, OSError): + except OSError: # This differentiates from None, allowing us to catch # a failed `tell()` later when trying to rewind the body. pos = _FAILEDTELL @@ -116,7 +153,7 @@ def set_file_position(body, pos): return pos -def rewind_body(body, body_pos): +def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -> None: """ Attempt to rewind body to a certain position. Primarily used for request redirects and retries. @@ -128,13 +165,13 @@ def rewind_body(body, body_pos): Position to seek to in file. """ body_seek = getattr(body, "seek", None) - if body_seek is not None and isinstance(body_pos, integer_types): + if body_seek is not None and isinstance(body_pos, int): try: body_seek(body_pos) - except (IOError, OSError): + except OSError as e: raise UnrewindableBodyError( "An error occurred when rewinding request body for redirect/retry." - ) + ) from e elif body_pos is _FAILEDTELL: raise UnrewindableBodyError( "Unable to record file position for rewinding " @@ -142,5 +179,80 @@ def rewind_body(body, body_pos): ) else: raise ValueError( - "body_pos must be of type integer, instead it was %s." % type(body_pos) + f"body_pos must be of type integer, instead it was {type(body_pos)}." ) + + +class ChunksAndContentLength(typing.NamedTuple): + chunks: typing.Iterable[bytes] | None + content_length: int | None + + +def body_to_chunks( + body: typing.Any | None, method: str, blocksize: int +) -> ChunksAndContentLength: + """Takes the HTTP request method, body, and blocksize and + transforms them into an iterable of chunks to pass to + socket.sendall() and an optional 'Content-Length' header. + + A 'Content-Length' of 'None' indicates the length of the body + can't be determined so should use 'Transfer-Encoding: chunked' + for framing instead. + """ + + chunks: typing.Iterable[bytes] | None + content_length: int | None + + # No body, we need to make a recommendation on 'Content-Length' + # based on whether that request method is expected to have + # a body or not. + if body is None: + chunks = None + if method.upper() not in _METHODS_NOT_EXPECTING_BODY: + content_length = 0 + else: + content_length = None + + # Bytes or strings become bytes + elif isinstance(body, (str, bytes)): + chunks = (to_bytes(body),) + content_length = len(chunks[0]) + + # File-like object, TODO: use seek() and tell() for length? + elif hasattr(body, "read"): + + def chunk_readable() -> typing.Iterable[bytes]: + nonlocal body, blocksize + encode = isinstance(body, io.TextIOBase) + while True: + datablock = body.read(blocksize) + if not datablock: + break + if encode: + datablock = datablock.encode("utf-8") + yield datablock + + chunks = chunk_readable() + content_length = None + + # Otherwise we need to start checking via duck-typing. + else: + try: + # Check if the body implements the buffer API. + mv = memoryview(body) + except TypeError: + try: + # Check if the body is an iterable + chunks = iter(body) + content_length = None + except TypeError: + raise TypeError( + f"'body' must be a bytes-like object, file-like " + f"object, or iterable. Instead was {body!r}" + ) from None + else: + # Since it implements the buffer API can be passed directly to socket.sendall() + chunks = (body,) + content_length = mv.nbytes + + return ChunksAndContentLength(chunks=chunks, content_length=content_length) diff --git a/src/urllib3/util/response.py b/src/urllib3/util/response.py index 5ea609c..0f45786 100644 --- a/src/urllib3/util/response.py +++ b/src/urllib3/util/response.py @@ -1,12 +1,12 @@ -from __future__ import absolute_import +from __future__ import annotations +import http.client as httplib from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect from ..exceptions import HeaderParsingError -from ..packages.six.moves import http_client as httplib -def is_fp_closed(obj): +def is_fp_closed(obj: object) -> bool: """ Checks whether a given file-like object is closed. @@ -17,27 +17,27 @@ def is_fp_closed(obj): try: # Check `isclosed()` first, in case Python3 doesn't set `closed`. # GH Issue #928 - return obj.isclosed() + return obj.isclosed() # type: ignore[no-any-return, attr-defined] except AttributeError: pass try: # Check via the official file-like-object way. - return obj.closed + return obj.closed # type: ignore[no-any-return, attr-defined] except AttributeError: pass try: # Check if the object is a container for another file-like object that # gets released on exhaustion (e.g. HTTPResponse). - return obj.fp is None + return obj.fp is None # type: ignore[attr-defined] except AttributeError: pass raise ValueError("Unable to determine whether fp is closed.") -def assert_header_parsing(headers): +def assert_header_parsing(headers: httplib.HTTPMessage) -> None: """ Asserts whether all headers have been successfully parsed. Extracts encountered errors from the result of parsing headers. @@ -53,55 +53,49 @@ def assert_header_parsing(headers): # This will fail silently if we pass in the wrong kind of parameter. # To make debugging easier add an explicit check. if not isinstance(headers, httplib.HTTPMessage): - raise TypeError("expected httplib.Message, got {0}.".format(type(headers))) - - defects = getattr(headers, "defects", None) - get_payload = getattr(headers, "get_payload", None) + raise TypeError(f"expected httplib.Message, got {type(headers)}.") unparsed_data = None - if get_payload: - # get_payload is actually email.message.Message.get_payload; - # we're only interested in the result if it's not a multipart message - if not headers.is_multipart(): - payload = get_payload() - - if isinstance(payload, (bytes, str)): - unparsed_data = payload - if defects: - # httplib is assuming a response body is available - # when parsing headers even when httplib only sends - # header data to parse_headers() This results in - # defects on multipart responses in particular. - # See: https://github.com/urllib3/urllib3/issues/800 - - # So we ignore the following defects: - # - StartBoundaryNotFoundDefect: - # The claimed start boundary was never found. - # - MultipartInvariantViolationDefect: - # A message claimed to be a multipart but no subparts were found. - defects = [ - defect - for defect in defects - if not isinstance( - defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect) - ) - ] + + # get_payload is actually email.message.Message.get_payload; + # we're only interested in the result if it's not a multipart message + if not headers.is_multipart(): + payload = headers.get_payload() + + if isinstance(payload, (bytes, str)): + unparsed_data = payload + + # httplib is assuming a response body is available + # when parsing headers even when httplib only sends + # header data to parse_headers() This results in + # defects on multipart responses in particular. + # See: https://github.com/urllib3/urllib3/issues/800 + + # So we ignore the following defects: + # - StartBoundaryNotFoundDefect: + # The claimed start boundary was never found. + # - MultipartInvariantViolationDefect: + # A message claimed to be a multipart but no subparts were found. + defects = [ + defect + for defect in headers.defects + if not isinstance( + defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect) + ) + ] if defects or unparsed_data: raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) -def is_response_to_head(response): +def is_response_to_head(response: httplib.HTTPResponse) -> bool: """ Checks whether the request of a response has been a HEAD-request. - Handles the quirks of AppEngine. :param http.client.HTTPResponse response: Response to check if the originating request used 'HEAD' as a method. """ # FIXME: Can we do this somehow without accessing private httplib _method? - method = response._method - if isinstance(method, int): # Platform-specific: Appengine - return method == 3 - return method.upper() == "HEAD" + method_str = response._method # type: str # type: ignore[attr-defined] + return method_str.upper() == "HEAD" diff --git a/src/urllib3/util/retry.py b/src/urllib3/util/retry.py index 60ef6c4..0456cce 100644 --- a/src/urllib3/util/retry.py +++ b/src/urllib3/util/retry.py @@ -1,12 +1,13 @@ -from __future__ import absolute_import +from __future__ import annotations import email import logging +import random import re import time -import warnings -from collections import namedtuple +import typing from itertools import takewhile +from types import TracebackType from ..exceptions import ( ConnectTimeoutError, @@ -17,97 +18,51 @@ ReadTimeoutError, ResponseError, ) -from ..packages import six +from .util import reraise -log = logging.getLogger(__name__) - - -# Data structure for representing the metadata of requests that result in a retry. -RequestHistory = namedtuple( - "RequestHistory", ["method", "url", "error", "status", "redirect_location"] -) +if typing.TYPE_CHECKING: + from typing_extensions import Self + from ..connectionpool import ConnectionPool + from ..response import BaseHTTPResponse -# TODO: In v2 we can remove this sentinel and metaclass with deprecated options. -_Default = object() +log = logging.getLogger(__name__) -class _RetryMeta(type): - @property - def DEFAULT_METHOD_WHITELIST(cls): - warnings.warn( - "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead", - DeprecationWarning, - ) - return cls.DEFAULT_ALLOWED_METHODS - - @DEFAULT_METHOD_WHITELIST.setter - def DEFAULT_METHOD_WHITELIST(cls, value): - warnings.warn( - "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead", - DeprecationWarning, - ) - cls.DEFAULT_ALLOWED_METHODS = value - - @property - def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls): - warnings.warn( - "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead", - DeprecationWarning, - ) - return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT - - @DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter - def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value): - warnings.warn( - "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead", - DeprecationWarning, - ) - cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value - - @property - def BACKOFF_MAX(cls): - warnings.warn( - "Using 'Retry.BACKOFF_MAX' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead", - DeprecationWarning, - ) - return cls.DEFAULT_BACKOFF_MAX - - @BACKOFF_MAX.setter - def BACKOFF_MAX(cls, value): - warnings.warn( - "Using 'Retry.BACKOFF_MAX' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead", - DeprecationWarning, - ) - cls.DEFAULT_BACKOFF_MAX = value +# Data structure for representing the metadata of requests that result in a retry. +class RequestHistory(typing.NamedTuple): + method: str | None + url: str | None + error: Exception | None + status: int | None + redirect_location: str | None -@six.add_metaclass(_RetryMeta) -class Retry(object): +class Retry: """Retry configuration. Each retry attempt will create a new Retry object with updated values, so they can be safely reused. - Retries can be defined as a default for a pool:: + Retries can be defined as a default for a pool: + + .. code-block:: python retries = Retry(connect=5, read=2, redirect=5) http = PoolManager(retries=retries) - response = http.request('GET', 'http://example.com/') + response = http.request("GET", "https://example.com/") + + Or per-request (which overrides the default for the pool): - Or per-request (which overrides the default for the pool):: + .. code-block:: python - response = http.request('GET', 'http://example.com/', retries=Retry(10)) + response = http.request("GET", "https://example.com/", retries=Retry(10)) - Retries can be disabled by passing ``False``:: + Retries can be disabled by passing ``False``: - response = http.request('GET', 'http://example.com/', retries=False) + .. code-block:: python + + response = http.request("GET", "https://example.com/", retries=False) Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless retries are disabled, in which case the causing exception will be raised. @@ -169,21 +124,16 @@ class Retry(object): If ``total`` is not set, it's a good idea to set this to 0 to account for unexpected edge cases and avoid infinite retry loops. - :param iterable allowed_methods: + :param Collection allowed_methods: Set of uppercased HTTP method verbs that we should retry on. By default, we only retry on methods which are considered to be idempotent (multiple requests with the same parameters end with the same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`. - Set to a ``False`` value to retry on any verb. - - .. warning:: + Set to a ``None`` value to retry on any verb. - Previously this parameter was named ``method_whitelist``, that - usage is deprecated in v1.26.0 and will be removed in v2.0. - - :param iterable status_forcelist: + :param Collection status_forcelist: A set of integer HTTP status codes that we should force a retry on. A retry is initiated if the request method is in ``allowed_methods`` and the response status code is in ``status_forcelist``. @@ -195,13 +145,17 @@ class Retry(object): (most errors are resolved immediately by a second try without a delay). urllib3 will sleep for:: - {backoff factor} * (2 ** ({number of total retries} - 1)) + {backoff factor} * (2 ** ({number of previous retries})) + + seconds. If `backoff_jitter` is non-zero, this sleep is extended by:: - seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep - for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer - than :attr:`Retry.DEFAULT_BACKOFF_MAX`. + random.uniform(0, {backoff jitter}) - By default, backoff is disabled (set to 0). + seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will + sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever + be longer than `backoff_max`. + + By default, backoff is disabled (factor set to 0). :param bool raise_on_redirect: Whether, if the number of redirects is exhausted, to raise a MaxRetryError, or to return a response with a @@ -220,7 +174,7 @@ class Retry(object): Whether to respect Retry-After header on status codes defined as :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. - :param iterable remove_headers_on_redirect: + :param Collection remove_headers_on_redirect: Sequence of headers to remove from the request when a response indicating a redirect is returned before firing off the redirected request. @@ -235,50 +189,37 @@ class Retry(object): RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) #: Default headers to be used for ``remove_headers_on_redirect`` - DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( + ["Cookie", "Authorization", "Proxy-Authorization"] + ) - #: Maximum backoff time. + #: Default maximum backoff time. DEFAULT_BACKOFF_MAX = 120 + # Backward compatibility; assigned outside of the class. + DEFAULT: typing.ClassVar[Retry] + def __init__( self, - total=10, - connect=None, - read=None, - redirect=None, - status=None, - other=None, - allowed_methods=_Default, - status_forcelist=None, - backoff_factor=0, - raise_on_redirect=True, - raise_on_status=True, - history=None, - respect_retry_after_header=True, - remove_headers_on_redirect=_Default, - # TODO: Deprecated, remove in v2.0 - method_whitelist=_Default, - ): - - if method_whitelist is not _Default: - if allowed_methods is not _Default: - raise ValueError( - "Using both 'allowed_methods' and " - "'method_whitelist' together is not allowed. " - "Instead only use 'allowed_methods'" - ) - warnings.warn( - "Using 'method_whitelist' with Retry is deprecated and " - "will be removed in v2.0. Use 'allowed_methods' instead", - DeprecationWarning, - stacklevel=2, - ) - allowed_methods = method_whitelist - if allowed_methods is _Default: - allowed_methods = self.DEFAULT_ALLOWED_METHODS - if remove_headers_on_redirect is _Default: - remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT - + total: bool | int | None = 10, + connect: int | None = None, + read: int | None = None, + redirect: bool | int | None = None, + status: int | None = None, + other: int | None = None, + allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS, + status_forcelist: typing.Collection[int] | None = None, + backoff_factor: float = 0, + backoff_max: float = DEFAULT_BACKOFF_MAX, + raise_on_redirect: bool = True, + raise_on_status: bool = True, + history: tuple[RequestHistory, ...] | None = None, + respect_retry_after_header: bool = True, + remove_headers_on_redirect: typing.Collection[ + str + ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT, + backoff_jitter: float = 0.0, + ) -> None: self.total = total self.connect = connect self.read = read @@ -293,15 +234,17 @@ def __init__( self.status_forcelist = status_forcelist or set() self.allowed_methods = allowed_methods self.backoff_factor = backoff_factor + self.backoff_max = backoff_max self.raise_on_redirect = raise_on_redirect self.raise_on_status = raise_on_status - self.history = history or tuple() + self.history = history or () self.respect_retry_after_header = respect_retry_after_header self.remove_headers_on_redirect = frozenset( - [h.lower() for h in remove_headers_on_redirect] + h.lower() for h in remove_headers_on_redirect ) + self.backoff_jitter = backoff_jitter - def new(self, **kw): + def new(self, **kw: typing.Any) -> Self: params = dict( total=self.total, connect=self.connect, @@ -309,36 +252,28 @@ def new(self, **kw): redirect=self.redirect, status=self.status, other=self.other, + allowed_methods=self.allowed_methods, status_forcelist=self.status_forcelist, backoff_factor=self.backoff_factor, + backoff_max=self.backoff_max, raise_on_redirect=self.raise_on_redirect, raise_on_status=self.raise_on_status, history=self.history, remove_headers_on_redirect=self.remove_headers_on_redirect, respect_retry_after_header=self.respect_retry_after_header, + backoff_jitter=self.backoff_jitter, ) - # TODO: If already given in **kw we use what's given to us - # If not given we need to figure out what to pass. We decide - # based on whether our class has the 'method_whitelist' property - # and if so we pass the deprecated 'method_whitelist' otherwise - # we use 'allowed_methods'. Remove in v2.0 - if "method_whitelist" not in kw and "allowed_methods" not in kw: - if "method_whitelist" in self.__dict__: - warnings.warn( - "Using 'method_whitelist' with Retry is deprecated and " - "will be removed in v2.0. Use 'allowed_methods' instead", - DeprecationWarning, - ) - params["method_whitelist"] = self.allowed_methods - else: - params["allowed_methods"] = self.allowed_methods - params.update(kw) - return type(self)(**params) + return type(self)(**params) # type: ignore[arg-type] @classmethod - def from_int(cls, retries, redirect=True, default=None): + def from_int( + cls, + retries: Retry | bool | int | None, + redirect: bool | int | None = True, + default: Retry | bool | int | None = None, + ) -> Retry: """Backwards-compatibility for the old retries format.""" if retries is None: retries = default if default is not None else cls.DEFAULT @@ -351,7 +286,7 @@ def from_int(cls, retries, redirect=True, default=None): log.debug("Converted retries value: %r -> %r", retries, new_retries) return new_retries - def get_backoff_time(self): + def get_backoff_time(self) -> float: """Formula for computing the current backoff :rtype: float @@ -366,32 +301,28 @@ def get_backoff_time(self): return 0 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) - return min(self.DEFAULT_BACKOFF_MAX, backoff_value) + if self.backoff_jitter != 0.0: + backoff_value += random.random() * self.backoff_jitter + return float(max(0, min(self.backoff_max, backoff_value))) - def parse_retry_after(self, retry_after): + def parse_retry_after(self, retry_after: str) -> float: + seconds: float # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 if re.match(r"^\s*[0-9]+\s*$", retry_after): seconds = int(retry_after) else: retry_date_tuple = email.utils.parsedate_tz(retry_after) if retry_date_tuple is None: - raise InvalidHeader("Invalid Retry-After header: %s" % retry_after) - if retry_date_tuple[9] is None: # Python 2 - # Assume UTC if no timezone was specified - # On Python2.7, parsedate_tz returns None for a timezone offset - # instead of 0 if no timezone is given, where mktime_tz treats - # a None timezone offset as local time. - retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:] + raise InvalidHeader(f"Invalid Retry-After header: {retry_after}") retry_date = email.utils.mktime_tz(retry_date_tuple) seconds = retry_date - time.time() - if seconds < 0: - seconds = 0 + seconds = max(seconds, 0) return seconds - def get_retry_after(self, response): + def get_retry_after(self, response: BaseHTTPResponse) -> float | None: """Get the value of Retry-After in seconds.""" retry_after = response.headers.get("Retry-After") @@ -401,7 +332,7 @@ def get_retry_after(self, response): return self.parse_retry_after(retry_after) - def sleep_for_retry(self, response=None): + def sleep_for_retry(self, response: BaseHTTPResponse) -> bool: retry_after = self.get_retry_after(response) if retry_after: time.sleep(retry_after) @@ -409,13 +340,13 @@ def sleep_for_retry(self, response=None): return False - def _sleep_backoff(self): + def _sleep_backoff(self) -> None: backoff = self.get_backoff_time() if backoff <= 0: return time.sleep(backoff) - def sleep(self, response=None): + def sleep(self, response: BaseHTTPResponse | None = None) -> None: """Sleep between retry attempts. This method will respect a server's ``Retry-After`` response header @@ -431,7 +362,7 @@ def sleep(self, response=None): self._sleep_backoff() - def _is_connection_error(self, err): + def _is_connection_error(self, err: Exception) -> bool: """Errors when we're fairly sure that the server did not receive the request, so it should be safe to retry. """ @@ -439,33 +370,23 @@ def _is_connection_error(self, err): err = err.original_error return isinstance(err, ConnectTimeoutError) - def _is_read_error(self, err): + def _is_read_error(self, err: Exception) -> bool: """Errors that occur after the request has been started, so we should assume that the server began processing it. """ return isinstance(err, (ReadTimeoutError, ProtocolError)) - def _is_method_retryable(self, method): + def _is_method_retryable(self, method: str) -> bool: """Checks if a given HTTP method should be retried upon, depending if it is included in the allowed_methods """ - # TODO: For now favor if the Retry implementation sets its own method_whitelist - # property outside of our constructor to avoid breaking custom implementations. - if "method_whitelist" in self.__dict__: - warnings.warn( - "Using 'method_whitelist' with Retry is deprecated and " - "will be removed in v2.0. Use 'allowed_methods' instead", - DeprecationWarning, - ) - allowed_methods = self.method_whitelist - else: - allowed_methods = self.allowed_methods - - if allowed_methods and method.upper() not in allowed_methods: + if self.allowed_methods and method.upper() not in self.allowed_methods: return False return True - def is_retry(self, method, status_code, has_retry_after=False): + def is_retry( + self, method: str, status_code: int, has_retry_after: bool = False + ) -> bool: """Is this method/status code retryable? (Based on allowlists and control variables such as the number of total retries to allow, whether to respect the Retry-After header, whether this header is present, and @@ -478,24 +399,27 @@ def is_retry(self, method, status_code, has_retry_after=False): if self.status_forcelist and status_code in self.status_forcelist: return True - return ( + return bool( self.total and self.respect_retry_after_header and has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES) ) - def is_exhausted(self): + def is_exhausted(self) -> bool: """Are we out of retries?""" - retry_counts = ( - self.total, - self.connect, - self.read, - self.redirect, - self.status, - self.other, - ) - retry_counts = list(filter(None, retry_counts)) + retry_counts = [ + x + for x in ( + self.total, + self.connect, + self.read, + self.redirect, + self.status, + self.other, + ) + if x + ] if not retry_counts: return False @@ -503,18 +427,18 @@ def is_exhausted(self): def increment( self, - method=None, - url=None, - response=None, - error=None, - _pool=None, - _stacktrace=None, - ): + method: str | None = None, + url: str | None = None, + response: BaseHTTPResponse | None = None, + error: Exception | None = None, + _pool: ConnectionPool | None = None, + _stacktrace: TracebackType | None = None, + ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. - :type response: :class:`~urllib3.response.HTTPResponse` + :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. @@ -522,7 +446,7 @@ def increment( """ if self.total is False and error: # Disabled, indicate to re-raise the error. - raise six.reraise(type(error), error, _stacktrace) + raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: @@ -540,14 +464,14 @@ def increment( if error and self._is_connection_error(error): # Connect retry? if connect is False: - raise six.reraise(type(error), error, _stacktrace) + raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? - if read is False or not self._is_method_retryable(method): - raise six.reraise(type(error), error, _stacktrace) + if read is False or method is None or not self._is_method_retryable(method): + raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 @@ -561,7 +485,9 @@ def increment( if redirect is not None: redirect -= 1 cause = "too many redirects" - redirect_location = response.get_redirect_location() + response_redirect_location = response.get_redirect_location() + if response_redirect_location: + redirect_location = response_redirect_location status = response.status else: @@ -589,31 +515,18 @@ def increment( ) if new_retry.is_exhausted(): - raise MaxRetryError(_pool, url, error or ResponseError(cause)) + reason = error or ResponseError(cause) + raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) return new_retry - def __repr__(self): + def __repr__(self) -> str: return ( - "{cls.__name__}(total={self.total}, connect={self.connect}, " - "read={self.read}, redirect={self.redirect}, status={self.status})" - ).format(cls=type(self), self=self) - - def __getattr__(self, item): - if item == "method_whitelist": - # TODO: Remove this deprecated alias in v2.0 - warnings.warn( - "Using 'method_whitelist' with Retry is deprecated and " - "will be removed in v2.0. Use 'allowed_methods' instead", - DeprecationWarning, - ) - return self.allowed_methods - try: - return getattr(super(Retry, self), item) - except AttributeError: - return getattr(Retry, item) + f"{type(self).__name__}(total={self.total}, connect={self.connect}, " + f"read={self.read}, redirect={self.redirect}, status={self.status})" + ) # For backwards compatibility (equivalent to pre-v1.9): diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index 8f86781..278128e 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -1,185 +1,149 @@ -from __future__ import absolute_import +from __future__ import annotations +import hashlib import hmac import os +import socket import sys +import typing import warnings -from binascii import hexlify, unhexlify -from hashlib import md5, sha1, sha256 - -from ..exceptions import ( - InsecurePlatformWarning, - ProxySchemeUnsupported, - SNIMissingWarning, - SSLError, -) -from ..packages import six -from .url import BRACELESS_IPV6_ADDRZ_RE, IPV4_RE +from binascii import unhexlify + +from ..exceptions import ProxySchemeUnsupported, SSLError +from .url import _BRACELESS_IPV6_ADDRZ_RE, _IPV4_RE SSLContext = None SSLTransport = None -HAS_SNI = False +HAS_NEVER_CHECK_COMMON_NAME = False IS_PYOPENSSL = False -IS_SECURETRANSPORT = False ALPN_PROTOCOLS = ["http/1.1"] -# Maps the length of a digest to a possible hash function producing this digest -HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256} - +_TYPE_VERSION_INFO = tuple[int, int, int, str, int] -def _const_compare_digest_backport(a, b): +# Maps the length of a digest to a possible hash function producing this digest +HASHFUNC_MAP = { + length: getattr(hashlib, algorithm, None) + for length, algorithm in ((32, "md5"), (40, "sha1"), (64, "sha256")) +} + + +def _is_bpo_43522_fixed( + implementation_name: str, + version_info: _TYPE_VERSION_INFO, + pypy_version_info: _TYPE_VERSION_INFO | None, +) -> bool: + """Return True for CPython 3.9.3+ or 3.10+ and PyPy 7.3.8+ where + setting SSLContext.hostname_checks_common_name to False works. + + Outside of CPython and PyPy we don't know which implementations work + or not so we conservatively use our hostname matching as we know that works + on all implementations. + + https://github.com/urllib3/urllib3/issues/2192#issuecomment-821832963 + https://foss.heptapod.net/pypy/pypy/-/issues/3539 """ - Compare two digests of equal length in constant time. + if implementation_name == "pypy": + # https://foss.heptapod.net/pypy/pypy/-/issues/3129 + return pypy_version_info >= (7, 3, 8) # type: ignore[operator] + elif implementation_name == "cpython": + major_minor = version_info[:2] + micro = version_info[2] + return (major_minor == (3, 9) and micro >= 3) or major_minor >= (3, 10) + else: # Defensive: + return False + + +def _is_has_never_check_common_name_reliable( + openssl_version: str, + openssl_version_number: int, + implementation_name: str, + version_info: _TYPE_VERSION_INFO, + pypy_version_info: _TYPE_VERSION_INFO | None, +) -> bool: + # As of May 2023, all released versions of LibreSSL fail to reject certificates with + # only common names, see https://github.com/urllib3/urllib3/pull/3024 + is_openssl = openssl_version.startswith("OpenSSL ") + # Before fixing OpenSSL issue #14579, the SSL_new() API was not copying hostflags + # like X509_CHECK_FLAG_NEVER_CHECK_SUBJECT, which tripped up CPython. + # https://github.com/openssl/openssl/issues/14579 + # This was released in OpenSSL 1.1.1l+ (>=0x101010cf) + is_openssl_issue_14579_fixed = openssl_version_number >= 0x101010CF + + return is_openssl and ( + is_openssl_issue_14579_fixed + or _is_bpo_43522_fixed(implementation_name, version_info, pypy_version_info) + ) - The digests must be of type str/bytes. - Returns True if the digests match, and False otherwise. - """ - result = abs(len(a) - len(b)) - for left, right in zip(bytearray(a), bytearray(b)): - result |= left ^ right - return result == 0 +if typing.TYPE_CHECKING: + from ssl import VerifyMode + from typing import TypedDict -_const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport) + from .ssltransport import SSLTransport as SSLTransportType -try: # Test for SSL features - import ssl - from ssl import CERT_REQUIRED, wrap_socket -except ImportError: - pass + class _TYPE_PEER_CERT_RET_DICT(TypedDict, total=False): + subjectAltName: tuple[tuple[str, str], ...] + subject: tuple[tuple[tuple[str, str], ...], ...] + serialNumber: str -try: - from ssl import HAS_SNI # Has SNI? -except ImportError: - pass - -try: - from .ssltransport import SSLTransport -except ImportError: - pass +# Mapping from 'ssl.PROTOCOL_TLSX' to 'TLSVersion.X' +_SSL_VERSION_TO_TLS_VERSION: dict[int, int] = {} -try: # Platform-specific: Python 3.6 - from ssl import PROTOCOL_TLS +try: # Do we have ssl at all? + import ssl + from ssl import ( # type: ignore[assignment] + CERT_REQUIRED, + HAS_NEVER_CHECK_COMMON_NAME, + OP_NO_COMPRESSION, + OP_NO_TICKET, + OPENSSL_VERSION, + OPENSSL_VERSION_NUMBER, + PROTOCOL_TLS, + PROTOCOL_TLS_CLIENT, + OP_NO_SSLv2, + OP_NO_SSLv3, + SSLContext, + TLSVersion, + ) PROTOCOL_SSLv23 = PROTOCOL_TLS -except ImportError: - try: - from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS - - PROTOCOL_SSLv23 = PROTOCOL_TLS - except ImportError: - PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 - -try: - from ssl import PROTOCOL_TLS_CLIENT -except ImportError: - PROTOCOL_TLS_CLIENT = PROTOCOL_TLS - - -try: - from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3 -except ImportError: - OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 - OP_NO_COMPRESSION = 0x20000 + # Setting SSLContext.hostname_checks_common_name = False didn't work before CPython + # 3.9.3, and 3.10 (but OK on PyPy) or OpenSSL 1.1.1l+ + if HAS_NEVER_CHECK_COMMON_NAME and not _is_has_never_check_common_name_reliable( + OPENSSL_VERSION, + OPENSSL_VERSION_NUMBER, + sys.implementation.name, + sys.version_info, + sys.pypy_version_info if sys.implementation.name == "pypy" else None, # type: ignore[attr-defined] + ): + HAS_NEVER_CHECK_COMMON_NAME = False + + # Need to be careful here in case old TLS versions get + # removed in future 'ssl' module implementations. + for attr in ("TLSv1", "TLSv1_1", "TLSv1_2"): + try: + _SSL_VERSION_TO_TLS_VERSION[getattr(ssl, f"PROTOCOL_{attr}")] = getattr( + TLSVersion, attr + ) + except AttributeError: # Defensive: + continue -try: # OP_NO_TICKET was added in Python 3.6 - from ssl import OP_NO_TICKET + from .ssltransport import SSLTransport # type: ignore[assignment] except ImportError: - OP_NO_TICKET = 0x4000 - - -# A secure default. -# Sources for more information on TLS ciphers: -# -# - https://wiki.mozilla.org/Security/Server_Side_TLS -# - https://www.ssllabs.com/projects/best-practices/index.html -# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ -# -# The general intent is: -# - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), -# - prefer ECDHE over DHE for better performance, -# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and -# security, -# - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common, -# - disable NULL authentication, MD5 MACs, DSS, and other -# insecure ciphers for security reasons. -# - NOTE: TLS 1.3 cipher suites are managed through a different interface -# not exposed by CPython (yet!) and are enabled by default if they're available. -DEFAULT_CIPHERS = ":".join( - [ - "ECDHE+AESGCM", - "ECDHE+CHACHA20", - "DHE+AESGCM", - "DHE+CHACHA20", - "ECDH+AESGCM", - "DH+AESGCM", - "ECDH+AES", - "DH+AES", - "RSA+AESGCM", - "RSA+AES", - "!aNULL", - "!eNULL", - "!MD5", - "!DSS", - ] -) - -try: - from ssl import SSLContext # Modern SSL? -except ImportError: - - class SSLContext(object): # Platform-specific: Python 2 - def __init__(self, protocol_version): - self.protocol = protocol_version - # Use default values from a real SSLContext - self.check_hostname = False - self.verify_mode = ssl.CERT_NONE - self.ca_certs = None - self.options = 0 - self.certfile = None - self.keyfile = None - self.ciphers = None + OP_NO_COMPRESSION = 0x20000 # type: ignore[assignment] + OP_NO_TICKET = 0x4000 # type: ignore[assignment] + OP_NO_SSLv2 = 0x1000000 # type: ignore[assignment] + OP_NO_SSLv3 = 0x2000000 # type: ignore[assignment] + PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 # type: ignore[assignment] + PROTOCOL_TLS_CLIENT = 16 # type: ignore[assignment] - def load_cert_chain(self, certfile, keyfile): - self.certfile = certfile - self.keyfile = keyfile - def load_verify_locations(self, cafile=None, capath=None, cadata=None): - self.ca_certs = cafile +_TYPE_PEER_CERT_RET = typing.Union["_TYPE_PEER_CERT_RET_DICT", bytes, None] - if capath is not None: - raise SSLError("CA directories not supported in older Pythons") - if cadata is not None: - raise SSLError("CA data not supported in older Pythons") - - def set_ciphers(self, cipher_suite): - self.ciphers = cipher_suite - - def wrap_socket(self, socket, server_hostname=None, server_side=False): - warnings.warn( - "A true SSLContext object is not available. This prevents " - "urllib3 from configuring SSL appropriately and may cause " - "certain SSL connections to fail. You can upgrade to a newer " - "version of Python to solve this. For more information, see " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#ssl-warnings", - InsecurePlatformWarning, - ) - kwargs = { - "keyfile": self.keyfile, - "certfile": self.certfile, - "ca_certs": self.ca_certs, - "cert_reqs": self.verify_mode, - "ssl_version": self.protocol, - "server_side": server_side, - } - return wrap_socket(socket, ciphers=self.ciphers, **kwargs) - - -def assert_fingerprint(cert, fingerprint): +def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None: """ Checks if given fingerprint matches the supplied certificate. @@ -189,26 +153,31 @@ def assert_fingerprint(cert, fingerprint): Fingerprint as string of hexdigits, can be interspersed by colons. """ + if cert is None: + raise SSLError("No certificate for the peer.") + fingerprint = fingerprint.replace(":", "").lower() digest_length = len(fingerprint) + if digest_length not in HASHFUNC_MAP: + raise SSLError(f"Fingerprint of invalid length: {fingerprint}") hashfunc = HASHFUNC_MAP.get(digest_length) - if not hashfunc: - raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint)) + if hashfunc is None: + raise SSLError( + f"Hash function implementation unavailable for fingerprint length: {digest_length}" + ) # We need encode() here for py32; works on py2 and p33. fingerprint_bytes = unhexlify(fingerprint.encode()) cert_digest = hashfunc(cert).digest() - if not _const_compare_digest(cert_digest, fingerprint_bytes): + if not hmac.compare_digest(cert_digest, fingerprint_bytes): raise SSLError( - 'Fingerprints did not match. Expected "{0}", got "{1}".'.format( - fingerprint, hexlify(cert_digest) - ) + f'Fingerprints did not match. Expected "{fingerprint}", got "{cert_digest.hex()}"' ) -def resolve_cert_reqs(candidate): +def resolve_cert_reqs(candidate: None | int | str) -> VerifyMode: """ Resolves the argument to a numeric constant, which can be passed to the wrap_socket function/method from the ssl module. @@ -226,12 +195,12 @@ def resolve_cert_reqs(candidate): res = getattr(ssl, candidate, None) if res is None: res = getattr(ssl, "CERT_" + candidate) - return res + return res # type: ignore[no-any-return] - return candidate + return candidate # type: ignore[return-value] -def resolve_ssl_version(candidate): +def resolve_ssl_version(candidate: None | int | str) -> int: """ like resolve_cert_reqs """ @@ -242,35 +211,33 @@ def resolve_ssl_version(candidate): res = getattr(ssl, candidate, None) if res is None: res = getattr(ssl, "PROTOCOL_" + candidate) - return res + return typing.cast(int, res) return candidate def create_urllib3_context( - ssl_version=None, cert_reqs=None, options=None, ciphers=None -): - """All arguments have the same meaning as ``ssl_wrap_socket``. - - By default, this function does a lot of the same work that - ``ssl.create_default_context`` does on Python 3.4+. It: - - - Disables SSLv2, SSLv3, and compression - - Sets a restricted set of server ciphers - - If you wish to enable SSLv3, you can do:: - - from urllib3.util import ssl_ - context = ssl_.create_urllib3_context() - context.options &= ~ssl_.OP_NO_SSLv3 - - You can do the same to enable compression (substituting ``COMPRESSION`` - for ``SSLv3`` in the last line above). + ssl_version: int | None = None, + cert_reqs: int | None = None, + options: int | None = None, + ciphers: str | None = None, + ssl_minimum_version: int | None = None, + ssl_maximum_version: int | None = None, +) -> ssl.SSLContext: + """Creates and configures an :class:`ssl.SSLContext` instance for use with urllib3. :param ssl_version: The desired protocol version to use. This will default to PROTOCOL_SSLv23 which will negotiate the highest protocol that both the server and your installation of OpenSSL support. + + This parameter is deprecated instead use 'ssl_minimum_version'. + :param ssl_minimum_version: + The minimum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value. + :param ssl_maximum_version: + The maximum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value. + Not recommended to set to anything other than 'ssl.TLSVersion.MAXIMUM_SUPPORTED' which is the + default value. :param cert_reqs: Whether to require the certificate verification. This defaults to ``ssl.CERT_REQUIRED``. @@ -278,18 +245,60 @@ def create_urllib3_context( Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``, ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``. :param ciphers: - Which cipher suites to allow the server to select. + Which cipher suites to allow the server to select. Defaults to either system configured + ciphers if OpenSSL 1.1.1+, otherwise uses a secure default set of ciphers. :returns: Constructed SSLContext object with specified options :rtype: SSLContext """ - # PROTOCOL_TLS is deprecated in Python 3.10 - if not ssl_version or ssl_version == PROTOCOL_TLS: - ssl_version = PROTOCOL_TLS_CLIENT + if SSLContext is None: + raise TypeError("Can't create an SSLContext object without an ssl module") + + # This means 'ssl_version' was specified as an exact value. + if ssl_version not in (None, PROTOCOL_TLS, PROTOCOL_TLS_CLIENT): + # Disallow setting 'ssl_version' and 'ssl_minimum|maximum_version' + # to avoid conflicts. + if ssl_minimum_version is not None or ssl_maximum_version is not None: + raise ValueError( + "Can't specify both 'ssl_version' and either " + "'ssl_minimum_version' or 'ssl_maximum_version'" + ) + + # 'ssl_version' is deprecated and will be removed in the future. + else: + # Use 'ssl_minimum_version' and 'ssl_maximum_version' instead. + ssl_minimum_version = _SSL_VERSION_TO_TLS_VERSION.get( + ssl_version, TLSVersion.MINIMUM_SUPPORTED + ) + ssl_maximum_version = _SSL_VERSION_TO_TLS_VERSION.get( + ssl_version, TLSVersion.MAXIMUM_SUPPORTED + ) + + # This warning message is pushing users to use 'ssl_minimum_version' + # instead of both min/max. Best practice is to only set the minimum version and + # keep the maximum version to be it's default value: 'TLSVersion.MAXIMUM_SUPPORTED' + warnings.warn( + "'ssl_version' option is deprecated and will be " + "removed in urllib3 v2.1.0. Instead use 'ssl_minimum_version'", + category=DeprecationWarning, + stacklevel=2, + ) + + # PROTOCOL_TLS is deprecated in Python 3.10 so we always use PROTOCOL_TLS_CLIENT + context = SSLContext(PROTOCOL_TLS_CLIENT) + + if ssl_minimum_version is not None: + context.minimum_version = ssl_minimum_version + else: # Python <3.10 defaults to 'MINIMUM_SUPPORTED' so explicitly set TLSv1.2 here + context.minimum_version = TLSVersion.TLSv1_2 - context = SSLContext(ssl_version) + if ssl_maximum_version is not None: + context.maximum_version = ssl_maximum_version - context.set_ciphers(ciphers or DEFAULT_CIPHERS) + # Unless we're given ciphers defer to either system ciphers in + # the case of OpenSSL 1.1.1+ or use our own secure default ciphers. + if ciphers: + context.set_ciphers(ciphers) # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs @@ -313,63 +322,91 @@ def create_urllib3_context( # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is # necessary for conditional client cert authentication with TLS 1.3. - # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older - # versions of Python. We only enable on Python 3.7.4+ or if certificate - # verification is enabled to work around Python issue #37428 - # See: https://bugs.python.org/issue37428 - if (cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)) and getattr( - context, "post_handshake_auth", None - ) is not None: + # The attribute is None for OpenSSL <= 1.1.0 or does not exist when using + # an SSLContext created by pyOpenSSL. + if getattr(context, "post_handshake_auth", None) is not None: context.post_handshake_auth = True - def disable_check_hostname(): - if ( - getattr(context, "check_hostname", None) is not None - ): # Platform-specific: Python 3.2 - # We do our own verification, including fingerprints and alternative - # hostnames. So disable it here - context.check_hostname = False - # The order of the below lines setting verify_mode and check_hostname # matter due to safe-guards SSLContext has to prevent an SSLContext with - # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more - # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used - # or not so we don't know the initial state of the freshly created SSLContext. - if cert_reqs == ssl.CERT_REQUIRED: + # check_hostname=True, verify_mode=NONE/OPTIONAL. + # We always set 'check_hostname=False' for pyOpenSSL so we rely on our own + # 'ssl.match_hostname()' implementation. + if cert_reqs == ssl.CERT_REQUIRED and not IS_PYOPENSSL: context.verify_mode = cert_reqs - disable_check_hostname() + context.check_hostname = True else: - disable_check_hostname() + context.check_hostname = False context.verify_mode = cert_reqs - # Enable logging of TLS session keys via defacto standard environment variable - # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. - if hasattr(context, "keylog_filename"): - sslkeylogfile = os.environ.get("SSLKEYLOGFILE") - if sslkeylogfile: - context.keylog_filename = sslkeylogfile + try: + context.hostname_checks_common_name = False + except AttributeError: # Defensive: for CPython < 3.9.3; for PyPy < 7.3.8 + pass + + sslkeylogfile = os.environ.get("SSLKEYLOGFILE") + if sslkeylogfile: + context.keylog_filename = sslkeylogfile return context +@typing.overload +def ssl_wrap_socket( + sock: socket.socket, + keyfile: str | None = ..., + certfile: str | None = ..., + cert_reqs: int | None = ..., + ca_certs: str | None = ..., + server_hostname: str | None = ..., + ssl_version: int | None = ..., + ciphers: str | None = ..., + ssl_context: ssl.SSLContext | None = ..., + ca_cert_dir: str | None = ..., + key_password: str | None = ..., + ca_cert_data: None | str | bytes = ..., + tls_in_tls: typing.Literal[False] = ..., +) -> ssl.SSLSocket: ... + + +@typing.overload +def ssl_wrap_socket( + sock: socket.socket, + keyfile: str | None = ..., + certfile: str | None = ..., + cert_reqs: int | None = ..., + ca_certs: str | None = ..., + server_hostname: str | None = ..., + ssl_version: int | None = ..., + ciphers: str | None = ..., + ssl_context: ssl.SSLContext | None = ..., + ca_cert_dir: str | None = ..., + key_password: str | None = ..., + ca_cert_data: None | str | bytes = ..., + tls_in_tls: bool = ..., +) -> ssl.SSLSocket | SSLTransportType: ... + + def ssl_wrap_socket( - sock, - keyfile=None, - certfile=None, - cert_reqs=None, - ca_certs=None, - server_hostname=None, - ssl_version=None, - ciphers=None, - ssl_context=None, - ca_cert_dir=None, - key_password=None, - ca_cert_data=None, - tls_in_tls=False, -): + sock: socket.socket, + keyfile: str | None = None, + certfile: str | None = None, + cert_reqs: int | None = None, + ca_certs: str | None = None, + server_hostname: str | None = None, + ssl_version: int | None = None, + ciphers: str | None = None, + ssl_context: ssl.SSLContext | None = None, + ca_cert_dir: str | None = None, + key_password: str | None = None, + ca_cert_data: None | str | bytes = None, + tls_in_tls: bool = False, +) -> ssl.SSLSocket | SSLTransportType: """ - All arguments except for server_hostname, ssl_context, and ca_cert_dir have - the same meaning as they do when using :func:`ssl.wrap_socket`. + All arguments except for server_hostname, ssl_context, tls_in_tls, ca_cert_data and + ca_cert_dir have the same meaning as they do when using + :func:`ssl.create_default_context`, :meth:`ssl.SSLContext.load_cert_chain`, + :meth:`ssl.SSLContext.set_ciphers` and :meth:`ssl.SSLContext.wrap_socket`. :param server_hostname: When SNI is supported, the expected hostname of the certificate @@ -392,19 +429,18 @@ def ssl_wrap_socket( """ context = ssl_context if context is None: - # Note: This branch of code and all the variables in it are no longer - # used by urllib3 itself. We should consider deprecating and removing - # this code. + # Note: This branch of code and all the variables in it are only used in tests. + # We should consider deprecating and removing this code. context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) if ca_certs or ca_cert_dir or ca_cert_data: try: context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data) - except (IOError, OSError) as e: - raise SSLError(e) + except OSError as e: + raise SSLError(e) from e elif ssl_context is None and hasattr(context, "load_default_certs"): - # try to load OS default certs; works well on Windows (require Python3.4+) + # try to load OS default certs; works well on Windows. context.load_default_certs() # Attempt to detect if we get the goofy behavior of the @@ -419,57 +455,28 @@ def ssl_wrap_socket( else: context.load_cert_chain(certfile, keyfile, key_password) - try: - if hasattr(context, "set_alpn_protocols"): - context.set_alpn_protocols(ALPN_PROTOCOLS) - except NotImplementedError: # Defensive: in CI, we always have set_alpn_protocols - pass - - # If we detect server_hostname is an IP address then the SNI - # extension should not be used according to RFC3546 Section 3.1 - use_sni_hostname = server_hostname and not is_ipaddress(server_hostname) - # SecureTransport uses server_hostname in certificate verification. - send_sni = (use_sni_hostname and HAS_SNI) or ( - IS_SECURETRANSPORT and server_hostname - ) - # Do not warn the user if server_hostname is an invalid SNI hostname. - if not HAS_SNI and use_sni_hostname: - warnings.warn( - "An HTTPS request has been made, but the SNI (Server Name " - "Indication) extension to TLS is not available on this platform. " - "This may cause the server to present an incorrect TLS " - "certificate, which can cause validation failures. You can upgrade to " - "a newer version of Python to solve this. For more information, see " - "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" - "#ssl-warnings", - SNIMissingWarning, - ) + context.set_alpn_protocols(ALPN_PROTOCOLS) - if send_sni: - ssl_sock = _ssl_wrap_socket_impl( - sock, context, tls_in_tls, server_hostname=server_hostname - ) - else: - ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls) + ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) return ssl_sock -def is_ipaddress(hostname): +def is_ipaddress(hostname: str | bytes) -> bool: """Detects whether the hostname given is an IPv4 or IPv6 address. Also detects IPv6 addresses with Zone IDs. :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if not six.PY2 and isinstance(hostname, bytes): + if isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. hostname = hostname.decode("ascii") - return bool(IPV4_RE.match(hostname) or BRACELESS_IPV6_ADDRZ_RE.match(hostname)) + return bool(_IPV4_RE.match(hostname) or _BRACELESS_IPV6_ADDRZ_RE.match(hostname)) -def _is_key_file_encrypted(key_file): +def _is_key_file_encrypted(key_file: str) -> bool: """Detects if a key file is encrypted or not.""" - with open(key_file, "r") as f: + with open(key_file) as f: for line in f: # Look for Proc-Type: 4,ENCRYPTED if "ENCRYPTED" in line: @@ -478,7 +485,12 @@ def _is_key_file_encrypted(key_file): return False -def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None): +def _ssl_wrap_socket_impl( + sock: socket.socket, + ssl_context: ssl.SSLContext, + tls_in_tls: bool, + server_hostname: str | None = None, +) -> ssl.SSLSocket | SSLTransportType: if tls_in_tls: if not SSLTransport: # Import error, ssl is not available. @@ -489,7 +501,4 @@ def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None): SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context) return SSLTransport(sock, ssl_context, server_hostname) - if server_hostname: - return ssl_context.wrap_socket(sock, server_hostname=server_hostname) - else: - return ssl_context.wrap_socket(sock) + return ssl_context.wrap_socket(sock, server_hostname=server_hostname) diff --git a/src/urllib3/util/ssl_match_hostname.py b/src/urllib3/util/ssl_match_hostname.py index 1dd950c..453cfd4 100644 --- a/src/urllib3/util/ssl_match_hostname.py +++ b/src/urllib3/util/ssl_match_hostname.py @@ -1,19 +1,18 @@ -"""The match_hostname() function from Python 3.3.3, essential when using SSL.""" +"""The match_hostname() function from Python 3.5, essential when using SSL.""" # Note: This file is under the PSF license as the code comes from the python # stdlib. http://docs.python.org/3/license.html +# It is modified to remove commonName support. +from __future__ import annotations + +import ipaddress import re -import sys +import typing +from ipaddress import IPv4Address, IPv6Address -# ipaddress has been backported to 2.6+ in pypi. If it is installed on the -# system, use it to handle IPAddress ServerAltnames (this was added in -# python-3.5) otherwise only do DNS matching. This allows -# util.ssl_match_hostname to continue to be used in Python 2.7. -try: - import ipaddress -except ImportError: - ipaddress = None +if typing.TYPE_CHECKING: + from .ssl_ import _TYPE_PEER_CERT_RET_DICT __version__ = "3.5.0.1" @@ -22,7 +21,9 @@ class CertificateError(ValueError): pass -def _dnsname_match(dn, hostname, max_wildcards=1): +def _dnsname_match( + dn: typing.Any, hostname: str, max_wildcards: int = 1 +) -> typing.Match[str] | None | bool: """Matching according to RFC 6125, section 6.4.3 http://tools.ietf.org/html/rfc6125#section-6.4.3 @@ -49,7 +50,7 @@ def _dnsname_match(dn, hostname, max_wildcards=1): # speed up common case w/o wildcards if not wildcards: - return dn.lower() == hostname.lower() + return bool(dn.lower() == hostname.lower()) # RFC 6125, section 6.4.3, subitem 1. # The client SHOULD NOT attempt to match a presented identifier in which @@ -76,26 +77,26 @@ def _dnsname_match(dn, hostname, max_wildcards=1): return pat.match(hostname) -def _to_unicode(obj): - if isinstance(obj, str) and sys.version_info < (3,): - # ignored flake8 # F821 to support python 2.7 function - obj = unicode(obj, encoding="ascii", errors="strict") # noqa: F821 - return obj - - -def _ipaddress_match(ipname, host_ip): +def _ipaddress_match(ipname: str, host_ip: IPv4Address | IPv6Address) -> bool: """Exact matching of IP addresses. - RFC 6125 explicitly doesn't define an algorithm for this - (section 1.7.2 - "Out of Scope"). + RFC 9110 section 4.3.5: "A reference identity of IP-ID contains the decoded + bytes of the IP address. An IP version 4 address is 4 octets, and an IP + version 6 address is 16 octets. [...] A reference identity of type IP-ID + matches if the address is identical to an iPAddress value of the + subjectAltName extension of the certificate." """ # OpenSSL may add a trailing newline to a subjectAltName's IP address # Divergence from upstream: ipaddress can't handle byte str - ip = ipaddress.ip_address(_to_unicode(ipname).rstrip()) - return ip == host_ip + ip = ipaddress.ip_address(ipname.rstrip()) + return bool(ip.packed == host_ip.packed) -def match_hostname(cert, hostname): +def match_hostname( + cert: _TYPE_PEER_CERT_RET_DICT | None, + hostname: str, + hostname_checks_common_name: bool = False, +) -> None: """Verify that *cert* (in decoded format as returned by SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 rules are followed, but IP addresses are not accepted for *hostname*. @@ -111,21 +112,22 @@ def match_hostname(cert, hostname): ) try: # Divergence from upstream: ipaddress can't handle byte str - host_ip = ipaddress.ip_address(_to_unicode(hostname)) - except (UnicodeError, ValueError): - # ValueError: Not an IP address (common case) - # UnicodeError: Divergence from upstream: Have to deal with ipaddress not taking - # byte strings. addresses should be all ascii, so we consider it not - # an ipaddress in this case + # + # The ipaddress module shipped with Python < 3.9 does not support + # scoped IPv6 addresses so we unconditionally strip the Zone IDs for + # now. Once we drop support for Python 3.9 we can remove this branch. + if "%" in hostname: + host_ip = ipaddress.ip_address(hostname[: hostname.rfind("%")]) + else: + host_ip = ipaddress.ip_address(hostname) + + except ValueError: + # Not an IP address (common case) host_ip = None - except AttributeError: - # Divergence from upstream: Make ipaddress library optional - if ipaddress is None: - host_ip = None - else: # Defensive - raise dnsnames = [] - san = cert.get("subjectAltName", ()) + san: tuple[tuple[str, str], ...] = cert.get("subjectAltName", ()) + key: str + value: str for key, value in san: if key == "DNS": if host_ip is None and _dnsname_match(value, hostname): @@ -135,25 +137,23 @@ def match_hostname(cert, hostname): if host_ip is not None and _ipaddress_match(value, host_ip): return dnsnames.append(value) - if not dnsnames: - # The subject is only checked when there is no dNSName entry - # in subjectAltName + + # We only check 'commonName' if it's enabled and we're not verifying + # an IP address. IP addresses aren't valid within 'commonName'. + if hostname_checks_common_name and host_ip is None and not dnsnames: for sub in cert.get("subject", ()): for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. if key == "commonName": if _dnsname_match(value, hostname): return dnsnames.append(value) + if len(dnsnames) > 1: raise CertificateError( "hostname %r " "doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames))) ) elif len(dnsnames) == 1: - raise CertificateError("hostname %r doesn't match %r" % (hostname, dnsnames[0])) + raise CertificateError(f"hostname {hostname!r} doesn't match {dnsnames[0]!r}") else: - raise CertificateError( - "no appropriate commonName or subjectAltName fields were found" - ) + raise CertificateError("no appropriate subjectAltName fields were found") diff --git a/src/urllib3/util/ssltransport.py b/src/urllib3/util/ssltransport.py index 4a7105d..6d59bc3 100644 --- a/src/urllib3/util/ssltransport.py +++ b/src/urllib3/util/ssltransport.py @@ -1,9 +1,20 @@ +from __future__ import annotations + import io import socket import ssl +import typing from ..exceptions import ProxySchemeUnsupported -from ..packages import six + +if typing.TYPE_CHECKING: + from typing_extensions import Self + + from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT + + +_WriteBuffer = typing.Union[bytearray, memoryview] +_ReturnValue = typing.TypeVar("_ReturnValue") SSL_BLOCKSIZE = 16384 @@ -20,7 +31,7 @@ class SSLTransport: """ @staticmethod - def _validate_ssl_context_for_tls_in_tls(ssl_context): + def _validate_ssl_context_for_tls_in_tls(ssl_context: ssl.SSLContext) -> None: """ Raises a ProxySchemeUnsupported if the provided ssl_context can't be used for TLS in TLS. @@ -30,20 +41,18 @@ def _validate_ssl_context_for_tls_in_tls(ssl_context): """ if not hasattr(ssl_context, "wrap_bio"): - if six.PY2: - raise ProxySchemeUnsupported( - "TLS in TLS requires SSLContext.wrap_bio() which isn't " - "supported on Python 2" - ) - else: - raise ProxySchemeUnsupported( - "TLS in TLS requires SSLContext.wrap_bio() which isn't " - "available on non-native SSLContext" - ) + raise ProxySchemeUnsupported( + "TLS in TLS requires SSLContext.wrap_bio() which isn't " + "available on non-native SSLContext" + ) def __init__( - self, socket, ssl_context, server_hostname=None, suppress_ragged_eofs=True - ): + self, + socket: socket.socket, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + suppress_ragged_eofs: bool = True, + ) -> None: """ Create an SSLTransport around socket using the provided ssl_context. """ @@ -60,33 +69,36 @@ def __init__( # Perform initial handshake. self._ssl_io_loop(self.sslobj.do_handshake) - def __enter__(self): + def __enter__(self) -> Self: return self - def __exit__(self, *_): + def __exit__(self, *_: typing.Any) -> None: self.close() - def fileno(self): + def fileno(self) -> int: return self.socket.fileno() - def read(self, len=1024, buffer=None): + def read(self, len: int = 1024, buffer: typing.Any | None = None) -> int | bytes: return self._wrap_ssl_read(len, buffer) - def recv(self, len=1024, flags=0): + def recv(self, buflen: int = 1024, flags: int = 0) -> int | bytes: if flags != 0: raise ValueError("non-zero flags not allowed in calls to recv") - return self._wrap_ssl_read(len) - - def recv_into(self, buffer, nbytes=None, flags=0): + return self._wrap_ssl_read(buflen) + + def recv_into( + self, + buffer: _WriteBuffer, + nbytes: int | None = None, + flags: int = 0, + ) -> None | int | bytes: if flags != 0: raise ValueError("non-zero flags not allowed in calls to recv_into") - if buffer and (nbytes is None): + if nbytes is None: nbytes = len(buffer) - elif nbytes is None: - nbytes = 1024 return self.read(nbytes, buffer) - def sendall(self, data, flags=0): + def sendall(self, data: bytes, flags: int = 0) -> None: if flags != 0: raise ValueError("non-zero flags not allowed in calls to sendall") count = 0 @@ -96,15 +108,20 @@ def sendall(self, data, flags=0): v = self.send(byte_view[count:]) count += v - def send(self, data, flags=0): + def send(self, data: bytes, flags: int = 0) -> int: if flags != 0: raise ValueError("non-zero flags not allowed in calls to send") - response = self._ssl_io_loop(self.sslobj.write, data) - return response + return self._ssl_io_loop(self.sslobj.write, data) def makefile( - self, mode="r", buffering=None, encoding=None, errors=None, newline=None - ): + self, + mode: str, + buffering: int | None = None, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> typing.BinaryIO | typing.TextIO | socket.SocketIO: """ Python's httpclient uses makefile and buffered io when reading HTTP messages and we need to support it. @@ -113,7 +130,7 @@ def makefile( changes to point to the socket directly. """ if not set(mode) <= {"r", "w", "b"}: - raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) + raise ValueError(f"invalid mode {mode!r} (only r, w, b allowed)") writing = "w" in mode reading = "r" in mode or not writing @@ -124,8 +141,8 @@ def makefile( rawmode += "r" if writing: rawmode += "w" - raw = socket.SocketIO(self, rawmode) - self.socket._io_refs += 1 + raw = socket.SocketIO(self, rawmode) # type: ignore[arg-type] + self.socket._io_refs += 1 # type: ignore[attr-defined] if buffering is None: buffering = -1 if buffering < 0: @@ -134,8 +151,9 @@ def makefile( if not binary: raise ValueError("unbuffered streams must be binary") return raw + buffer: typing.BinaryIO if reading and writing: - buffer = io.BufferedRWPair(raw, raw, buffering) + buffer = io.BufferedRWPair(raw, raw, buffering) # type: ignore[assignment] elif reading: buffer = io.BufferedReader(raw, buffering) else: @@ -144,46 +162,51 @@ def makefile( if binary: return buffer text = io.TextIOWrapper(buffer, encoding, errors, newline) - text.mode = mode + text.mode = mode # type: ignore[misc] return text - def unwrap(self): + def unwrap(self) -> None: self._ssl_io_loop(self.sslobj.unwrap) - def close(self): + def close(self) -> None: self.socket.close() - def getpeercert(self, binary_form=False): - return self.sslobj.getpeercert(binary_form) + @typing.overload + def getpeercert( + self, binary_form: typing.Literal[False] = ... + ) -> _TYPE_PEER_CERT_RET_DICT | None: ... + + @typing.overload + def getpeercert(self, binary_form: typing.Literal[True]) -> bytes | None: ... + + def getpeercert(self, binary_form: bool = False) -> _TYPE_PEER_CERT_RET: + return self.sslobj.getpeercert(binary_form) # type: ignore[return-value] - def version(self): + def version(self) -> str | None: return self.sslobj.version() - def cipher(self): + def cipher(self) -> tuple[str, str, int] | None: return self.sslobj.cipher() - def selected_alpn_protocol(self): + def selected_alpn_protocol(self) -> str | None: return self.sslobj.selected_alpn_protocol() - def selected_npn_protocol(self): - return self.sslobj.selected_npn_protocol() - - def shared_ciphers(self): + def shared_ciphers(self) -> list[tuple[str, str, int]] | None: return self.sslobj.shared_ciphers() - def compression(self): + def compression(self) -> str | None: return self.sslobj.compression() - def settimeout(self, value): + def settimeout(self, value: float | None) -> None: self.socket.settimeout(value) - def gettimeout(self): + def gettimeout(self) -> float | None: return self.socket.gettimeout() - def _decref_socketios(self): - self.socket._decref_socketios() + def _decref_socketios(self) -> None: + self.socket._decref_socketios() # type: ignore[attr-defined] - def _wrap_ssl_read(self, len, buffer=None): + def _wrap_ssl_read(self, len: int, buffer: bytearray | None = None) -> int | bytes: try: return self._ssl_io_loop(self.sslobj.read, len, buffer) except ssl.SSLError as e: @@ -192,7 +215,29 @@ def _wrap_ssl_read(self, len, buffer=None): else: raise - def _ssl_io_loop(self, func, *args): + # func is sslobj.do_handshake or sslobj.unwrap + @typing.overload + def _ssl_io_loop(self, func: typing.Callable[[], None]) -> None: ... + + # func is sslobj.write, arg1 is data + @typing.overload + def _ssl_io_loop(self, func: typing.Callable[[bytes], int], arg1: bytes) -> int: ... + + # func is sslobj.read, arg1 is len, arg2 is buffer + @typing.overload + def _ssl_io_loop( + self, + func: typing.Callable[[int, bytearray | None], bytes], + arg1: int, + arg2: bytearray | None, + ) -> bytes: ... + + def _ssl_io_loop( + self, + func: typing.Callable[..., _ReturnValue], + arg1: None | bytes | int = None, + arg2: bytearray | None = None, + ) -> _ReturnValue: """Performs an I/O loop between incoming/outgoing and the socket.""" should_loop = True ret = None @@ -200,7 +245,12 @@ def _ssl_io_loop(self, func, *args): while should_loop: errno = None try: - ret = func(*args) + if arg1 is None and arg2 is None: + ret = func() + elif arg2 is None: + ret = func(arg1) + else: + ret = func(arg1, arg2) except ssl.SSLError as e: if e.errno not in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE): # WANT_READ, and WANT_WRITE are expected, others are not. @@ -218,4 +268,4 @@ def _ssl_io_loop(self, func, *args): self.incoming.write(buf) else: self.incoming.write_eof() - return ret + return typing.cast(_ReturnValue, ret) diff --git a/src/urllib3/util/timeout.py b/src/urllib3/util/timeout.py index 78e18a6..4bb1be1 100644 --- a/src/urllib3/util/timeout.py +++ b/src/urllib3/util/timeout.py @@ -1,44 +1,56 @@ -from __future__ import absolute_import +from __future__ import annotations import time - -# The default socket timeout, used by httplib to indicate that no timeout was; specified by the user -from socket import _GLOBAL_DEFAULT_TIMEOUT, getdefaulttimeout +import typing +from enum import Enum +from socket import getdefaulttimeout from ..exceptions import TimeoutStateError -# A sentinel value to indicate that no timeout was specified by the user in -# urllib3 -_Default = object() +if typing.TYPE_CHECKING: + from typing import Final + + +class _TYPE_DEFAULT(Enum): + # This value should never be passed to socket.settimeout() so for safety we use a -1. + # socket.settimout() raises a ValueError for negative values. + token = -1 + +_DEFAULT_TIMEOUT: Final[_TYPE_DEFAULT] = _TYPE_DEFAULT.token -# Use time.monotonic if available. -current_time = getattr(time, "monotonic", time.time) +_TYPE_TIMEOUT = typing.Optional[typing.Union[float, _TYPE_DEFAULT]] -class Timeout(object): +class Timeout: """Timeout configuration. Timeouts can be defined as a default for a pool: .. code-block:: python - timeout = Timeout(connect=2.0, read=7.0) - http = PoolManager(timeout=timeout) - response = http.request('GET', 'http://example.com/') + import urllib3 + + timeout = urllib3.util.Timeout(connect=2.0, read=7.0) + + http = urllib3.PoolManager(timeout=timeout) + + resp = http.request("GET", "https://example.com/") + + print(resp.status) Or per-request (which overrides the default for the pool): .. code-block:: python - response = http.request('GET', 'http://example.com/', timeout=Timeout(10)) + response = http.request("GET", "https://example.com/", timeout=Timeout(10)) Timeouts can be disabled by setting all the parameters to ``None``: .. code-block:: python no_timeout = Timeout(connect=None, read=None) - response = http.request('GET', 'http://example.com/, timeout=no_timeout) + response = http.request("GET", "https://example.com/", timeout=no_timeout) :param total: @@ -89,38 +101,34 @@ class Timeout(object): the case; if a server streams one byte every fifteen seconds, a timeout of 20 seconds will not trigger, even though the request will take several minutes to complete. - - If your goal is to cut off any request after a set amount of wall clock - time, consider having a second "watcher" thread to cut off a slow - request. """ #: A sentinel object representing the default timeout value - DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT - - def __init__(self, total=None, connect=_Default, read=_Default): + DEFAULT_TIMEOUT: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT + + def __init__( + self, + total: _TYPE_TIMEOUT = None, + connect: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + read: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, + ) -> None: self._connect = self._validate_timeout(connect, "connect") self._read = self._validate_timeout(read, "read") self.total = self._validate_timeout(total, "total") - self._start_connect = None + self._start_connect: float | None = None - def __repr__(self): - return "%s(connect=%r, read=%r, total=%r)" % ( - type(self).__name__, - self._connect, - self._read, - self.total, - ) + def __repr__(self) -> str: + return f"{type(self).__name__}(connect={self._connect!r}, read={self._read!r}, total={self.total!r})" # __str__ provided for backwards compatibility __str__ = __repr__ - @classmethod - def resolve_default_timeout(cls, timeout): - return getdefaulttimeout() if timeout is cls.DEFAULT_TIMEOUT else timeout + @staticmethod + def resolve_default_timeout(timeout: _TYPE_TIMEOUT) -> float | None: + return getdefaulttimeout() if timeout is _DEFAULT_TIMEOUT else timeout @classmethod - def _validate_timeout(cls, value, name): + def _validate_timeout(cls, value: _TYPE_TIMEOUT, name: str) -> _TYPE_TIMEOUT: """Check that a timeout attribute is valid. :param value: The timeout value to validate @@ -130,10 +138,7 @@ def _validate_timeout(cls, value, name): :raises ValueError: If it is a numeric value less than or equal to zero, or the type is not an integer, float, or None. """ - if value is _Default: - return cls.DEFAULT_TIMEOUT - - if value is None or value is cls.DEFAULT_TIMEOUT: + if value is None or value is _DEFAULT_TIMEOUT: return value if isinstance(value, bool): @@ -147,7 +152,7 @@ def _validate_timeout(cls, value, name): raise ValueError( "Timeout value %s was %s, but it must be an " "int, float or None." % (name, value) - ) + ) from None try: if value <= 0: @@ -157,16 +162,15 @@ def _validate_timeout(cls, value, name): "than or equal to 0." % (name, value) ) except TypeError: - # Python 3 raise ValueError( "Timeout value %s was %s, but it must be an " "int, float or None." % (name, value) - ) + ) from None return value @classmethod - def from_float(cls, timeout): + def from_float(cls, timeout: _TYPE_TIMEOUT) -> Timeout: """Create a new Timeout from a legacy timeout value. The timeout value used by httplib.py sets the same timeout on the @@ -175,13 +179,13 @@ def from_float(cls, timeout): passed to this function. :param timeout: The legacy timeout value. - :type timeout: integer, float, sentinel default object, or None + :type timeout: integer, float, :attr:`urllib3.util.Timeout.DEFAULT_TIMEOUT`, or None :return: Timeout object :rtype: :class:`Timeout` """ return Timeout(read=timeout, connect=timeout) - def clone(self): + def clone(self) -> Timeout: """Create a copy of the timeout object Timeout properties are stored per-pool but each request needs a fresh @@ -195,7 +199,7 @@ def clone(self): # detect the user default. return Timeout(connect=self._connect, read=self._read, total=self.total) - def start_connect(self): + def start_connect(self) -> float: """Start the timeout clock, used during a connect() attempt :raises urllib3.exceptions.TimeoutStateError: if you attempt @@ -203,10 +207,10 @@ def start_connect(self): """ if self._start_connect is not None: raise TimeoutStateError("Timeout timer has already been started.") - self._start_connect = current_time() + self._start_connect = time.monotonic() return self._start_connect - def get_connect_duration(self): + def get_connect_duration(self) -> float: """Gets the time elapsed since the call to :meth:`start_connect`. :return: Elapsed time in seconds. @@ -218,10 +222,10 @@ def get_connect_duration(self): raise TimeoutStateError( "Can't get connect duration for timer that has not started." ) - return current_time() - self._start_connect + return time.monotonic() - self._start_connect @property - def connect_timeout(self): + def connect_timeout(self) -> _TYPE_TIMEOUT: """Get the value to use when setting a connection timeout. This will be a positive float or integer, the value None @@ -233,13 +237,13 @@ def connect_timeout(self): if self.total is None: return self._connect - if self._connect is None or self._connect is self.DEFAULT_TIMEOUT: + if self._connect is None or self._connect is _DEFAULT_TIMEOUT: return self.total - return min(self._connect, self.total) + return min(self._connect, self.total) # type: ignore[type-var] @property - def read_timeout(self): + def read_timeout(self) -> float | None: """Get the value for the read timeout. This assumes some time has elapsed in the connection timeout and @@ -251,21 +255,21 @@ def read_timeout(self): raised. :return: Value to use for the read timeout. - :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None + :rtype: int, float or None :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` has not yet been called on this object. """ if ( self.total is not None - and self.total is not self.DEFAULT_TIMEOUT + and self.total is not _DEFAULT_TIMEOUT and self._read is not None - and self._read is not self.DEFAULT_TIMEOUT + and self._read is not _DEFAULT_TIMEOUT ): # In case the connect timeout has not yet been established. if self._start_connect is None: return self._read return max(0, min(self.total - self.get_connect_duration(), self._read)) - elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: + elif self.total is not None and self.total is not _DEFAULT_TIMEOUT: return max(0, self.total - self.get_connect_duration()) else: - return self._read + return self.resolve_default_timeout(self._read) diff --git a/src/urllib3/util/url.py b/src/urllib3/util/url.py index e5682d3..db057f1 100644 --- a/src/urllib3/util/url.py +++ b/src/urllib3/util/url.py @@ -1,22 +1,20 @@ -from __future__ import absolute_import +from __future__ import annotations import re -from collections import namedtuple +import typing from ..exceptions import LocationParseError -from ..packages import six - -url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"] +from .util import to_str # We only want to normalize urls with an HTTP(S) scheme. # urllib3 infers URLs without a scheme (None) to be http. -NORMALIZABLE_SCHEMES = ("http", "https", None) +_NORMALIZABLE_SCHEMES = ("http", "https", None) # Almost all of these patterns were derived from the # 'rfc3986' module: https://github.com/python-hyper/rfc3986 -PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}") -SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)") -URI_RE = re.compile( +_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}") +_SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)") +_URI_RE = re.compile( r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?" r"(?://([^\\/?#]*))?" r"([^?#]*)" @@ -25,10 +23,10 @@ re.UNICODE | re.DOTALL, ) -IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" -HEX_PAT = "[0-9A-Fa-f]{1,4}" -LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=HEX_PAT, ipv4=IPV4_PAT) -_subs = {"hex": HEX_PAT, "ls32": LS32_PAT} +_IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" +_HEX_PAT = "[0-9A-Fa-f]{1,4}" +_LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=_HEX_PAT, ipv4=_IPV4_PAT) +_subs = {"hex": _HEX_PAT, "ls32": _LS32_PAT} _variations = [ # 6( h16 ":" ) ls32 "(?:%(hex)s:){6}%(ls32)s", @@ -50,69 +48,78 @@ "(?:(?:%(hex)s:){0,6}%(hex)s)?::", ] -UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~" -IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")" -ZONE_ID_PAT = "(?:%25|%)(?:[" + UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+" -IPV6_ADDRZ_PAT = r"\[" + IPV6_PAT + r"(?:" + ZONE_ID_PAT + r")?\]" -REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*" -TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$") +_UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~" +_IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")" +_ZONE_ID_PAT = "(?:%25|%)(?:[" + _UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+" +_IPV6_ADDRZ_PAT = r"\[" + _IPV6_PAT + r"(?:" + _ZONE_ID_PAT + r")?\]" +_REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*" +_TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$") -IPV4_RE = re.compile("^" + IPV4_PAT + "$") -IPV6_RE = re.compile("^" + IPV6_PAT + "$") -IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT + "$") -BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT[2:-2] + "$") -ZONE_ID_RE = re.compile("(" + ZONE_ID_PAT + r")\]$") +_IPV4_RE = re.compile("^" + _IPV4_PAT + "$") +_IPV6_RE = re.compile("^" + _IPV6_PAT + "$") +_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT + "$") +_BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT[2:-2] + "$") +_ZONE_ID_RE = re.compile("(" + _ZONE_ID_PAT + r")\]$") _HOST_PORT_PAT = ("^(%s|%s|%s)(?::0*?(|0|[1-9][0-9]{0,4}))?$") % ( - REG_NAME_PAT, - IPV4_PAT, - IPV6_ADDRZ_PAT, + _REG_NAME_PAT, + _IPV4_PAT, + _IPV6_ADDRZ_PAT, ) _HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL) -UNRESERVED_CHARS = set( +_UNRESERVED_CHARS = set( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~" ) -SUB_DELIM_CHARS = set("!$&'()*+,;=") -USERINFO_CHARS = UNRESERVED_CHARS | SUB_DELIM_CHARS | {":"} -PATH_CHARS = USERINFO_CHARS | {"@", "/"} -QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {"?"} - - -class Url(namedtuple("Url", url_attrs)): +_SUB_DELIM_CHARS = set("!$&'()*+,;=") +_USERINFO_CHARS = _UNRESERVED_CHARS | _SUB_DELIM_CHARS | {":"} +_PATH_CHARS = _USERINFO_CHARS | {"@", "/"} +_QUERY_CHARS = _FRAGMENT_CHARS = _PATH_CHARS | {"?"} + + +class Url( + typing.NamedTuple( + "Url", + [ + ("scheme", typing.Optional[str]), + ("auth", typing.Optional[str]), + ("host", typing.Optional[str]), + ("port", typing.Optional[int]), + ("path", typing.Optional[str]), + ("query", typing.Optional[str]), + ("fragment", typing.Optional[str]), + ], + ) +): """ Data structure for representing an HTTP URL. Used as a return value for :func:`parse_url`. Both the scheme and host are normalized as they are both case-insensitive according to RFC 3986. """ - __slots__ = () - - def __new__( + def __new__( # type: ignore[no-untyped-def] cls, - scheme=None, - auth=None, - host=None, - port=None, - path=None, - query=None, - fragment=None, + scheme: str | None = None, + auth: str | None = None, + host: str | None = None, + port: int | None = None, + path: str | None = None, + query: str | None = None, + fragment: str | None = None, ): if path and not path.startswith("/"): path = "/" + path if scheme is not None: scheme = scheme.lower() - return super(Url, cls).__new__( - cls, scheme, auth, host, port, path, query, fragment - ) + return super().__new__(cls, scheme, auth, host, port, path, query, fragment) @property - def hostname(self): + def hostname(self) -> str | None: """For backwards-compatibility with urlparse. We're nice like that.""" return self.host @property - def request_uri(self): + def request_uri(self) -> str: """Absolute path including the query string.""" uri = self.path or "/" @@ -122,14 +129,37 @@ def request_uri(self): return uri @property - def netloc(self): - """Network location including host and port""" + def authority(self) -> str | None: + """ + Authority component as defined in RFC 3986 3.2. + This includes userinfo (auth), host and port. + + i.e. + userinfo@host:port + """ + userinfo = self.auth + netloc = self.netloc + if netloc is None or userinfo is None: + return netloc + else: + return f"{userinfo}@{netloc}" + + @property + def netloc(self) -> str | None: + """ + Network location including host and port. + + If you need the equivalent of urllib.parse's ``netloc``, + use the ``authority`` property instead. + """ + if self.host is None: + return None if self.port: - return "%s:%d" % (self.host, self.port) + return f"{self.host}:{self.port}" return self.host @property - def url(self): + def url(self) -> str: """ Convert self into a url @@ -138,88 +168,77 @@ def url(self): :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls with a blank port will have : removed). - Example: :: + Example: + + .. code-block:: python + + import urllib3 - >>> U = parse_url('http://google.com/mail/') - >>> U.url - 'http://google.com/mail/' - >>> Url('http', 'username:password', 'host.com', 80, - ... '/path', 'query', 'fragment').url - 'http://username:password@host.com:80/path?query#fragment' + U = urllib3.util.parse_url("https://google.com/mail/") + + print(U.url) + # "https://google.com/mail/" + + print( urllib3.util.Url("https", "username:password", + "host.com", 80, "/path", "query", "fragment" + ).url + ) + # "https://username:password@host.com:80/path?query#fragment" """ scheme, auth, host, port, path, query, fragment = self - url = u"" + url = "" # We use "is not None" we want things to happen with empty strings (or 0 port) if scheme is not None: - url += scheme + u"://" + url += scheme + "://" if auth is not None: - url += auth + u"@" + url += auth + "@" if host is not None: url += host if port is not None: - url += u":" + str(port) + url += ":" + str(port) if path is not None: url += path if query is not None: - url += u"?" + query + url += "?" + query if fragment is not None: - url += u"#" + fragment + url += "#" + fragment return url - def __str__(self): + def __str__(self) -> str: return self.url -def split_first(s, delims): - """ - .. deprecated:: 1.25 - - Given a string and an iterable of delimiters, split on the first found - delimiter. Return two split parts and the matched delimiter. - - If not found, then the first part is the full input string. - - Example:: - - >>> split_first('foo/bar?baz', '?/=') - ('foo', 'bar?baz', '/') - >>> split_first('foo/bar?baz', '123') - ('foo/bar?baz', '', None) - - Scales linearly with number of delims. Not ideal for large number of delims. - """ - min_idx = None - min_delim = None - for d in delims: - idx = s.find(d) - if idx < 0: - continue +@typing.overload +def _encode_invalid_chars( + component: str, allowed_chars: typing.Container[str] +) -> str: # Abstract + ... - if min_idx is None or idx < min_idx: - min_idx = idx - min_delim = d - if min_idx is None or min_idx < 0: - return s, "", None +@typing.overload +def _encode_invalid_chars( + component: None, allowed_chars: typing.Container[str] +) -> None: # Abstract + ... - return s[:min_idx], s[min_idx + 1 :], min_delim - -def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): +def _encode_invalid_chars( + component: str | None, allowed_chars: typing.Container[str] +) -> str | None: """Percent-encodes a URI component without reapplying onto an already percent-encoded component. """ if component is None: return component - component = six.ensure_text(component) + component = to_str(component) # Normalize existing percent-encoded bytes. # Try to see if the component we're encoding is already percent-encoded # so we can skip all '%' characters but still encode all others. - component, percent_encodings = PERCENT_RE.subn( + component, percent_encodings = _PERCENT_RE.subn( lambda match: match.group(0).upper(), component ) @@ -228,7 +247,7 @@ def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): encoded_component = bytearray() for i in range(0, len(uri_bytes)): - # Will return a single character bytestring on both Python 2 & 3 + # Will return a single character bytestring byte = uri_bytes[i : i + 1] byte_ord = ord(byte) if (is_percent_encoded and byte == b"%") or ( @@ -238,10 +257,10 @@ def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): continue encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper())) - return encoded_component.decode(encoding) + return encoded_component.decode() -def _remove_path_dot_segments(path): +def _remove_path_dot_segments(path: str) -> str: # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code segments = path.split("/") # Turn the path into a list of segments output = [] # Initialize the variable to use to store output @@ -251,7 +270,7 @@ def _remove_path_dot_segments(path): if segment == ".": continue # Anything other than '..', should be appended to the output - elif segment != "..": + if segment != "..": output.append(segment) # In this case segment == '..', if we can, we should pop the last # element @@ -271,18 +290,23 @@ def _remove_path_dot_segments(path): return "/".join(output) -def _normalize_host(host, scheme): - if host: - if isinstance(host, six.binary_type): - host = six.ensure_str(host) +@typing.overload +def _normalize_host(host: None, scheme: str | None) -> None: ... + + +@typing.overload +def _normalize_host(host: str, scheme: str | None) -> str: ... + - if scheme in NORMALIZABLE_SCHEMES: - is_ipv6 = IPV6_ADDRZ_RE.match(host) +def _normalize_host(host: str | None, scheme: str | None) -> str | None: + if host: + if scheme in _NORMALIZABLE_SCHEMES: + is_ipv6 = _IPV6_ADDRZ_RE.match(host) if is_ipv6: # IPv6 hosts of the form 'a::b%zone' are encoded in a URL as # such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID # separator as necessary to return a valid RFC 4007 scoped IP. - match = ZONE_ID_RE.search(host) + match = _ZONE_ID_RE.search(host) if match: start, end = match.span(1) zone_id = host[start:end] @@ -291,46 +315,56 @@ def _normalize_host(host, scheme): zone_id = zone_id[3:] else: zone_id = zone_id[1:] - zone_id = "%" + _encode_invalid_chars(zone_id, UNRESERVED_CHARS) - return host[:start].lower() + zone_id + host[end:] + zone_id = _encode_invalid_chars(zone_id, _UNRESERVED_CHARS) + return f"{host[:start].lower()}%{zone_id}{host[end:]}" else: return host.lower() - elif not IPV4_RE.match(host): - return six.ensure_str( - b".".join([_idna_encode(label) for label in host.split(".")]) + elif not _IPV4_RE.match(host): + return to_str( + b".".join([_idna_encode(label) for label in host.split(".")]), + "ascii", ) return host -def _idna_encode(name): - if name and any(ord(x) >= 128 for x in name): +def _idna_encode(name: str) -> bytes: + if not name.isascii(): try: import idna except ImportError: - six.raise_from( - LocationParseError("Unable to parse URL without the 'idna' module"), - None, - ) + raise LocationParseError( + "Unable to parse URL without the 'idna' module" + ) from None + try: return idna.encode(name.lower(), strict=True, std3_rules=True) except idna.IDNAError: - six.raise_from( - LocationParseError(u"Name '%s' is not a valid IDNA label" % name), None - ) + raise LocationParseError( + f"Name '{name}' is not a valid IDNA label" + ) from None + return name.lower().encode("ascii") -def _encode_target(target): - """Percent-encodes a request target so that there are no invalid characters""" - path, query = TARGET_RE.match(target).groups() - target = _encode_invalid_chars(path, PATH_CHARS) - query = _encode_invalid_chars(query, QUERY_CHARS) +def _encode_target(target: str) -> str: + """Percent-encodes a request target so that there are no invalid characters + + Pre-condition for this function is that 'target' must start with '/'. + If that is the case then _TARGET_RE will always produce a match. + """ + match = _TARGET_RE.match(target) + if not match: # Defensive: + raise LocationParseError(f"{target!r} is not a valid request URI") + + path, query = match.groups() + encoded_target = _encode_invalid_chars(path, _PATH_CHARS) if query is not None: - target += "?" + query - return target + query = _encode_invalid_chars(query, _QUERY_CHARS) + encoded_target += "?" + query + return encoded_target -def parse_url(url): +def parse_url(url: str) -> Url: """ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is performed to parse incomplete urls. Fields not provided will be None. @@ -341,28 +375,44 @@ def parse_url(url): :param str url: URL to parse into a :class:`.Url` namedtuple. - Partly backwards-compatible with :mod:`urlparse`. + Partly backwards-compatible with :mod:`urllib.parse`. - Example:: + Example: - >>> parse_url('http://google.com/mail/') - Url(scheme='http', host='google.com', port=None, path='/mail/', ...) - >>> parse_url('google.com:80') - Url(scheme=None, host='google.com', port=80, path=None, ...) - >>> parse_url('/foo?bar') - Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) + .. code-block:: python + + import urllib3 + + print( urllib3.util.parse_url('http://google.com/mail/')) + # Url(scheme='http', host='google.com', port=None, path='/mail/', ...) + + print( urllib3.util.parse_url('google.com:80')) + # Url(scheme=None, host='google.com', port=80, path=None, ...) + + print( urllib3.util.parse_url('/foo?bar')) + # Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) """ if not url: # Empty return Url() source_url = url - if not SCHEME_RE.search(url): + if not _SCHEME_RE.search(url): url = "//" + url + scheme: str | None + authority: str | None + auth: str | None + host: str | None + port: str | None + port_int: int | None + path: str | None + query: str | None + fragment: str | None + try: - scheme, authority, path, query, fragment = URI_RE.match(url).groups() - normalize_uri = scheme is None or scheme.lower() in NORMALIZABLE_SCHEMES + scheme, authority, path, query, fragment = _URI_RE.match(url).groups() # type: ignore[union-attr] + normalize_uri = scheme is None or scheme.lower() in _NORMALIZABLE_SCHEMES if scheme: scheme = scheme.lower() @@ -370,31 +420,33 @@ def parse_url(url): if authority: auth, _, host_port = authority.rpartition("@") auth = auth or None - host, port = _HOST_PORT_RE.match(host_port).groups() + host, port = _HOST_PORT_RE.match(host_port).groups() # type: ignore[union-attr] if auth and normalize_uri: - auth = _encode_invalid_chars(auth, USERINFO_CHARS) + auth = _encode_invalid_chars(auth, _USERINFO_CHARS) if port == "": port = None else: auth, host, port = None, None, None if port is not None: - port = int(port) - if not (0 <= port <= 65535): + port_int = int(port) + if not (0 <= port_int <= 65535): raise LocationParseError(url) + else: + port_int = None host = _normalize_host(host, scheme) if normalize_uri and path: path = _remove_path_dot_segments(path) - path = _encode_invalid_chars(path, PATH_CHARS) + path = _encode_invalid_chars(path, _PATH_CHARS) if normalize_uri and query: - query = _encode_invalid_chars(query, QUERY_CHARS) + query = _encode_invalid_chars(query, _QUERY_CHARS) if normalize_uri and fragment: - fragment = _encode_invalid_chars(fragment, FRAGMENT_CHARS) + fragment = _encode_invalid_chars(fragment, _FRAGMENT_CHARS) - except (ValueError, AttributeError): - return six.raise_from(LocationParseError(source_url), None) + except (ValueError, AttributeError) as e: + raise LocationParseError(source_url) from e # For the sake of backwards compatibility we put empty # string values for path if there are any defined values @@ -406,30 +458,12 @@ def parse_url(url): else: path = None - # Ensure that each part of the URL is a `str` for - # backwards compatibility. - if isinstance(url, six.text_type): - ensure_func = six.ensure_text - else: - ensure_func = six.ensure_str - - def ensure_type(x): - return x if x is None else ensure_func(x) - return Url( - scheme=ensure_type(scheme), - auth=ensure_type(auth), - host=ensure_type(host), - port=port, - path=ensure_type(path), - query=ensure_type(query), - fragment=ensure_type(fragment), + scheme=scheme, + auth=auth, + host=host, + port=port_int, + path=path, + query=query, + fragment=fragment, ) - - -def get_host(url): - """ - Deprecated. Use :func:`parse_url` instead. - """ - p = parse_url(url) - return p.scheme or "http", p.hostname, p.port diff --git a/src/urllib3/util/util.py b/src/urllib3/util/util.py new file mode 100644 index 0000000..35c77e4 --- /dev/null +++ b/src/urllib3/util/util.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import typing +from types import TracebackType + + +def to_bytes( + x: str | bytes, encoding: str | None = None, errors: str | None = None +) -> bytes: + if isinstance(x, bytes): + return x + elif not isinstance(x, str): + raise TypeError(f"not expecting type {type(x).__name__}") + if encoding or errors: + return x.encode(encoding or "utf-8", errors=errors or "strict") + return x.encode() + + +def to_str( + x: str | bytes, encoding: str | None = None, errors: str | None = None +) -> str: + if isinstance(x, str): + return x + elif not isinstance(x, bytes): + raise TypeError(f"not expecting type {type(x).__name__}") + if encoding or errors: + return x.decode(encoding or "utf-8", errors=errors or "strict") + return x.decode() + + +def reraise( + tp: type[BaseException] | None, + value: BaseException, + tb: TracebackType | None = None, +) -> typing.NoReturn: + try: + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None # type: ignore[assignment] + tb = None diff --git a/src/urllib3/util/wait.py b/src/urllib3/util/wait.py index 21b4590..aeca0c7 100644 --- a/src/urllib3/util/wait.py +++ b/src/urllib3/util/wait.py @@ -1,18 +1,10 @@ -import errno +from __future__ import annotations + import select -import sys +import socket from functools import partial -try: - from time import monotonic -except ImportError: - from time import time as monotonic - -__all__ = ["NoWayToWaitForSocketError", "wait_for_read", "wait_for_write"] - - -class NoWayToWaitForSocketError(Exception): - pass +__all__ = ["wait_for_read", "wait_for_write"] # How should we wait on sockets? @@ -37,37 +29,13 @@ class NoWayToWaitForSocketError(Exception): # So: on Windows we use select(), and everywhere else we use poll(). We also # fall back to select() in case poll() is somehow broken or missing. -if sys.version_info >= (3, 5): - # Modern Python, that retries syscalls by default - def _retry_on_intr(fn, timeout): - return fn(timeout) - -else: - # Old and broken Pythons. - def _retry_on_intr(fn, timeout): - if timeout is None: - deadline = float("inf") - else: - deadline = monotonic() + timeout - - while True: - try: - return fn(timeout) - # OSError for 3 <= pyver < 3.5, select.error for pyver <= 2.7 - except (OSError, select.error) as e: - # 'e.args[0]' incantation works for both OSError and select.error - if e.args[0] != errno.EINTR: - raise - else: - timeout = deadline - monotonic() - if timeout < 0: - timeout = 0 - if timeout == float("inf"): - timeout = None - continue - - -def select_wait_for_socket(sock, read=False, write=False, timeout=None): + +def select_wait_for_socket( + sock: socket.socket, + read: bool = False, + write: bool = False, + timeout: float | None = None, +) -> bool: if not read and not write: raise RuntimeError("must specify at least one of read=True, write=True") rcheck = [] @@ -82,11 +50,16 @@ def select_wait_for_socket(sock, read=False, write=False, timeout=None): # sockets for both conditions. (The stdlib selectors module does the same # thing.) fn = partial(select.select, rcheck, wcheck, wcheck) - rready, wready, xready = _retry_on_intr(fn, timeout) + rready, wready, xready = fn(timeout) return bool(rready or wready or xready) -def poll_wait_for_socket(sock, read=False, write=False, timeout=None): +def poll_wait_for_socket( + sock: socket.socket, + read: bool = False, + write: bool = False, + timeout: float | None = None, +) -> bool: if not read and not write: raise RuntimeError("must specify at least one of read=True, write=True") mask = 0 @@ -98,32 +71,33 @@ def poll_wait_for_socket(sock, read=False, write=False, timeout=None): poll_obj.register(sock, mask) # For some reason, poll() takes timeout in milliseconds - def do_poll(t): + def do_poll(t: float | None) -> list[tuple[int, int]]: if t is not None: t *= 1000 return poll_obj.poll(t) - return bool(_retry_on_intr(do_poll, timeout)) - - -def null_wait_for_socket(*args, **kwargs): - raise NoWayToWaitForSocketError("no select-equivalent available") + return bool(do_poll(timeout)) -def _have_working_poll(): +def _have_working_poll() -> bool: # Apparently some systems have a select.poll that fails as soon as you try # to use it, either due to strange configuration or broken monkeypatching # from libraries like eventlet/greenlet. try: poll_obj = select.poll() - _retry_on_intr(poll_obj.poll, 0) + poll_obj.poll(0) except (AttributeError, OSError): return False else: return True -def wait_for_socket(*args, **kwargs): +def wait_for_socket( + sock: socket.socket, + read: bool = False, + write: bool = False, + timeout: float | None = None, +) -> bool: # We delay choosing which implementation to use until the first time we're # called. We could do it at import time, but then we might make the wrong # decision if someone goes wild with monkeypatching select.poll after @@ -133,19 +107,17 @@ def wait_for_socket(*args, **kwargs): wait_for_socket = poll_wait_for_socket elif hasattr(select, "select"): wait_for_socket = select_wait_for_socket - else: # Platform-specific: Appengine. - wait_for_socket = null_wait_for_socket - return wait_for_socket(*args, **kwargs) + return wait_for_socket(sock, read, write, timeout) -def wait_for_read(sock, timeout=None): +def wait_for_read(sock: socket.socket, timeout: float | None = None) -> bool: """Waits for reading to be available on a given socket. Returns True if the socket is readable, or False if the timeout expired. """ return wait_for_socket(sock, read=True, timeout=timeout) -def wait_for_write(sock, timeout=None): +def wait_for_write(sock: socket.socket, timeout: float | None = None) -> bool: """Waits for writing to be available on a given socket. Returns True if the socket is readable, or False if the timeout expired. """ diff --git a/test/__init__.py b/test/__init__.py index 2307b2d..b5eedc7 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,30 +1,49 @@ +from __future__ import annotations + import errno +import importlib.util import logging import os import platform import socket import sys +import typing import warnings +from collections.abc import Sequence +from functools import wraps +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec +from types import ModuleType, TracebackType import pytest try: try: - import brotlicffi as brotli + import brotlicffi as brotli # type: ignore[import-not-found] except ImportError: - import brotli + import brotli # type: ignore[import-not-found] except ImportError: brotli = None -from urllib3 import util +try: + import zstandard as _unused_module_zstd # noqa: F401 +except ImportError: + HAS_ZSTD = False +else: + HAS_ZSTD = True + +from urllib3.connectionpool import ConnectionPool from urllib3.exceptions import HTTPWarning -from urllib3.packages import six -from urllib3.util import ssl_ try: import urllib3.contrib.pyopenssl as pyopenssl except ImportError: - pyopenssl = None + pyopenssl = None # type: ignore[assignment] + + +_RT = typing.TypeVar("_RT") # return type +_TestFuncT = typing.TypeVar("_TestFuncT", bound=typing.Callable[..., typing.Any]) + # We need a host that will not immediately close the connection with a TCP # Reset. @@ -40,7 +59,7 @@ VALID_SOURCE_ADDRESSES = [(("::1", 0), True), (("127.0.0.1", 0), False)] # RFC 5737: 192.0.2.0/24 is for testing only. # RFC 3849: 2001:db8::/32 is for documentation only. -INVALID_SOURCE_ADDRESSES = [("192.0.2.255", 0), ("2001:db8::1", 0)] +INVALID_SOURCE_ADDRESSES = [(("192.0.2.255", 0), False), (("2001:db8::1", 0), True)] # We use timeouts in three different ways in our tests # @@ -50,12 +69,14 @@ # 3. To test our timeout logic by using two different values, eg. by using different # values at the pool level and at the request level. SHORT_TIMEOUT = 0.001 -LONG_TIMEOUT = 0.01 +LONG_TIMEOUT = 0.1 if os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS") == "true": LONG_TIMEOUT = 0.5 +DUMMY_POOL = ConnectionPool("dummy") -def _can_resolve(host): + +def _can_resolve(host: str) -> bool: """Returns True if the system can resolve host to an address.""" try: socket.getaddrinfo(host, None, socket.AF_UNSPEC) @@ -64,214 +85,122 @@ def _can_resolve(host): return False -def has_alpn(ctx_cls=None): - """Detect if ALPN support is enabled.""" - ctx_cls = ctx_cls or util.SSLContext - ctx = ctx_cls(protocol=ssl_.PROTOCOL_TLS) - try: - if hasattr(ctx, "set_alpn_protocols"): - ctx.set_alpn_protocols(ssl_.ALPN_PROTOCOLS) - return True - except NotImplementedError: - pass - return False - - # Some systems might not resolve "localhost." correctly. # See https://github.com/urllib3/urllib3/issues/1809 and # https://github.com/urllib3/urllib3/pull/1475#issuecomment-440788064. RESOLVES_LOCALHOST_FQDN = _can_resolve("localhost.") -def clear_warnings(cls=HTTPWarning): +def clear_warnings(cls: type[Warning] = HTTPWarning) -> None: new_filters = [] for f in warnings.filters: if issubclass(f[2], cls): continue new_filters.append(f) - warnings.filters[:] = new_filters + warnings.filters[:] = new_filters # type: ignore[index] -def setUp(): +def setUp() -> None: clear_warnings() warnings.simplefilter("ignore", HTTPWarning) -def onlyPy279OrNewer(test): - """Skips this test unless you are on Python 2.7.9 or later.""" - - @six.wraps(test) - def wrapper(*args, **kwargs): - msg = "{name} requires Python 2.7.9+ to run".format(name=test.__name__) - if sys.version_info < (2, 7, 9): - pytest.skip(msg) - return test(*args, **kwargs) - - return wrapper - - -def onlyPy2(test): - """Skips this test unless you are on Python 2.x""" - - @six.wraps(test) - def wrapper(*args, **kwargs): - msg = "{name} requires Python 2.x to run".format(name=test.__name__) - if not six.PY2: - pytest.skip(msg) - return test(*args, **kwargs) - - return wrapper - - -def onlyPy3(test): - """Skips this test unless you are on Python3.x""" - - @six.wraps(test) - def wrapper(*args, **kwargs): - msg = "{name} requires Python3.x to run".format(name=test.__name__) - if six.PY2: - pytest.skip(msg) - return test(*args, **kwargs) - - return wrapper - - -def notPyPy2(test): - """Skips this test on PyPy2""" - - @six.wraps(test) - def wrapper(*args, **kwargs): - # https://github.com/testing-cabal/mock/issues/438 - msg = "{} fails with PyPy 2 dues to funcsigs bugs".format(test.__name__) - if platform.python_implementation() == "PyPy" and sys.version_info[0] == 2: - pytest.xfail(msg) - return test(*args, **kwargs) - - return wrapper - - -def notWindows(test): +def notWindows() -> typing.Callable[[_TestFuncT], _TestFuncT]: """Skips this test on Windows""" - - @six.wraps(test) - def wrapper(*args, **kwargs): - msg = "{name} does not run on Windows".format(name=test.__name__) - if platform.system() == "Windows": - pytest.skip(msg) - return test(*args, **kwargs) - - return wrapper - - -def onlyBrotlipy(): - return pytest.mark.skipif(brotli is None, reason="only run if brotlipy is present") - - -def notBrotlipy(): return pytest.mark.skipif( - brotli is not None, reason="only run if brotlipy is absent" + platform.system() == "Windows", + reason="Test does not run on Windows", ) -def onlySecureTransport(test): - """Runs this test when SecureTransport is in use.""" +def onlyBrotli() -> typing.Callable[[_TestFuncT], _TestFuncT]: + return pytest.mark.skipif( + brotli is None, reason="only run if brotli library is present" + ) - @six.wraps(test) - def wrapper(*args, **kwargs): - msg = "{name} only runs with SecureTransport".format(name=test.__name__) - if not ssl_.IS_SECURETRANSPORT: - pytest.skip(msg) - return test(*args, **kwargs) - return wrapper +def notBrotli() -> typing.Callable[[_TestFuncT], _TestFuncT]: + return pytest.mark.skipif( + brotli is not None, reason="only run if a brotli library is absent" + ) -def notSecureTransport(test): - """Skips this test when SecureTransport is in use.""" +def onlyZstd() -> typing.Callable[[_TestFuncT], _TestFuncT]: + return pytest.mark.skipif( + not HAS_ZSTD, reason="only run if a python-zstandard library is installed" + ) - @six.wraps(test) - def wrapper(*args, **kwargs): - msg = "{name} does not run with SecureTransport".format(name=test.__name__) - if ssl_.IS_SECURETRANSPORT: - pytest.skip(msg) - return test(*args, **kwargs) - return wrapper +def notZstd() -> typing.Callable[[_TestFuncT], _TestFuncT]: + return pytest.mark.skipif( + HAS_ZSTD, + reason="only run if a python-zstandard library is not installed", + ) _requires_network_has_route = None -def requires_network(test): +def requires_network() -> typing.Callable[[_TestFuncT], _TestFuncT]: """Helps you skip tests that require the network""" - def _is_unreachable_err(err): + def _is_unreachable_err(err: Exception) -> bool: return getattr(err, "errno", None) in ( errno.ENETUNREACH, errno.EHOSTUNREACH, # For OSX ) - def _has_route(): + def _has_route() -> bool: try: sock = socket.create_connection((TARPIT_HOST, 80), 0.0001) sock.close() return True except socket.timeout: return True - except socket.error as e: + except OSError as e: if _is_unreachable_err(e): return False else: raise - @six.wraps(test) - def wrapper(*args, **kwargs): - global _requires_network_has_route + def _skip_if_no_route(f: _TestFuncT) -> _TestFuncT: + """Skip test exuction if network is unreachable""" - if _requires_network_has_route is None: - _requires_network_has_route = _has_route() + @wraps(f) + def wrapper(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + global _requires_network_has_route + if _requires_network_has_route is None: + _requires_network_has_route = _has_route() + if not _requires_network_has_route: + pytest.skip("Can't run the test because the network is unreachable") + return f(*args, **kwargs) - if _requires_network_has_route: - return test(*args, **kwargs) - else: - msg = "Can't run {name} because the network is unreachable".format( - name=test.__name__ - ) - pytest.skip(msg) + return typing.cast(_TestFuncT, wrapper) - return wrapper + def _decorator_requires_internet( + decorator: typing.Callable[[_TestFuncT], _TestFuncT] + ) -> typing.Callable[[_TestFuncT], _TestFuncT]: + """Mark a decorator with the "requires_internet" mark""" + def wrapper(f: _TestFuncT) -> typing.Any: + return pytest.mark.requires_network(decorator(f)) -def requires_ssl_context_keyfile_password(test): - @six.wraps(test) - def wrapper(*args, **kwargs): - if ( - not ssl_.IS_PYOPENSSL and sys.version_info < (2, 7, 9) - ) or ssl_.IS_SECURETRANSPORT: - pytest.skip( - "%s requires password parameter for " - "SSLContext.load_cert_chain()" % test.__name__ - ) - return test(*args, **kwargs) + return wrapper - return wrapper + return _decorator_requires_internet(_skip_if_no_route) -def resolvesLocalhostFQDN(test): +def resolvesLocalhostFQDN() -> typing.Callable[[_TestFuncT], _TestFuncT]: """Test requires successful resolving of 'localhost.'""" - - @six.wraps(test) - def wrapper(*args, **kwargs): - if not RESOLVES_LOCALHOST_FQDN: - pytest.skip("Can't resolve localhost.") - return test(*args, **kwargs) - - return wrapper + return pytest.mark.skipif( + not RESOLVES_LOCALHOST_FQDN, + reason="Can't resolve localhost.", + ) -def withPyOpenSSL(test): - @six.wraps(test) - def wrapper(*args, **kwargs): +def withPyOpenSSL(test: typing.Callable[..., _RT]) -> typing.Callable[..., _RT]: + @wraps(test) + def wrapper(*args: typing.Any, **kwargs: typing.Any) -> _RT: if not pyopenssl: pytest.skip("pyopenssl not available, skipping test.") return test(*args, **kwargs) @@ -285,34 +214,114 @@ def wrapper(*args, **kwargs): class _ListHandler(logging.Handler): - def __init__(self): - super(_ListHandler, self).__init__() - self.records = [] + def __init__(self) -> None: + super().__init__() + self.records: list[logging.LogRecord] = [] - def emit(self, record): + def emit(self, record: logging.LogRecord) -> None: self.records.append(record) -class LogRecorder(object): - def __init__(self, target=logging.root): - super(LogRecorder, self).__init__() +class LogRecorder: + def __init__(self, target: logging.Logger = logging.root) -> None: + super().__init__() self._target = target self._handler = _ListHandler() @property - def records(self): + def records(self) -> list[logging.LogRecord]: return self._handler.records - def install(self): + def install(self) -> None: self._target.addHandler(self._handler) - def uninstall(self): + def uninstall(self) -> None: self._target.removeHandler(self._handler) - def __enter__(self): + def __enter__(self) -> list[logging.LogRecord]: self.install() return self.records - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> typing.Literal[False]: self.uninstall() return False + + +class ImportBlockerLoader(Loader): + def __init__(self, fullname: str) -> None: + self._fullname = fullname + + def load_module(self, fullname: str) -> ModuleType: + raise ImportError(f"import of {fullname} is blocked") + + def exec_module(self, module: ModuleType) -> None: + raise ImportError(f"import of {self._fullname} is blocked") + + +class ImportBlocker(MetaPathFinder): + """ + Block Imports + + To be placed on ``sys.meta_path``. This ensures that the modules + specified cannot be imported, even if they are a builtin. + """ + + def __init__(self, *namestoblock: str) -> None: + self.namestoblock = namestoblock + + def find_module( + self, fullname: str, path: typing.Sequence[bytes | str] | None = None + ) -> Loader | None: + if fullname in self.namestoblock: + return ImportBlockerLoader(fullname) + return None + + def find_spec( + self, + fullname: str, + path: Sequence[bytes | str] | None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + loader = self.find_module(fullname, path) + if loader is None: + return None + + return importlib.util.spec_from_loader(fullname, loader) + + +class ModuleStash(MetaPathFinder): + """ + Stashes away previously imported modules + + If we reimport a module the data from coverage is lost, so we reuse the old + modules + """ + + def __init__( + self, namespace: str, modules: dict[str, ModuleType] = sys.modules + ) -> None: + self.namespace = namespace + self.modules = modules + self._data: dict[str, ModuleType] = {} + + def stash(self) -> None: + if self.namespace in self.modules: + self._data[self.namespace] = self.modules.pop(self.namespace) + + for module in list(self.modules.keys()): + if module.startswith(self.namespace + "."): + self._data[module] = self.modules.pop(module) + + def pop(self) -> None: + self.modules.pop(self.namespace, None) + + for module in list(self.modules.keys()): + if module.startswith(self.namespace + "."): + self.modules.pop(module) + + self.modules.update(self._data) diff --git a/test/appengine/__init__.py b/test/appengine/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/appengine/conftest.py b/test/appengine/conftest.py deleted file mode 100644 index 0b9d1f1..0000000 --- a/test/appengine/conftest.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -# Import py.test hooks and fixtures for App Engine -try: - from gcp_devrel.testing.appengine import ( - pytest_configure, - pytest_runtest_call, - testbed, - ) -except ImportError: - pass - -import pytest -import six - -__all__ = [ - "pytest_configure", - "pytest_runtest_call", - "pytest_ignore_collect", - "testbed", - "sandbox", -] - - -@pytest.fixture -def sandbox(testbed): - """ - Enables parts of the GAE sandbox that are relevant. - Inserts the stub module import hook which causes the usage of - appengine-specific httplib, httplib2, socket, etc. - """ - try: - from google.appengine.tools.devappserver2.python import sandbox - except ImportError: - from google.appengine.tools.devappserver2.python.runtime import sandbox - - for name in list(sys.modules): - if name in sandbox.dist27.MODULE_OVERRIDES: - del sys.modules[name] - sys.meta_path.insert(0, sandbox.StubModuleImportHook()) - sys.path_importer_cache = {} - - yield testbed - - sys.meta_path = [ - x for x in sys.meta_path if not isinstance(x, sandbox.StubModuleImportHook) - ] - sys.path_importer_cache = {} - - # Delete any instances of sandboxed modules. - for name in list(sys.modules): - if name in sandbox.dist27.MODULE_OVERRIDES: - del sys.modules[name] - - -def pytest_ignore_collect(path, config): - """Skip App Engine tests in python 3 or if no SDK is available.""" - if "appengine" in str(path): - if not six.PY2: - return True - if not os.environ.get("GAE_SDK_PATH"): - return True - return False diff --git a/test/appengine/test_gae_manager.py b/test/appengine/test_gae_manager.py deleted file mode 100644 index 30a9c1e..0000000 --- a/test/appengine/test_gae_manager.py +++ /dev/null @@ -1,178 +0,0 @@ -from test import SHORT_TIMEOUT -from test.with_dummyserver import test_connectionpool - -import pytest - -import dummyserver.testcase -import urllib3.exceptions -import urllib3.util.retry -import urllib3.util.url -from urllib3.contrib import appengine - - -# This class is used so we can re-use the tests from the connection pool. -# It proxies all requests to the manager. -class MockPool(object): - def __init__(self, host, port, manager, scheme="http"): - self.host = host - self.port = port - self.manager = manager - self.scheme = scheme - - def request(self, method, url, *args, **kwargs): - url = self._absolute_url(url) - return self.manager.request(method, url, *args, **kwargs) - - def urlopen(self, method, url, *args, **kwargs): - url = self._absolute_url(url) - return self.manager.urlopen(method, url, *args, **kwargs) - - def _absolute_url(self, path): - return urllib3.util.url.Url( - scheme=self.scheme, host=self.host, port=self.port, path=path - ).url - - -# Note that this doesn't run in the sandbox, it only runs with the URLFetch -# API stub enabled. There's no need to enable the sandbox as we know for a fact -# that URLFetch is used by the connection manager. -@pytest.mark.usefixtures("testbed") -class TestGAEConnectionManager(test_connectionpool.TestConnectionPool): - def setup_method(self, method): - self.manager = appengine.AppEngineManager() - self.pool = MockPool(self.host, self.port, self.manager) - - # Tests specific to AppEngineManager - - def test_exceptions(self): - # DeadlineExceededError -> TimeoutError - with pytest.raises(urllib3.exceptions.TimeoutError): - self.pool.request( - "GET", - "/sleep?seconds={}".format(5 * SHORT_TIMEOUT), - timeout=SHORT_TIMEOUT, - ) - - # InvalidURLError -> ProtocolError - with pytest.raises(urllib3.exceptions.ProtocolError): - self.manager.request("GET", "ftp://invalid/url") - - # DownloadError -> ProtocolError - with pytest.raises(urllib3.exceptions.ProtocolError): - self.manager.request("GET", "http://0.0.0.0") - - # ResponseTooLargeError -> AppEnginePlatformError - with pytest.raises(appengine.AppEnginePlatformError): - self.pool.request( - "GET", "/nbytes?length=33554433" - ) # One byte over 32 megabytes. - - # URLFetch reports the request too large error as a InvalidURLError, - # which maps to a AppEnginePlatformError. - body = b"1" * 10485761 # One byte over 10 megabytes. - with pytest.raises(appengine.AppEnginePlatformError): - self.manager.request("POST", "/", body=body) - - # Re-used tests below this line. - # Subsumed tests - test_timeout_float = None # Covered by test_exceptions. - - # Non-applicable tests - test_conn_closed = None - test_nagle = None - test_socket_options = None - test_disable_default_socket_options = None - test_defaults_are_applied = None - test_tunnel = None - test_keepalive = None - test_keepalive_close = None - test_connection_count = None - test_connection_count_bigpool = None - test_for_double_release = None - test_release_conn_parameter = None - test_stream_keepalive = None - test_cleanup_on_connection_error = None - test_read_chunked_short_circuit = None - test_read_chunked_on_closed_response = None - - # Tests that should likely be modified for appengine specific stuff - test_timeout = None - test_connect_timeout = None - test_connection_error_retries = None - test_total_timeout = None - test_none_total_applies_connect = None - test_timeout_success = None - test_source_address_error = None - test_bad_connect = None - test_partial_response = None - test_dns_error = None - - -@pytest.mark.usefixtures("testbed") -class TestGAEConnectionManagerWithSSL(dummyserver.testcase.HTTPSDummyServerTestCase): - def setup_method(self, method): - self.manager = appengine.AppEngineManager() - self.pool = MockPool(self.host, self.port, self.manager, "https") - - def test_exceptions(self): - # SSLCertificateError -> SSLError - # SSLError is raised with dummyserver because URLFetch doesn't allow - # self-signed certs. - with pytest.raises(urllib3.exceptions.SSLError): - self.pool.request("GET", "/") - - -@pytest.mark.usefixtures("testbed") -class TestGAERetry(test_connectionpool.TestRetry): - def setup_method(self, method): - self.manager = appengine.AppEngineManager() - self.pool = MockPool(self.host, self.port, self.manager) - - def test_default_method_whitelist_retried(self): - """urllib3 should retry methods in the default method whitelist""" - retry = urllib3.util.retry.Retry(total=1, status_forcelist=[418]) - # Use HEAD instead of OPTIONS, as URLFetch doesn't support OPTIONS - resp = self.pool.request( - "HEAD", - "/successful_retry", - headers={"test-name": "test_default_whitelist"}, - retries=retry, - ) - assert resp.status == 200 - - def test_retry_return_in_response(self): - headers = {"test-name": "test_retry_return_in_response"} - retry = urllib3.util.retry.Retry(total=2, status_forcelist=[418]) - resp = self.pool.request( - "GET", "/successful_retry", headers=headers, retries=retry - ) - assert resp.status == 200 - assert resp.retries.total == 1 - # URLFetch use absolute urls. - assert resp.retries.history == ( - urllib3.util.retry.RequestHistory( - "GET", self.pool._absolute_url("/successful_retry"), None, 418, None - ), - ) - - # test_max_retry = None - # test_disabled_retry = None - # We don't need these tests because URLFetch resolves its own redirects. - test_retry_redirect_history = None - test_multi_redirect_history = None - - -@pytest.mark.usefixtures("testbed") -class TestGAERetryAfter(test_connectionpool.TestRetryAfter): - def setup_method(self, method): - # Disable urlfetch which doesn't respect Retry-After header. - self.manager = appengine.AppEngineManager(urlfetch_retries=False) - self.pool = MockPool(self.host, self.port, self.manager) - - -def test_gae_environ(): - assert not appengine.is_appengine() - assert not appengine.is_appengine_sandbox() - assert not appengine.is_local_appengine() - assert not appengine.is_prod_appengine() - assert not appengine.is_prod_appengine_mvms() diff --git a/test/appengine/test_urlfetch.py b/test/appengine/test_urlfetch.py deleted file mode 100644 index 74484ea..0000000 --- a/test/appengine/test_urlfetch.py +++ /dev/null @@ -1,66 +0,0 @@ -"""These tests ensure that when running in App Engine standard with the -App Engine sandbox enabled that urllib3 appropriately uses the App -Engine-patched version of httplib to make requests.""" - -import httplib -import pytest -import StringIO -from mock import patch - -from ..test_no_ssl import TestWithoutSSL - - -class MockResponse(object): - def __init__(self, content, status_code, content_was_truncated, final_url, headers): - - self.content = content - self.status_code = status_code - self.content_was_truncated = content_was_truncated - self.final_url = final_url - self.header_msg = httplib.HTTPMessage( - StringIO.StringIO( - "".join(["%s: %s\n" % (k, v) for k, v in headers.iteritems()] + ["\n"]) - ) - ) - self.headers = headers - - -@pytest.mark.usefixtures("sandbox") -class TestHTTP(TestWithoutSSL): - def test_urlfetch_called_with_http(self): - """Check that URLFetch is used to fetch non-https resources.""" - resp = MockResponse( - "OK", 200, False, "http://www.google.com", {"content-type": "text/plain"} - ) - fetch_patch = patch("google.appengine.api.urlfetch.fetch", return_value=resp) - with fetch_patch as fetch_mock: - import urllib3 - - pool = urllib3.HTTPConnectionPool("www.google.com", "80") - r = pool.request("GET", "/") - assert r.status == 200, r.data - assert fetch_mock.call_count == 1 - - -@pytest.mark.usefixtures("sandbox") -class TestHTTPS(object): - @pytest.mark.xfail( - reason="This is not yet supported by urlfetch, presence of the ssl " - "module will bypass urlfetch." - ) - def test_urlfetch_called_with_https(self): - """ - Check that URLFetch is used when fetching https resources - """ - resp = MockResponse( - "OK", 200, False, "https://www.google.com", {"content-type": "text/plain"} - ) - fetch_patch = patch("google.appengine.api.urlfetch.fetch", return_value=resp) - with fetch_patch as fetch_mock: - import urllib3 - - pool = urllib3.HTTPSConnectionPool("www.google.com", "443") - pool.ConnectionCls = urllib3.connection.UnverifiedHTTPSConnection - r = pool.request("GET", "/") - assert r.status == 200, r.data - assert fetch_mock.call_count == 1 diff --git a/test/benchmark.py b/test/benchmark.py deleted file mode 100644 index 67d141b..0000000 --- a/test/benchmark.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python - -""" -Really simple rudimentary benchmark to compare ConnectionPool versus standard -urllib to demonstrate the usefulness of connection re-using. -""" -from __future__ import print_function - -import sys -import time -import urllib - -sys.path.append("../") -import urllib3 # noqa: E402 - -# URLs to download. Doesn't matter as long as they're from the same host, so we -# can take advantage of connection re-using. -TO_DOWNLOAD = [ - "http://code.google.com/apis/apps/", - "http://code.google.com/apis/base/", - "http://code.google.com/apis/blogger/", - "http://code.google.com/apis/calendar/", - "http://code.google.com/apis/codesearch/", - "http://code.google.com/apis/contact/", - "http://code.google.com/apis/books/", - "http://code.google.com/apis/documents/", - "http://code.google.com/apis/finance/", - "http://code.google.com/apis/health/", - "http://code.google.com/apis/notebook/", - "http://code.google.com/apis/picasaweb/", - "http://code.google.com/apis/spreadsheets/", - "http://code.google.com/apis/webmastertools/", - "http://code.google.com/apis/youtube/", -] - - -def urllib_get(url_list): - assert url_list - for url in url_list: - now = time.time() - urllib.urlopen(url) - elapsed = time.time() - now - print("Got in %0.3f: %s" % (elapsed, url)) - - -def pool_get(url_list): - assert url_list - pool = urllib3.PoolManager() - for url in url_list: - now = time.time() - pool.request("GET", url, assert_same_host=False) - elapsed = time.time() - now - print("Got in %0.3fs: %s" % (elapsed, url)) - - -if __name__ == "__main__": - print("Running pool_get ...") - now = time.time() - pool_get(TO_DOWNLOAD) - pool_elapsed = time.time() - now - - print("Running urllib_get ...") - now = time.time() - urllib_get(TO_DOWNLOAD) - urllib_elapsed = time.time() - now - - print("Completed pool_get in %0.3fs" % pool_elapsed) - print("Completed urllib_get in %0.3fs" % urllib_elapsed) - - -""" -Example results: - -Completed pool_get in 1.163s -Completed urllib_get in 2.318s -""" diff --git a/test/conftest.py b/test/conftest.py index 656493a..23413f5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,37 +1,69 @@ -import collections +from __future__ import annotations + import contextlib -import platform import socket import ssl -import sys -import threading +import typing +from pathlib import Path import pytest import trustme -from tornado import ioloop, web -from dummyserver.handlers import TestingApp -from dummyserver.proxy import ProxyHandler -from dummyserver.server import HAS_IPV6, run_tornado_app -from dummyserver.testcase import HTTPSDummyServerTestCase +import urllib3.http2 +import urllib3.http2.probe as http2_probe +from dummyserver.app import hypercorn_app +from dummyserver.asgi_proxy import ProxyApp +from dummyserver.hypercornserver import run_hypercorn_in_thread +from dummyserver.socketserver import HAS_IPV6 +from dummyserver.testcase import HTTPSHypercornDummyServerTestCase from urllib3.util import ssl_ from .tz_stub import stub_timezone_ctx -# The Python 3.8+ default loop on Windows breaks Tornado -@pytest.fixture(scope="session", autouse=True) -def configure_windows_event_loop(): - if sys.version_info >= (3, 8) and platform.system() == "Windows": - import asyncio - - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - - -ServerConfig = collections.namedtuple("ServerConfig", ["host", "port", "ca_certs"]) +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--integration", + action="store_true", + default=False, + help="run integration tests only", + ) -def _write_cert_to_dir(cert, tmpdir, file_prefix="server"): +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + integration_mode = bool(config.getoption("--integration")) + skip_integration = pytest.mark.skip( + reason="skipping, need --integration option to run" + ) + skip_normal = pytest.mark.skip( + reason="skipping non integration tests in --integration mode" + ) + for item in items: + if "integration" in item.keywords and not integration_mode: + item.add_marker(skip_integration) + elif integration_mode and "integration" not in item.keywords: + item.add_marker(skip_normal) + + +class ServerConfig(typing.NamedTuple): + scheme: str + host: str + port: int + ca_certs: str + + @property + def base_url(self) -> str: + host = self.host + if ":" in host: + host = f"[{host}]" + return f"{self.scheme}://{host}:{self.port}" + + +def _write_cert_to_dir( + cert: trustme.LeafCert, tmpdir: Path, file_prefix: str = "server" +) -> dict[str, str]: cert_path = str(tmpdir / ("%s.pem" % file_prefix)) key_path = str(tmpdir / ("%s.key" % file_prefix)) cert.private_key_pem.write_to_path(key_path) @@ -41,85 +73,115 @@ def _write_cert_to_dir(cert, tmpdir, file_prefix="server"): @contextlib.contextmanager -def run_server_in_thread(scheme, host, tmpdir, ca, server_cert): +def run_server_in_thread( + scheme: str, host: str, tmpdir: Path, ca: trustme.CA, server_cert: trustme.LeafCert +) -> typing.Generator[ServerConfig]: ca_cert_path = str(tmpdir / "ca.pem") ca.cert_pem.write_to_path(ca_cert_path) - server_certs = _write_cert_to_dir(server_cert, tmpdir) - io_loop = ioloop.IOLoop.current() - app = web.Application([(r".*", TestingApp)]) - server, port = run_tornado_app(app, io_loop, server_certs, scheme, host) - server_thread = threading.Thread(target=io_loop.start) - server_thread.start() - - yield ServerConfig(host, port, ca_cert_path) - io_loop.add_callback(server.stop) - io_loop.add_callback(io_loop.stop) - server_thread.join() + with run_hypercorn_in_thread(host, server_certs, hypercorn_app) as port: + yield ServerConfig(scheme, host, port, ca_cert_path) @contextlib.contextmanager def run_server_and_proxy_in_thread( - proxy_scheme, proxy_host, tmpdir, ca, proxy_cert, server_cert -): + proxy_scheme: str, + proxy_host: str, + tmpdir: Path, + ca: trustme.CA, + proxy_cert: trustme.LeafCert, + server_cert: trustme.LeafCert, +) -> typing.Generator[tuple[ServerConfig, ServerConfig]]: ca_cert_path = str(tmpdir / "ca.pem") ca.cert_pem.write_to_path(ca_cert_path) server_certs = _write_cert_to_dir(server_cert, tmpdir) proxy_certs = _write_cert_to_dir(proxy_cert, tmpdir, "proxy") - io_loop = ioloop.IOLoop.current() - server = web.Application([(r".*", TestingApp)]) - server, port = run_tornado_app(server, io_loop, server_certs, "https", "localhost") - server_config = ServerConfig("localhost", port, ca_cert_path) + with contextlib.ExitStack() as stack: + port = stack.enter_context( + run_hypercorn_in_thread("localhost", server_certs, hypercorn_app) + ) + proxy_port = stack.enter_context( + run_hypercorn_in_thread(proxy_host, proxy_certs, ProxyApp()) + ) - proxy = web.Application([(r".*", ProxyHandler)]) - proxy_app, proxy_port = run_tornado_app( - proxy, io_loop, proxy_certs, proxy_scheme, proxy_host - ) - proxy_config = ServerConfig(proxy_host, proxy_port, ca_cert_path) + yield ( + ServerConfig(proxy_scheme, proxy_host, proxy_port, ca_cert_path), + ServerConfig("https", "localhost", port, ca_cert_path), + ) - server_thread = threading.Thread(target=io_loop.start) - server_thread.start() - yield (proxy_config, server_config) +@pytest.fixture(params=["localhost", "127.0.0.1", "::1"]) +def loopback_host(request: typing.Any) -> typing.Generator[str]: + host = request.param + if host == "::1" and not HAS_IPV6: + pytest.skip("Test requires IPv6 on loopback") + yield host - io_loop.add_callback(server.stop) - io_loop.add_callback(proxy_app.stop) - io_loop.add_callback(io_loop.stop) - server_thread.join() +@pytest.fixture() +def san_server( + loopback_host: str, tmp_path_factory: pytest.TempPathFactory +) -> typing.Generator[ServerConfig]: + tmpdir = tmp_path_factory.mktemp("certs") + ca = trustme.CA() + server_cert = ca.issue_cert(loopback_host) -@pytest.fixture -def no_san_server(tmp_path_factory): + with run_server_in_thread("https", loopback_host, tmpdir, ca, server_cert) as cfg: + yield cfg + + +@pytest.fixture() +def no_san_server( + loopback_host: str, tmp_path_factory: pytest.TempPathFactory +) -> typing.Generator[ServerConfig]: tmpdir = tmp_path_factory.mktemp("certs") ca = trustme.CA() - # only common name, no subject alternative names - server_cert = ca.issue_cert(common_name=u"localhost") + server_cert = ca.issue_cert(common_name=loopback_host) - with run_server_in_thread("https", "localhost", tmpdir, ca, server_cert) as cfg: + with run_server_in_thread("https", loopback_host, tmpdir, ca, server_cert) as cfg: yield cfg @pytest.fixture() -def no_san_server_with_different_commmon_name(tmp_path_factory): +def no_san_server_with_different_commmon_name( + tmp_path_factory: pytest.TempPathFactory, +) -> typing.Generator[ServerConfig]: tmpdir = tmp_path_factory.mktemp("certs") ca = trustme.CA() - server_cert = ca.issue_cert(common_name=u"example.com") + server_cert = ca.issue_cert(common_name="example.com") with run_server_in_thread("https", "localhost", tmpdir, ca, server_cert) as cfg: yield cfg @pytest.fixture -def no_san_proxy(tmp_path_factory): +def san_proxy_with_server( + loopback_host: str, tmp_path_factory: pytest.TempPathFactory +) -> typing.Generator[tuple[ServerConfig, ServerConfig]]: + tmpdir = tmp_path_factory.mktemp("certs") + ca = trustme.CA() + proxy_cert = ca.issue_cert(loopback_host) + server_cert = ca.issue_cert("localhost") + + with run_server_and_proxy_in_thread( + "https", loopback_host, tmpdir, ca, proxy_cert, server_cert + ) as cfg: + yield cfg + + +@pytest.fixture +def no_san_proxy_with_server( + tmp_path_factory: pytest.TempPathFactory, +) -> typing.Generator[tuple[ServerConfig, ServerConfig]]: tmpdir = tmp_path_factory.mktemp("certs") ca = trustme.CA() # only common name, no subject alternative names - proxy_cert = ca.issue_cert(common_name=u"localhost") - server_cert = ca.issue_cert(u"localhost") + proxy_cert = ca.issue_cert(common_name="localhost") + server_cert = ca.issue_cert("localhost") with run_server_and_proxy_in_thread( "https", "localhost", tmpdir, ca, proxy_cert, server_cert @@ -128,24 +190,28 @@ def no_san_proxy(tmp_path_factory): @pytest.fixture -def no_localhost_san_server(tmp_path_factory): +def no_localhost_san_server( + tmp_path_factory: pytest.TempPathFactory, +) -> typing.Generator[ServerConfig]: tmpdir = tmp_path_factory.mktemp("certs") ca = trustme.CA() # non localhost common name - server_cert = ca.issue_cert(u"example.com") + server_cert = ca.issue_cert("example.com") with run_server_in_thread("https", "localhost", tmpdir, ca, server_cert) as cfg: yield cfg @pytest.fixture -def ipv4_san_proxy(tmp_path_factory): +def ipv4_san_proxy_with_server( + tmp_path_factory: pytest.TempPathFactory, +) -> typing.Generator[tuple[ServerConfig, ServerConfig]]: tmpdir = tmp_path_factory.mktemp("certs") ca = trustme.CA() # IP address in Subject Alternative Name - proxy_cert = ca.issue_cert(u"127.0.0.1") + proxy_cert = ca.issue_cert("127.0.0.1") - server_cert = ca.issue_cert(u"localhost") + server_cert = ca.issue_cert("localhost") with run_server_and_proxy_in_thread( "https", "127.0.0.1", tmpdir, ca, proxy_cert, server_cert @@ -154,13 +220,15 @@ def ipv4_san_proxy(tmp_path_factory): @pytest.fixture -def ipv6_san_proxy(tmp_path_factory): +def ipv6_san_proxy_with_server( + tmp_path_factory: pytest.TempPathFactory, +) -> typing.Generator[tuple[ServerConfig, ServerConfig]]: tmpdir = tmp_path_factory.mktemp("certs") ca = trustme.CA() # IP addresses in Subject Alternative Name - proxy_cert = ca.issue_cert(u"::1") + proxy_cert = ca.issue_cert("::1") - server_cert = ca.issue_cert(u"localhost") + server_cert = ca.issue_cert("localhost") with run_server_and_proxy_in_thread( "https", "::1", tmpdir, ca, proxy_cert, server_cert @@ -169,46 +237,52 @@ def ipv6_san_proxy(tmp_path_factory): @pytest.fixture -def ipv4_san_server(tmp_path_factory): +def ipv4_san_server( + tmp_path_factory: pytest.TempPathFactory, +) -> typing.Generator[ServerConfig]: tmpdir = tmp_path_factory.mktemp("certs") ca = trustme.CA() # IP address in Subject Alternative Name - server_cert = ca.issue_cert(u"127.0.0.1") + server_cert = ca.issue_cert("127.0.0.1") with run_server_in_thread("https", "127.0.0.1", tmpdir, ca, server_cert) as cfg: yield cfg @pytest.fixture -def ipv6_addr_server(tmp_path_factory): +def ipv6_san_server( + tmp_path_factory: pytest.TempPathFactory, +) -> typing.Generator[ServerConfig]: if not HAS_IPV6: pytest.skip("Only runs on IPv6 systems") tmpdir = tmp_path_factory.mktemp("certs") ca = trustme.CA() - # IP address in Common Name - server_cert = ca.issue_cert(common_name=u"::1") + # IP address in Subject Alternative Name + server_cert = ca.issue_cert("::1") with run_server_in_thread("https", "::1", tmpdir, ca, server_cert) as cfg: yield cfg @pytest.fixture -def ipv6_san_server(tmp_path_factory): +def ipv6_no_san_server( + tmp_path_factory: pytest.TempPathFactory, +) -> typing.Generator[ServerConfig]: if not HAS_IPV6: pytest.skip("Only runs on IPv6 systems") tmpdir = tmp_path_factory.mktemp("certs") ca = trustme.CA() - # IP address in Subject Alternative Name - server_cert = ca.issue_cert(u"::1") + # IP address in Common Name + server_cert = ca.issue_cert(common_name="::1") with run_server_in_thread("https", "::1", tmpdir, ca, server_cert) as cfg: yield cfg -@pytest.yield_fixture -def stub_timezone(request): +@pytest.fixture +def stub_timezone(request: pytest.FixtureRequest) -> typing.Generator[None]: """ A pytest fixture that runs the test with a stub timezone. """ @@ -217,20 +291,20 @@ def stub_timezone(request): @pytest.fixture(scope="session") -def supported_tls_versions(): +def supported_tls_versions() -> typing.AbstractSet[str | None]: # We have to create an actual TLS connection # to test if the TLS version is not disabled by # OpenSSL config. Ubuntu 20.04 specifically # disables TLSv1 and TLSv1.1. tls_versions = set() - _server = HTTPSDummyServerTestCase() - _server._start_server() - for _ssl_version_name in ( - "PROTOCOL_TLSv1", - "PROTOCOL_TLSv1_1", - "PROTOCOL_TLSv1_2", - "PROTOCOL_TLS", + _server = HTTPSHypercornDummyServerTestCase + _server.setup_class() + for _ssl_version_name, min_max_version in ( + ("PROTOCOL_TLSv1", ssl.TLSVersion.TLSv1), + ("PROTOCOL_TLSv1_1", ssl.TLSVersion.TLSv1_1), + ("PROTOCOL_TLSv1_2", ssl.TLSVersion.TLSv1_2), + ("PROTOCOL_TLS", None), ): _ssl_version = getattr(ssl, _ssl_version_name, 0) if _ssl_version == 0: @@ -238,43 +312,87 @@ def supported_tls_versions(): _sock = socket.create_connection((_server.host, _server.port)) try: _sock = ssl_.ssl_wrap_socket( - _sock, cert_reqs=ssl.CERT_NONE, ssl_version=_ssl_version + _sock, + ssl_context=ssl_.create_urllib3_context( + cert_reqs=ssl.CERT_NONE, + ssl_minimum_version=min_max_version, + ssl_maximum_version=min_max_version, + ), ) except ssl.SSLError: pass else: tls_versions.add(_sock.version()) _sock.close() - _server._stop_server() + _server.teardown_class() return tls_versions @pytest.fixture(scope="function") -def requires_tlsv1(supported_tls_versions): +def requires_tlsv1(supported_tls_versions: typing.AbstractSet[str]) -> None: """Test requires TLSv1 available""" if not hasattr(ssl, "PROTOCOL_TLSv1") or "TLSv1" not in supported_tls_versions: pytest.skip("Test requires TLSv1") @pytest.fixture(scope="function") -def requires_tlsv1_1(supported_tls_versions): +def requires_tlsv1_1(supported_tls_versions: typing.AbstractSet[str]) -> None: """Test requires TLSv1.1 available""" if not hasattr(ssl, "PROTOCOL_TLSv1_1") or "TLSv1.1" not in supported_tls_versions: pytest.skip("Test requires TLSv1.1") @pytest.fixture(scope="function") -def requires_tlsv1_2(supported_tls_versions): +def requires_tlsv1_2(supported_tls_versions: typing.AbstractSet[str]) -> None: """Test requires TLSv1.2 available""" if not hasattr(ssl, "PROTOCOL_TLSv1_2") or "TLSv1.2" not in supported_tls_versions: pytest.skip("Test requires TLSv1.2") @pytest.fixture(scope="function") -def requires_tlsv1_3(supported_tls_versions): +def requires_tlsv1_3(supported_tls_versions: typing.AbstractSet[str]) -> None: """Test requires TLSv1.3 available""" if ( not getattr(ssl, "HAS_TLSv1_3", False) or "TLSv1.3" not in supported_tls_versions ): pytest.skip("Test requires TLSv1.3") + + +class ErroringHTTPConnection: + def __init__(self, *args: typing.Any, **kwargs: typing.Any): + raise ValueError( + "HTTP/2 support currently only applies to HTTPS, don't use http_version for HTTP tests" + ) + + +@pytest.fixture(params=["h11", "h2"]) +def http_version(request: pytest.FixtureRequest) -> typing.Generator[str]: + orig_HTTPConnection: typing.Any = None + + if request.param == "h2": + urllib3.http2.inject_into_urllib3() + + from urllib3 import connection as urllib3_connection + from urllib3.connectionpool import HTTPConnectionPool + + orig_HTTPConnection = urllib3_connection.HTTPConnection + urllib3_connection.HTTPConnection = ErroringHTTPConnection # type: ignore[misc,assignment] + HTTPConnectionPool.ConnectionCls = ErroringHTTPConnection # type: ignore[assignment] + try: + yield request.param + finally: + if request.param == "h2": + urllib3_connection.HTTPConnection = orig_HTTPConnection # type: ignore[misc] + HTTPConnectionPool.ConnectionCls = orig_HTTPConnection + + urllib3.http2.extract_from_urllib3() + + +@pytest.fixture(autouse=True, scope="function") +def reset_http2_probe_cache() -> typing.Generator[None]: + # Always reset the HTTP/2 probe cache per test case. + try: + yield + finally: + http2_probe._reset() diff --git a/src/urllib3/contrib/_securetransport/__init__.py b/test/contrib/emscripten/__init__.py similarity index 100% rename from src/urllib3/contrib/_securetransport/__init__.py rename to test/contrib/emscripten/__init__.py diff --git a/test/contrib/emscripten/conftest.py b/test/contrib/emscripten/conftest.py new file mode 100644 index 0000000..0a4573e --- /dev/null +++ b/test/contrib/emscripten/conftest.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import contextlib +import os +import random +import textwrap +from collections.abc import Generator +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest + +from dummyserver.app import pyodide_testing_app +from dummyserver.hypercornserver import run_hypercorn_in_thread +from dummyserver.socketserver import DEFAULT_CERTS + +_coverage_count = 0 + + +def _get_coverage_filename(prefix: str) -> str: + global _coverage_count + _coverage_count += 1 + rand_part = "".join([random.choice("1234567890") for x in range(20)]) + return prefix + rand_part + f".{_coverage_count}" + + +@pytest.fixture(scope="module") +def testserver_http( + request: pytest.FixtureRequest, +) -> Generator[PyodideServerInfo]: + pyodide_dist_dir = Path(os.getcwd(), request.config.getoption("--dist-dir")) + pyodide_testing_app.config["pyodide_dist_dir"] = str(pyodide_dist_dir) + http_host = "localhost" + with contextlib.ExitStack() as stack: + http_port = stack.enter_context( + run_hypercorn_in_thread(http_host, None, pyodide_testing_app) + ) + https_port = stack.enter_context( + run_hypercorn_in_thread(http_host, DEFAULT_CERTS, pyodide_testing_app) + ) + + yield PyodideServerInfo( + http_host=http_host, + http_port=http_port, + https_port=https_port, + pyodide_dist_dir=pyodide_dist_dir, + ) + print("Server teardown") + + +@dataclass +class PyodideServerInfo: + http_port: int + https_port: int + http_host: str + pyodide_dist_dir: Path + + +@pytest.fixture() +def selenium_with_jspi_if_possible( + request: pytest.FixtureRequest, runtime: str, has_jspi: bool +) -> Generator[Any]: + if runtime.startswith("firefox") or not has_jspi: + fixture_name = "selenium" + with_jspi = False + else: + fixture_name = "selenium_jspi" + with_jspi = True + selenium_obj = request.getfixturevalue(fixture_name) + selenium_obj.with_jspi = with_jspi + yield selenium_obj + + +@pytest.fixture() +def selenium_coverage( + selenium_with_jspi_if_possible: Any, testserver_http: PyodideServerInfo +) -> Generator[Any]: + def _install_packages(self: Any) -> None: + if self.browser == "node": + # stop Node.js checking our https certificates + self.run_js('process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;') + # install urllib3 from our test server, rather than from existing package + result = self.run_js( + f'await pyodide.loadPackage("http://{testserver_http.http_host}:{testserver_http.http_port}/dist/urllib3.whl")' + ) + print("Installed package:", result) + self.run_js( + """ + await pyodide.loadPackage("coverage") + await pyodide.runPythonAsync(`import coverage +_coverage= coverage.Coverage(source_pkgs=['urllib3']) +_coverage.start() + ` + )""" + ) + + setattr( + selenium_with_jspi_if_possible, + "_install_packages", + _install_packages.__get__( + selenium_with_jspi_if_possible, selenium_with_jspi_if_possible.__class__ + ), + ) + + selenium_with_jspi_if_possible._install_packages() + yield selenium_with_jspi_if_possible + # on teardown, save _coverage output + coverage_out_binary = bytes( + selenium_with_jspi_if_possible.run_js( + """ +return await pyodide.runPythonAsync(` +_coverage.stop() +_coverage.save() +_coverage_datafile = open(".coverage","rb") +_coverage_outdata = _coverage_datafile.read() +# avoid polluting main namespace too much +import js as _coverage_js +# convert to js Array (as default conversion is TypedArray which does +# bad things in firefox) +_coverage_js.Array.from_(_coverage_outdata) +`) + """ + ) + ) + with open(f"{_get_coverage_filename('.coverage.emscripten.')}", "wb") as outfile: + outfile.write(coverage_out_binary) + + +class ServerRunnerInfo: + def __init__(self, host: str, port: int, selenium: Any, dist_dir: Path) -> None: + self.host = host + self.port = port + self.selenium = selenium + self.dist_dir = dist_dir + + def run_webworker(self, code: str) -> Any: + if isinstance(code, str) and code.startswith("\n"): + # we have a multiline string, fix indentation + code = textwrap.dedent(code) + + # add coverage collection to this code + coverage_init_code = textwrap.dedent( + """ + import coverage + _coverage= coverage.Coverage(source_pkgs=['urllib3']) + _coverage.start() + """ + ) + + coverage_end_code = textwrap.dedent( + """ + _coverage.stop() + _coverage.save() + _coverage_datafile = open(".coverage","rb") + _coverage_outdata = _coverage_datafile.read() + # avoid polluting main namespace too much + import js as _coverage_js + # convert to js Array (as default conversion is TypedArray which does + # bad things in firefox) + _coverage_js.Array.from_(_coverage_outdata) + """ + ) + + # the ordering of these code blocks is important - makes sure + # that the first thing that happens is our wheel is loaded + code = coverage_init_code + "\n" + code + "\n" + coverage_end_code + + if self.selenium.browser == "firefox": + # running in worker is SLOW on firefox + self.selenium.set_script_timeout(30) + if self.selenium.browser == "node": + worker_path = str(self.dist_dir / "webworker_dev.js") + self.selenium.run_js( + f"""const {{ + Worker, isMainThread, parentPort, workerData, + }} = require('node:worker_threads'); + globalThis.Worker= Worker; + process.chdir('{self.dist_dir}'); + """ + ) + else: + worker_path = f"https://{self.host}:{self.port}/pyodide/webworker_dev.js" + coverage_out_binary = bytes( + self.selenium.run_js( + f""" + let worker = new Worker('{worker_path}'); + let p = new Promise((res, rej) => {{ + worker.onmessageerror = e => rej(e); + worker.onerror = e => rej(e); + worker.onmessage = e => {{ + if (e.data.results) {{ + res(e.data.results); + }} else {{ + rej(e.data.error); + }} + }}; + worker.postMessage({{ python: {repr(code)} }}); + }}); + return await p; + """, + pyodide_checks=False, + ) + ) + with open( + f"{_get_coverage_filename('.coverage.emscripten.worker.')}", "wb" + ) as outfile: + outfile.write(coverage_out_binary) + + +# run pyodide on our test server instead of on the default +# pytest-pyodide one - this makes it so that +# we are at the same origin as web requests to server_host +@pytest.fixture() +def run_from_server( + selenium_coverage: Any, testserver_http: PyodideServerInfo +) -> Generator[ServerRunnerInfo]: + if selenium_coverage.browser != "node": + # on node, we don't need to be on the same origin + # so we can ignore all this + addr = f"https://{testserver_http.http_host}:{testserver_http.https_port}/pyodide/test.html" + selenium_coverage.goto(addr) + selenium_coverage.javascript_setup() + selenium_coverage.load_pyodide() + selenium_coverage.initialize_pyodide() + selenium_coverage.save_state() + selenium_coverage.restore_state() + selenium_coverage._install_packages() + dist_dir = testserver_http.pyodide_dist_dir + yield ServerRunnerInfo( + testserver_http.http_host, + testserver_http.https_port, + selenium_coverage, + dist_dir, + ) + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Generate tests with WebAssembly JavaScript Promise Integration both + enabled and disabled depending on browser/node.js support for features. + Also drops any test that requires a browser or web-workers in Node.js. + ). + """ + if "has_jspi" in metafunc.fixturenames: + can_run_with_jspi = False + can_run_without_jspi = False + # node only supports JSPI and doesn't support workers or + # webbrowser specific tests + if metafunc.config.getoption("--runtime").startswith("node"): + if ( + metafunc.definition.get_closest_marker("webworkers") is None + and metafunc.definition.get_closest_marker("in_webbrowser") is None + ): + can_run_with_jspi = True + if metafunc.definition.get_closest_marker("node_without_jspi"): + can_run_without_jspi = True + can_run_with_jspi = False + # firefox doesn't support JSPI + elif metafunc.config.getoption("--runtime").startswith("firefox"): + can_run_without_jspi = True + else: + # chrome supports JSPI on or off + can_run_without_jspi = True + can_run_with_jspi = True + + # if the function is marked to only run with or without jspi, + # then disable the alternative option + if metafunc.definition.get_closest_marker("with_jspi"): + can_run_without_jspi = False + elif metafunc.definition.get_closest_marker("without_jspi"): + can_run_with_jspi = False + + jspi_options = [] + if can_run_without_jspi: + jspi_options.append(False) + if can_run_with_jspi: + jspi_options.append(True) + metafunc.parametrize("has_jspi", jspi_options) diff --git a/test/contrib/emscripten/templates/pyodide-console.html b/test/contrib/emscripten/templates/pyodide-console.html new file mode 100644 index 0000000..5c94e50 --- /dev/null +++ b/test/contrib/emscripten/templates/pyodide-console.html @@ -0,0 +1,271 @@ + + + + + + + + + + + + + + + + +
+ + + diff --git a/test/contrib/emscripten/test_emscripten.py b/test/contrib/emscripten/test_emscripten.py new file mode 100644 index 0000000..9317a09 --- /dev/null +++ b/test/contrib/emscripten/test_emscripten.py @@ -0,0 +1,1191 @@ +from __future__ import annotations + +import sys +import typing + +import pytest + +from urllib3.fields import _TYPE_FIELD_VALUE_TUPLE + +from ...port_helpers import find_unused_port + +if sys.version_info < (3, 11): + # pyodide only works on 3.11+ + pytest.skip(allow_module_level=True) + +# only run these tests if pytest_pyodide is installed +# so we don't break non-emscripten pytest running +pytest_pyodide = pytest.importorskip("pytest_pyodide") + +from pytest_pyodide import run_in_pyodide # type: ignore[import-not-found] # noqa: E402 + +from .conftest import PyodideServerInfo, ServerRunnerInfo # noqa: E402 + +# make our ssl certificates work in chrome +pyodide_config = pytest_pyodide.config.get_global_config() +pyodide_config.set_flags( + "chrome", ["ignore-certificate-errors"] + pyodide_config.get_flags("chrome") +) + + +def test_index( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, has_jspi: bool +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int, has_jspi: bool) -> None: # type: ignore[no-untyped-def] + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + assert urllib3.contrib.emscripten.fetch.has_jspi() == has_jspi + conn = HTTPConnection(host, port) + url = f"http://{host}:{port}/" + conn.request("GET", url) + response = conn.getresponse() + # check methods of response + assert isinstance(response, BaseHTTPResponse) + assert response.url == url + response.url = "http://woo" + assert response.url == "http://woo" + assert response.connection == conn + assert response.retries is None + data1 = response.data + decoded1 = data1.decode("utf-8") + data2 = response.data # check that getting data twice works + decoded2 = data2.decode("utf-8") + assert decoded1 == decoded2 == "Dummy server!" + + pyodide_test( + selenium_coverage, + testserver_http.http_host, + testserver_http.http_port, + has_jspi, + ) + + +def test_pool_requests( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, has_jspi: bool +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int, https_port: int, has_jspi: bool) -> None: # type: ignore[no-untyped-def] + # first with PoolManager + import urllib3 + import urllib3.contrib.emscripten.fetch + + assert urllib3.contrib.emscripten.fetch.has_jspi() == has_jspi + + http = urllib3.PoolManager() + resp = http.request("GET", f"http://{host}:{port}/") + assert resp.data.decode("utf-8") == "Dummy server!" + + resp2 = http.request("GET", f"http://{host}:{port}/index") + assert resp2.data.decode("utf-8") == "Dummy server!" + + # should all have come from one pool + assert len(http.pools) == 1 + + resp3 = http.request("GET", f"https://{host}:{https_port}/") + assert resp3.data.decode("utf-8") == "Dummy server!" + + # one http pool + one https pool + assert len(http.pools) == 2 + + # now with ConnectionPool + # because block == True, this will fail if the connection isn't + # returned to the pool correctly after the first request + pool = urllib3.HTTPConnectionPool(host, port, maxsize=1, block=True) + resp3 = pool.urlopen("GET", "/index") + assert resp3.data.decode("utf-8") == "Dummy server!" + + resp4 = pool.urlopen("GET", "/") + assert resp4.data.decode("utf-8") == "Dummy server!" + + # now with manual release of connection + # first - connection should be released once all + # data is read + pool2 = urllib3.HTTPConnectionPool(host, port, maxsize=1, block=True) + + resp5 = pool2.urlopen("GET", "/index", preload_content=False) + assert pool2.pool is not None + # at this point, the connection should not be in the pool + assert pool2.pool.qsize() == 0 + assert resp5.data.decode("utf-8") == "Dummy server!" + # now we've read all the data, connection should be back to the pool + assert pool2.pool.qsize() == 1 + resp6 = pool2.urlopen("GET", "/index", preload_content=False) + assert pool2.pool.qsize() == 0 + # force it back to the pool + resp6.release_conn() + assert pool2.pool.qsize() == 1 + read_str = resp6.read() + # for consistency with urllib3, this still returns the correct data even though + # we are in theory not using the connection any more + assert read_str.decode("utf-8") == "Dummy server!" + + pyodide_test( + selenium_coverage, + testserver_http.http_host, + testserver_http.http_port, + testserver_http.https_port, + has_jspi, + ) + + +# wrong protocol / protocol error etc. should raise an exception of http.client.HTTPException +def test_wrong_protocol( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, has_jspi: bool +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + with pytest.raises(http.client.HTTPException): + conn.request("GET", f"http://{host}:{port}/") + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +# wrong protocol / protocol error etc. should raise an exception of http.client.HTTPException +def test_bad_method( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, has_jspi: bool +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + with pytest.raises(http.client.HTTPException): + conn.request("TRACE", f"http://{host}:{port}/") + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +# no connection - should raise +def test_no_response( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, has_jspi: bool +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + with pytest.raises(http.client.HTTPException): + conn.request("GET", f"http://{host}:{port}/") + _ = conn.getresponse() + + pyodide_test(selenium_coverage, testserver_http.http_host, find_unused_port()) + + +def test_404( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, has_jspi: bool +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/status?status=404 NOT FOUND") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + assert response.status == 404 + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +# setting timeout should show a warning to js console +# if we're on the ui thread, because XMLHttpRequest doesn't +# support timeout in async mode if globalThis == Window +@pytest.mark.without_jspi +def test_timeout_warning( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, +) -> None: + @run_in_pyodide() # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import js # type: ignore[import-not-found] + + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPConnection + + old_log = js.console.warn + log_msgs = [] + + def capture_log(*args): # type: ignore[no-untyped-def] + log_msgs.append(str(args)) + old_log(*args) + + js.console.warn = capture_log + + conn = HTTPConnection(host, port, timeout=1.0) + conn.request("GET", f"http://{host}:{port}/") + conn.getresponse() + js.console.warn = old_log + # should have shown timeout warning exactly once by now + assert len([x for x in log_msgs if x.find("Warning: Timeout") != -1]) == 1 + assert urllib3.contrib.emscripten.fetch._SHOWN_TIMEOUT_WARNING + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@pytest.mark.webworkers +def test_timeout_in_worker_non_streaming( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, + has_jspi: bool, +) -> None: + worker_code = f""" + from urllib3.exceptions import TimeoutError + from urllib3.connection import HTTPConnection + from pyodide.ffi import JsException + from http.client import HTTPException + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) + result=-1 + try: + conn.request("GET","/slow",preload_content = True) + _response = conn.getresponse() + result=-3 + except TimeoutError as e: + result=1 # we've got the correct exception + except HTTPException as e: + result=-3 + except BaseException as e: + result=-2 + raise BaseException(str(result)+":"+str(type(e))+str(e.args) ) + except JsException as e: + result=-4 + assert result == 1 +""" + run_from_server.run_webworker(worker_code) + + +@pytest.mark.webworkers +def test_timeout_in_worker_streaming( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, + has_jspi: bool, +) -> None: + worker_code = f""" + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.exceptions import TimeoutError + from urllib3.connection import HTTPConnection + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) + result=-1 + try: + conn.request("GET","/slow",preload_content=False) + _response = conn.getresponse() + result=-3 + except TimeoutError as e: + result=1 # we've got the correct exception + except BaseException as e: + result=-2 + assert result == 1 +""" + run_from_server.run_webworker(worker_code) + + +def test_index_https( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, has_jspi: bool +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPSConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPSConnection(host, port) + conn.request("GET", f"https://{host}:{port}/") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + data = response.data + assert data.decode("utf-8") == "Dummy server!" + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +@pytest.mark.without_jspi +def test_non_streaming_no_fallback_warning( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import js + + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPSConnection + from urllib3.response import BaseHTTPResponse + + log_msgs = [] + old_log = js.console.warn + + def capture_log(*args): # type: ignore[no-untyped-def] + log_msgs.append(str(args)) + old_log(*args) + + js.console.warn = capture_log + conn = HTTPSConnection(host, port) + conn.request("GET", f"https://{host}:{port}/", preload_content=True) + response = conn.getresponse() + js.console.warn = old_log + assert isinstance(response, BaseHTTPResponse) + data = response.data + assert data.decode("utf-8") == "Dummy server!" + # no console warnings because we didn't ask it to stream the response + # check no log messages + assert ( + len([x for x in log_msgs if x.find("Can't stream HTTP requests") != -1]) + == 0 + ) + assert not urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +@pytest.mark.without_jspi +def test_streaming_fallback_warning( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import js + + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPSConnection + from urllib3.response import BaseHTTPResponse + + # monkeypatch is_cross_origin_isolated so that it warns about that + # even if we're serving it so it is fine + urllib3.contrib.emscripten.fetch.is_cross_origin_isolated = lambda: False + + log_msgs = [] + old_log = js.console.warn + + def capture_log(*args): # type: ignore[no-untyped-def] + log_msgs.append(str(args)) + old_log(*args) + + js.console.warn = capture_log + + conn = HTTPSConnection(host, port) + conn.request("GET", f"https://{host}:{port}/", preload_content=False) + response = conn.getresponse() + js.console.warn = old_log + assert isinstance(response, BaseHTTPResponse) + data = response.data + assert data.decode("utf-8") == "Dummy server!" + # check that it has warned about falling back to non-streaming fetch exactly once + assert ( + len([x for x in log_msgs if x.find("Can't stream HTTP requests") != -1]) + == 1 + ) + assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +def test_specific_method( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, + has_jspi: bool, +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3 import HTTPSConnectionPool + + with HTTPSConnectionPool(host, port) as pool: + path = "/specific_method?method=POST" + response = pool.request("POST", path) + assert response.status == 200 + + response = pool.request("PUT", path) + assert response.status == 400 + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.https_port + ) + + +@pytest.mark.webworkers +def test_streaming_download( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, + has_jspi: bool, +) -> None: + # test streaming download, which must be in a webworker + # as you can't do it on main thread + + # this should return the 17mb big file, and + # should not log any warning about falling back + bigfile_url = ( + f"http://{testserver_http.http_host}:{testserver_http.http_port}/bigfile" + ) + worker_code = f""" + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + conn.request("GET", "{bigfile_url}",preload_content=False) + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==False + assert(urllib3.contrib.emscripten.fetch.has_jspi() == {has_jspi}) + data=response.data.decode('utf-8') + assert len(data) == 17825792 +""" + run_from_server.run_webworker(worker_code) + + +@pytest.mark.webworkers +def test_streaming_close( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, + has_jspi: bool, +) -> None: + # test streaming download, which must be in a webworker + # as you can't do it on main thread + + # this should return the 17mb big file, and + # should not log any warning about falling back + url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/" + worker_code = f""" + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + from io import RawIOBase + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + conn.request("GET", "{url}",preload_content=False) + response = conn.getresponse() + # check body is a RawIOBase stream and isn't seekable, writeable + body_internal = response._response.body + assert(isinstance(body_internal,RawIOBase)) + assert(body_internal.writable() is False) + assert(body_internal.seekable() is False) + assert(body_internal.readable() is True) + assert(urllib3.contrib.emscripten.fetch.has_jspi() == {has_jspi}) + + response.drain_conn() + x=response.read() + assert(not x) + response.close() + conn.close() + # try and make destructor be covered + # by killing everything + del response + del body_internal + del conn +""" + run_from_server.run_webworker(worker_code) + + +@pytest.mark.webworkers +def test_streaming_bad_url( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, + has_jspi: bool, +) -> None: + # this should cause an error + # because the protocol is bad + bad_url = f"hsffsdft://{testserver_http.http_host}:{testserver_http.http_port}/" + # this must be in a webworker + # as you can't do it on main thread + worker_code = f""" + import pytest + import http.client + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + with pytest.raises(http.client.HTTPException): + conn.request("GET", "{bad_url}",preload_content=False) +""" + run_from_server.run_webworker(worker_code) + + +@pytest.mark.webworkers +def test_streaming_bad_method( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, + has_jspi: bool, +) -> None: + # this should cause an error + # because the protocol is bad + bad_url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/" + # this must be in a webworker + # as you can't do it on main thread + worker_code = f""" + import pytest + import http.client + + import urllib3.contrib.emscripten.fetch + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + with pytest.raises(http.client.HTTPException): + # TRACE method should throw SecurityError in Javascript + conn.request("TRACE", "{bad_url}",preload_content=False) +""" + run_from_server.run_webworker(worker_code) + + +@pytest.mark.webworkers +@pytest.mark.without_jspi +def test_streaming_notready_warning( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + # test streaming download but don't wait for + # worker to be ready - should fallback to non-streaming + # and log a warning + file_url = f"http://{testserver_http.http_host}:{testserver_http.http_port}/" + worker_code = f""" + import js + import urllib3.contrib.emscripten.fetch + from urllib3.response import BaseHTTPResponse + from urllib3.connection import HTTPConnection + + urllib3.contrib.emscripten.fetch.streaming_ready = lambda :False + log_msgs=[] + old_log=js.console.warn + def capture_log(*args): + log_msgs.append(str(args)) + old_log(*args) + js.console.warn=capture_log + + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port}) + conn.request("GET", "{file_url}",preload_content=False) + js.console.warn=old_log + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + data=response.data.decode('utf-8') + #assert len([x for x in log_msgs if x.find("Can't stream HTTP requests")!=-1])==1 + #assert urllib3.contrib.emscripten.fetch._SHOWN_STREAMING_WARNING==True + """ + run_from_server.run_webworker(worker_code) + + +def test_post_receive_json( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, has_jspi: bool +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import json + + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + json_data = { + "Bears": "like", + "to": {"eat": "buns", "with": ["marmalade", "and custard"]}, + } + conn = HTTPConnection(host, port) + conn.request( + "POST", + f"http://{host}:{port}/echo_json", + body=json.dumps(json_data).encode("utf-8"), + headers={"Content-type": "application/json"}, + ) + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + data = response.json() + assert data == json_data + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +def test_upload( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3 import HTTPConnectionPool + + data = "I'm in ur multipart form-data, hazing a cheezburgr" + fields: dict[str, _TYPE_FIELD_VALUE_TUPLE] = { + "upload_param": "filefield", + "upload_filename": "lolcat.txt", + "filefield": ("lolcat.txt", data), + } + fields["upload_size"] = str(len(data)) + with HTTPConnectionPool(host, port) as pool: + r = pool.request("POST", "/upload", fields=fields) + assert r.status == 200 + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@pytest.mark.without_jspi +@pytest.mark.in_webbrowser +def test_streaming_not_ready_in_browser( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + # streaming ready should always be false + # if we're in the main browser thread + selenium_coverage.run_async( + """ + import urllib3.contrib.emscripten.fetch + result=await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + assert(result is False) + assert(urllib3.contrib.emscripten.fetch.streaming_ready() is None ) + """ + ) + + +def test_requests_with_micropip( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + @run_in_pyodide(packages=["micropip"]) # type: ignore[misc] + async def test_fn( + selenium_coverage: typing.Any, http_host: str, http_port: int, https_port: int + ) -> None: + import micropip # type: ignore[import-not-found] + + await micropip.install("requests") + import requests + + r = requests.get(f"http://{http_host}:{http_port}/") + assert r.status_code == 200 + assert r.text == "Dummy server!" + json_data = {"woo": "yay"} + # try posting some json with requests on https + r = requests.post(f"https://{http_host}:{https_port}/echo_json", json=json_data) + assert r.json() == json_data + + test_fn( + selenium_coverage, + testserver_http.http_host, + testserver_http.http_port, + testserver_http.https_port, + ) + + +def test_open_close( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from http.client import ResponseNotReady + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + # initially connection should be closed + assert conn.is_closed is True + # connection should have no response + with pytest.raises(ResponseNotReady): + response = conn.getresponse() + # now make the response + conn.request("GET", f"http://{host}:{port}/") + # we never connect to proxy (or if we do, browser handles it) + assert conn.has_connected_to_proxy is False + # now connection should be open + assert conn.is_closed is False + # and should have a response + response = conn.getresponse() + assert response is not None + conn.close() + # now it is closed + assert conn.is_closed is True + # closed connection shouldn't have any response + with pytest.raises(ResponseNotReady): + conn.getresponse() + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +# check that various ways that the worker may be broken +# throw exceptions nicely, by deliberately breaking things +# this is for coverage +@pytest.mark.webworkers +@pytest.mark.without_jspi +def test_break_worker_streaming( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + worker_code = f""" + import pytest + import urllib3.contrib.emscripten.fetch + import js + import http.client + + await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready() + from urllib3.exceptions import TimeoutError + from urllib3.connection import HTTPConnection + conn = HTTPConnection("{testserver_http.http_host}", {testserver_http.http_port},timeout=1.0) + # make the fetch worker return a bad response by: + # 1) Clearing the int buffer + # in the receive stream + with pytest.raises(http.client.HTTPException): + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + body_internal = response._response.body + assert(body_internal.int_buffer!=None) + body_internal.int_buffer=None + data=response.read() + # 2) Monkeypatch postMessage so that it just sets an + # exception status + old_pm= body_internal.worker.postMessage + with pytest.raises(http.client.HTTPException): + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + # make posted messages set an exception + body_internal = response._response.body + def set_exception(*args): + body_internal.worker.postMessage = old_pm + body_internal.int_buffer[1]=4 + body_internal.byte_buffer[0]=ord("W") + body_internal.byte_buffer[1]=ord("O") + body_internal.byte_buffer[2]=ord("O") + body_internal.byte_buffer[3]=ord("!") + body_internal.byte_buffer[4]=0 + js.Atomics.store(body_internal.int_buffer, 0, -4) + js.Atomics.notify(body_internal.int_buffer,0) + body_internal.worker.postMessage = set_exception + data=response.read() + # monkeypatch so it returns an unknown value for the magic number on initial fetch call + with pytest.raises(http.client.HTTPException): + # make posted messages set an exception + worker=urllib3.contrib.emscripten.fetch._fetcher.js_worker + def set_exception(self,*args): + array=js.Int32Array.new(args[0].buffer) + array[0]=-1234 + worker.postMessage=set_exception.__get__(worker,worker.__class__) + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + data=response.read() + urllib3.contrib.emscripten.fetch._fetcher.js_worker.postMessage=old_pm + # 3) Stopping the worker receiving any messages which should cause a timeout error + # in the receive stream + with pytest.raises(TimeoutError): + conn.request("GET","/",preload_content=False) + response = conn.getresponse() + # make posted messages not be send + body_internal = response._response.body + def ignore_message(*args): + pass + old_pm= body_internal.worker.postMessage + body_internal.worker.postMessage = ignore_message + data=response.read() + body_internal.worker.postMessage = old_pm +""" + run_from_server.run_webworker(worker_code) + + +def test_response_init_length( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import pytest + + import urllib3.exceptions + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + # head shouldn't have length + length = response._init_length("HEAD") + assert length == 0 + # multiple inconsistent lengths - should raise invalid header + with pytest.raises(urllib3.exceptions.InvalidHeader): + response.headers["Content-Length"] = "4,5,6" + length = response._init_length("GET") + # non-numeric length - should return None + response.headers["Content-Length"] = "anna" + length = response._init_length("GET") + assert length is None + # numeric length - should return it + response.headers["Content-Length"] = "54" + length = response._init_length("GET") + assert length == 54 + # negative length - should return None + response.headers["Content-Length"] = "-12" + length = response._init_length("GET") + assert length is None + # none -> None + del response.headers["Content-Length"] + length = response._init_length("GET") + assert length is None + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +def test_response_close_connection( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/") + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + response.close() + assert conn.is_closed + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +def test_read_chunked( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + conn.request("GET", f"http://{host}:{port}/mediumfile", preload_content=False) + response = conn.getresponse() + count = 0 + for x in response.read_chunked(512): + count += 1 + if count < 10: + assert len(x) == 512 + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +def test_retries( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import pytest + + import urllib3 + + pool = urllib3.HTTPConnectionPool( + host, + port, + maxsize=1, + block=True, + retries=urllib3.util.Retry(connect=5, read=5, redirect=5), + ) + + # monkeypatch connection class to count calls + old_request = urllib3.connection.HTTPConnection.request + count = 0 + + def count_calls(self, *args, **argv): # type: ignore[no-untyped-def] + nonlocal count + count += 1 + return old_request(self, *args, **argv) + + urllib3.connection.HTTPConnection.request = count_calls # type: ignore[method-assign] + with pytest.raises(urllib3.exceptions.MaxRetryError): + pool.urlopen("GET", "/") + # this should fail, but should have tried 6 times total + assert count == 6 + + pyodide_test(selenium_coverage, testserver_http.http_host, find_unused_port()) + + +def test_insecure_requests_warning( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int, https_port: int) -> None: # type: ignore[no-untyped-def] + import warnings + + import urllib3 + import urllib3.exceptions + + http = urllib3.PoolManager() + + with warnings.catch_warnings(record=True) as w: + http.request("GET", f"https://{host}:{https_port}") + assert len(w) == 0 + + pyodide_test( + selenium_coverage, + testserver_http.http_host, + testserver_http.http_port, + testserver_http.https_port, + ) + + +@pytest.mark.webworkers +def test_has_jspi_worker( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, + has_jspi: bool, +) -> None: + worker_code = f""" + import urllib3.contrib.emscripten.fetch + assert(urllib3.contrib.emscripten.fetch.has_jspi() == {has_jspi}) + """ + + run_from_server.run_webworker(worker_code) + + +def test_has_jspi( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo, has_jspi: bool +) -> None: + @run_in_pyodide + def pyodide_test(selenium, has_jspi): # type: ignore[no-untyped-def] + import urllib3.contrib.emscripten.fetch + + assert urllib3.contrib.emscripten.fetch.has_jspi() == has_jspi + + pyodide_test(selenium_coverage, has_jspi) + + +@pytest.mark.with_jspi +def test_timeout_jspi( + selenium_coverage: typing.Any, + testserver_http: PyodideServerInfo, + run_from_server: ServerRunnerInfo, +) -> None: + @run_in_pyodide + def pyodide_test(selenium, host, port): # type: ignore[no-untyped-def] + import pytest + + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPConnection + from urllib3.exceptions import TimeoutError + + conn = HTTPConnection(host, port, timeout=0.1) + assert urllib3.contrib.emscripten.fetch.has_jspi() is True + with pytest.raises(TimeoutError): + conn.request("GET", "/slow") + conn.getresponse() + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@pytest.mark.with_jspi +def test_streaming_jspi( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + bigfile_url = ( + f"http://{testserver_http.http_host}:{testserver_http.http_port}/dripfeed" + ) + + @run_in_pyodide + def pyodide_test(selenium, host, port, bigfile_url): # type: ignore[no-untyped-def] + import time + + from urllib3.connection import HTTPConnection + from urllib3.response import BaseHTTPResponse + + conn = HTTPConnection(host, port) + start_time = time.time() + conn.request("GET", bigfile_url, preload_content=False) + response = conn.getresponse() + assert isinstance(response, BaseHTTPResponse) + # first data should be received before the timeout + # on the server + first_data = response.read(32768) + assert time.time() - start_time < 2 + all_data = first_data + response.read() + # make sure that the timeout on server side really happened + # by checking that it took greater than the timeout + assert time.time() - start_time > 2 + assert len(all_data.decode("utf-8")) == 17825792 + + pyodide_test( + selenium_coverage, + testserver_http.http_host, + testserver_http.http_port, + bigfile_url, + ) + + +@pytest.mark.node_without_jspi +def test_non_jspi_fail_in_node( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + if selenium_coverage.browser != "node": + pytest.skip("node only test") + + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + url = f"http://{host}:{port}/" + # check streaming and non-streaming requests both fail + with pytest.raises(http.client.HTTPException): + conn.request("GET", url) + conn.getresponse() + with pytest.raises(http.client.HTTPException): + conn.request("GET", url, preload_content=False) + conn.getresponse() + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@pytest.mark.with_jspi +def test_jspi_fetch_error( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import http.client + + import pytest + + from urllib3.connection import HTTPConnection + + conn = HTTPConnection(host, port) + url = f"sdfsdfsffhttp://{host}:{port}/" + with pytest.raises(http.client.HTTPException): + conn.request("GET", url) + conn.getresponse() + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@pytest.mark.with_jspi +def test_jspi_readstream_errors( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + import io + from http.client import HTTPException + + import pytest + + import urllib3.contrib.emscripten.fetch + from urllib3.connection import HTTPConnection + from urllib3.exceptions import TimeoutError + + conn = HTTPConnection(host, port) + url = f"http://{host}:{port}/" + conn.request("GET", url, preload_content=False) + response = conn.getresponse() + assert isinstance(response._response.body, io.RawIOBase) # type: ignore[attr-defined] + old_run_sync = urllib3.contrib.emscripten.fetch._run_sync_with_timeout + with pytest.raises(TimeoutError): + + def raise_timeout(*args, **argv): # type: ignore[no-untyped-def] + raise urllib3.contrib.emscripten.fetch._TimeoutError() + + urllib3.contrib.emscripten.fetch._run_sync_with_timeout = raise_timeout + response.read() + urllib3.contrib.emscripten.fetch._run_sync_with_timeout = old_run_sync + conn.request("GET", url, preload_content=False) + response = conn.getresponse() + with pytest.raises(HTTPException): + + def raise_error(*args, **argv): # type: ignore[no-untyped-def] + raise urllib3.contrib.emscripten.fetch._RequestError() + + urllib3.contrib.emscripten.fetch._run_sync_with_timeout = raise_error + response.read() + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) + + +@pytest.mark.with_jspi +def test_has_jspi_exception( + selenium_coverage: typing.Any, testserver_http: PyodideServerInfo +) -> None: + @run_in_pyodide # type: ignore[misc] + def pyodide_test(selenium_coverage, host: str, port: int) -> None: # type: ignore[no-untyped-def] + from unittest.mock import patch + + import pyodide.ffi # type: ignore[import-not-found] + + if hasattr(pyodide.ffi, "can_run_sync"): + + @patch("pyodide.ffi.can_run_sync") + def should_return_false(func): # type: ignore[no-untyped-def] + func.return_value = (20, False) + func.side_effect = ImportError() + from urllib3.contrib.emscripten.fetch import has_jspi + + assert has_jspi() is False + + else: + from unittest.mock import patch + + @patch("pyodide_js._module") + def should_return_false(func): # type: ignore[no-untyped-def] + from urllib3.contrib.emscripten.fetch import has_jspi + + assert has_jspi() is False + + should_return_false() + + pyodide_test( + selenium_coverage, testserver_http.http_host, testserver_http.http_port + ) diff --git a/test/contrib/test_pyopenssl.py b/test/contrib/test_pyopenssl.py index 40b7a34..eaca77b 100644 --- a/test/contrib/test_pyopenssl.py +++ b/test/contrib/test_pyopenssl.py @@ -1,28 +1,32 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations + import os +from unittest import mock -import mock import pytest try: from cryptography import x509 - from OpenSSL.crypto import FILETYPE_PEM, load_certificate + from OpenSSL.crypto import ( # type: ignore[import-untyped] + FILETYPE_PEM, + load_certificate, + ) from urllib3.contrib.pyopenssl import _dnsname_to_stdlib, get_subj_alt_name except ImportError: pass -def setup_module(): +def setup_module() -> None: try: from urllib3.contrib.pyopenssl import inject_into_urllib3 inject_into_urllib3() except ImportError as e: - pytest.skip("Could not import PyOpenSSL: %r" % e) + pytest.skip(f"Could not import PyOpenSSL: {e!r}") -def teardown_module(): +def teardown_module() -> None: try: from urllib3.contrib.pyopenssl import extract_from_urllib3 @@ -31,13 +35,11 @@ def teardown_module(): pass +from ..test_ssl import TestSSL # noqa: E402, F401 from ..test_util import TestUtilSSL # noqa: E402, F401 from ..with_dummyserver.test_https import ( # noqa: E402, F401 - TestHTTPS, TestHTTPS_IPV4SAN, - TestHTTPS_IPv6Addr, TestHTTPS_IPV6SAN, - TestHTTPS_NoSAN, TestHTTPS_TLSv1, TestHTTPS_TLSv1_1, TestHTTPS_TLSv1_2, @@ -47,51 +49,53 @@ def teardown_module(): TestClientCerts, TestSNI, TestSocketClosing, - TestSSL, +) +from ..with_dummyserver.test_socketlevel import ( # noqa: E402, F401 + TestSSL as TestSocketSSL, ) -class TestPyOpenSSLHelpers(object): +class TestPyOpenSSLHelpers: """ Tests for PyOpenSSL helper functions. """ - def test_dnsname_to_stdlib_simple(self): + def test_dnsname_to_stdlib_simple(self) -> None: """ We can convert a dnsname to a native string when the domain is simple. """ - name = u"उदाहरण.परीक" + name = "उदाहरण.परीक" expected_result = "xn--p1b6ci4b4b3a.xn--11b5bs8d" assert _dnsname_to_stdlib(name) == expected_result - def test_dnsname_to_stdlib_leading_period(self): + def test_dnsname_to_stdlib_leading_period(self) -> None: """ If there is a . in front of the domain name we correctly encode it. """ - name = u".उदाहरण.परीक" + name = ".उदाहरण.परीक" expected_result = ".xn--p1b6ci4b4b3a.xn--11b5bs8d" assert _dnsname_to_stdlib(name) == expected_result - def test_dnsname_to_stdlib_leading_splat(self): + def test_dnsname_to_stdlib_leading_splat(self) -> None: """ If there's a wildcard character in the front of the string we handle it appropriately. """ - name = u"*.उदाहरण.परीक" + name = "*.उदाहरण.परीक" expected_result = "*.xn--p1b6ci4b4b3a.xn--11b5bs8d" assert _dnsname_to_stdlib(name) == expected_result @mock.patch("urllib3.contrib.pyopenssl.log.warning") - def test_get_subj_alt_name(self, mock_warning): + def test_get_subj_alt_name(self, mock_warning: mock.MagicMock) -> None: """ If a certificate has two subject alternative names, cryptography raises an x509.DuplicateExtension exception. """ path = os.path.join(os.path.dirname(__file__), "duplicate_san.pem") - with open(path, "r") as fp: + with open(path) as fp: cert = load_certificate(FILETYPE_PEM, fp.read()) assert get_subj_alt_name(cert) == [] diff --git a/test/contrib/test_pyopenssl_dependencies.py b/test/contrib/test_pyopenssl_dependencies.py index d1498e9..d182727 100644 --- a/test/contrib/test_pyopenssl_dependencies.py +++ b/test/contrib/test_pyopenssl_dependencies.py @@ -1,6 +1,8 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations + +from unittest.mock import Mock, patch + import pytest -from mock import Mock, patch try: from urllib3.contrib.pyopenssl import extract_from_urllib3, inject_into_urllib3 @@ -8,16 +10,16 @@ pass -def setup_module(): +def setup_module() -> None: try: from urllib3.contrib.pyopenssl import inject_into_urllib3 inject_into_urllib3() except ImportError as e: - pytest.skip("Could not import PyOpenSSL: %r" % e) + pytest.skip(f"Could not import PyOpenSSL: {e!r}") -def teardown_module(): +def teardown_module() -> None: try: from urllib3.contrib.pyopenssl import extract_from_urllib3 @@ -26,12 +28,12 @@ def teardown_module(): pass -class TestPyOpenSSLInjection(object): +class TestPyOpenSSLInjection: """ Tests for error handling in pyopenssl's 'inject_into urllib3' """ - def test_inject_validate_fail_cryptography(self): + def test_inject_validate_fail_cryptography(self) -> None: """ Injection should not be supported if cryptography is too old. """ @@ -46,7 +48,7 @@ def test_inject_validate_fail_cryptography(self): # clean up so that subsequent tests are unaffected. extract_from_urllib3() - def test_inject_validate_fail_pyopenssl(self): + def test_inject_validate_fail_pyopenssl(self) -> None: """ Injection should not be supported if pyOpenSSL is too old. """ diff --git a/test/contrib/test_securetransport.py b/test/contrib/test_securetransport.py deleted file mode 100644 index 14a4090..0000000 --- a/test/contrib/test_securetransport.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -import base64 -import contextlib -import socket -import ssl - -import pytest - -try: - from urllib3.contrib.securetransport import WrappedSocket -except ImportError: - pass - - -def setup_module(): - try: - from urllib3.contrib.securetransport import inject_into_urllib3 - - inject_into_urllib3() - except ImportError as e: - pytest.skip("Could not import SecureTransport: %r" % e) - - -def teardown_module(): - try: - from urllib3.contrib.securetransport import extract_from_urllib3 - - extract_from_urllib3() - except ImportError: - pass - - -from ..test_util import TestUtilSSL # noqa: E402, F401 - -# SecureTransport does not support TLSv1.3 -# https://github.com/urllib3/urllib3/issues/1674 -from ..with_dummyserver.test_https import ( # noqa: E402, F401 - TestHTTPS, - TestHTTPS_TLSv1, - TestHTTPS_TLSv1_1, - TestHTTPS_TLSv1_2, -) -from ..with_dummyserver.test_socketlevel import ( # noqa: E402, F401 - TestClientCerts, - TestSNI, - TestSocketClosing, - TestSSL, -) - - -def test_no_crash_with_empty_trust_bundle(): - with contextlib.closing(socket.socket()) as s: - ws = WrappedSocket(s) - with pytest.raises(ssl.SSLError): - ws._custom_validate(True, b"") - - -def test_no_crash_with_invalid_trust_bundle(): - invalid_cert = base64.b64encode(b"invalid-cert") - cert_bundle = ( - b"-----BEGIN CERTIFICATE-----\n" + invalid_cert + b"\n-----END CERTIFICATE-----" - ) - - with contextlib.closing(socket.socket()) as s: - ws = WrappedSocket(s) - with pytest.raises(ssl.SSLError): - ws._custom_validate(True, cert_bundle) diff --git a/test/contrib/test_socks.py b/test/contrib/test_socks.py index 1966513..083854d 100644 --- a/test/contrib/test_socks.py +++ b/test/contrib/test_socks.py @@ -1,10 +1,17 @@ +from __future__ import annotations + import socket import threading +import typing +from socket import getaddrinfo as real_getaddrinfo +from socket import timeout as SocketTimeout from test import SHORT_TIMEOUT +from unittest.mock import Mock, patch import pytest +import socks as py_socks # type: ignore[import-not-found] -from dummyserver.server import DEFAULT_CA, DEFAULT_CERTS +from dummyserver.socketserver import DEFAULT_CA, DEFAULT_CERTS from dummyserver.testcase import IPV4SocketDummyServerTestCase from urllib3.contrib import socks from urllib3.exceptions import ConnectTimeoutError, NewConnectionError @@ -16,8 +23,8 @@ HAS_SSL = True except ImportError: - ssl = None - better_ssl = None + ssl = None # type: ignore[assignment] + better_ssl = None # type: ignore[assignment] HAS_SSL = False @@ -28,7 +35,7 @@ SOCKS_VERSION_SOCKS5 = b"\x05" -def _get_free_port(host): +def _get_free_port(host: str) -> int: """ Gets a free port by opening a socket, binding it, checking the assigned port, and then closing it. @@ -37,10 +44,10 @@ def _get_free_port(host): s.bind((host, 0)) port = s.getsockname()[1] s.close() - return port + return port # type: ignore[no-any-return] -def _read_exactly(sock, amt): +def _read_exactly(sock: socket.socket, amt: int) -> bytes: """ Read *exactly* ``amt`` bytes from the socket ``sock``. """ @@ -54,7 +61,7 @@ def _read_exactly(sock, amt): return data -def _read_until(sock, char): +def _read_until(sock: socket.socket, char: bytes) -> bytes: """ Read from the socket until the character is received. """ @@ -68,7 +75,7 @@ def _read_until(sock, char): return b"".join(chunks) -def _address_from_socket(sock): +def _address_from_socket(sock: socket.socket) -> bytes | str: """ Returns the address from the SOCKS socket """ @@ -84,10 +91,43 @@ def _address_from_socket(sock): addr_len = ord(sock.recv(1)) return _read_exactly(sock, addr_len) else: - raise RuntimeError("Unexpected addr type: %r" % addr_type) - - -def handle_socks5_negotiation(sock, negotiate, username=None, password=None): + raise RuntimeError(f"Unexpected addr type: {addr_type!r}") + + +def _set_up_fake_getaddrinfo(monkeypatch: pytest.MonkeyPatch) -> None: + # Work around https://github.com/urllib3/urllib3/pull/2034 + # Nothing prevents localhost to point to two different IPs. For example, in the + # Ubuntu set up by GitHub Actions, localhost points both to 127.0.0.1 and ::1. + # + # In case of failure, PySocks will try the same request on both IPs, but our + # handle_socks[45]_negotiation functions don't handle retries, which leads either to + # a deadlock or a timeout in case of a failure on the first address. + # + # However, some tests need to exercise failure. We don't want retries there, but + # can't affect PySocks retries via its API. Instead, we monkeypatch PySocks so that + # it only sees a single address, which effectively disables retries. + def fake_getaddrinfo(addr: str, port: int, family: int, socket_type: int) -> list[ + tuple[ + socket.AddressFamily, + socket.SocketKind, + int, + str, + tuple[str, int] | tuple[str, int, int, int], + ] + ]: + gai_list = real_getaddrinfo(addr, port, family, socket_type) + gai_list = [gai for gai in gai_list if gai[0] == socket.AF_INET] + return gai_list[:1] + + monkeypatch.setattr(py_socks.socket, "getaddrinfo", fake_getaddrinfo) + + +def handle_socks5_negotiation( + sock: socket.socket, + negotiate: bool, + username: bytes | None = None, + password: bytes | None = None, +) -> typing.Generator[tuple[bytes | str, int], bool, None]: """ Handle the SOCKS5 handshake. @@ -117,7 +157,6 @@ def handle_socks5_negotiation(sock, negotiate, username=None, password=None): else: sock.sendall(b"\x01\x01") sock.close() - yield False return else: assert SOCKS_NEGOTIATION_NONE in methods @@ -129,8 +168,8 @@ def handle_socks5_negotiation(sock, negotiate, username=None, password=None): command = sock.recv(1) reserved = sock.recv(1) addr = _address_from_socket(sock) - port = _read_exactly(sock, 2) - port = (ord(port[0:1]) << 8) + (ord(port[1:2])) + port_raw = _read_exactly(sock, 2) + port = (ord(port_raw[0:1]) << 8) + (ord(port_raw[1:2])) # Check some basic stuff. assert received_version == SOCKS_VERSION_SOCKS5 @@ -148,10 +187,11 @@ def handle_socks5_negotiation(sock, negotiate, username=None, password=None): response = SOCKS_VERSION_SOCKS5 + b"\x01\00" sock.sendall(response) - yield True # Avoid StopIteration exceptions getting fired. -def handle_socks4_negotiation(sock, username=None): +def handle_socks4_negotiation( + sock: socket.socket, username: bytes | None = None +) -> typing.Generator[tuple[bytes | str, int], bool, None]: """ Handle the SOCKS4 handshake. @@ -160,16 +200,17 @@ def handle_socks4_negotiation(sock, username=None): """ received_version = sock.recv(1) command = sock.recv(1) - port = _read_exactly(sock, 2) - port = (ord(port[0:1]) << 8) + (ord(port[1:2])) - addr = _read_exactly(sock, 4) + port_raw = _read_exactly(sock, 2) + port = (ord(port_raw[0:1]) << 8) + (ord(port_raw[1:2])) + addr_raw = _read_exactly(sock, 4) provided_username = _read_until(sock, b"\x00")[:-1] # Strip trailing null. - if addr == b"\x00\x00\x00\x01": + addr: bytes | str + if addr_raw == b"\x00\x00\x00\x01": # Magic string: means DNS name. addr = _read_until(sock, b"\x00")[:-1] # Strip trailing null. else: - addr = socket.inet_ntoa(addr) + addr = socket.inet_ntoa(addr_raw) # Check some basic stuff. assert received_version == SOCKS_VERSION_SOCKS4 @@ -178,7 +219,6 @@ def handle_socks4_negotiation(sock, username=None): if username is not None and username != provided_username: sock.sendall(b"\x00\x5d\x00\x00\x00\x00\x00\x00") sock.close() - yield False return # Yield the address port tuple. @@ -190,14 +230,12 @@ def handle_socks4_negotiation(sock, username=None): response = b"\x00\x5b\x00\x00\x00\x00\x00\x00" sock.sendall(response) - yield True # Avoid StopIteration exceptions getting fired. -class TestSOCKSProxyManager(object): - def test_invalid_socks_version_is_valueerror(self): - with pytest.raises(ValueError) as e: +class TestSOCKSProxyManager: + def test_invalid_socks_version_is_valueerror(self) -> None: + with pytest.raises(ValueError, match="Unable to determine SOCKS version"): socks.SOCKSProxyManager(proxy_url="http://example.org") - assert "Unable to determine SOCKS version" in e.value.args[0] class TestSocks5Proxy(IPV4SocketDummyServerTestCase): @@ -205,8 +243,8 @@ class TestSocks5Proxy(IPV4SocketDummyServerTestCase): Test the SOCKS proxy in SOCKS5 mode. """ - def test_basic_request(self): - def request_handler(listener): + def test_basic_request(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks5_negotiation(sock, negotiate=False) @@ -214,7 +252,8 @@ def request_handler(listener): assert addr == "16.17.18.19" assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) while True: buf = sock.recv(65535) @@ -230,7 +269,7 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks5://%s:%s" % (self.host, self.port) + proxy_url = f"socks5://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: response = pm.request("GET", "http://16.17.18.19") @@ -238,8 +277,8 @@ def request_handler(listener): assert response.data == b"" assert response.headers["Server"] == "SocksTestServer" - def test_local_dns(self): - def request_handler(listener): + def test_local_dns(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks5_negotiation(sock, negotiate=False) @@ -247,7 +286,8 @@ def request_handler(listener): assert addr in ["127.0.0.1", "::1"] assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) while True: buf = sock.recv(65535) @@ -263,7 +303,7 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks5://%s:%s" % (self.host, self.port) + proxy_url = f"socks5://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: response = pm.request("GET", "http://localhost") @@ -271,8 +311,8 @@ def request_handler(listener): assert response.data == b"" assert response.headers["Server"] == "SocksTestServer" - def test_correct_header_line(self): - def request_handler(listener): + def test_correct_header_line(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks5_negotiation(sock, negotiate=False) @@ -280,7 +320,8 @@ def request_handler(listener): assert addr == b"example.com" assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) buf = b"" while True: @@ -300,19 +341,19 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks5h://%s:%s" % (self.host, self.port) + proxy_url = f"socks5h://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: response = pm.request("GET", "http://example.com") assert response.status == 200 - def test_connection_timeouts(self): + def test_connection_timeouts(self) -> None: event = threading.Event() - def request_handler(listener): + def request_handler(listener: socket.socket) -> None: event.wait() self._start_server(request_handler) - proxy_url = "socks5h://%s:%s" % (self.host, self.port) + proxy_url = f"socks5h://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: with pytest.raises(ConnectTimeoutError): pm.request( @@ -320,42 +361,52 @@ def request_handler(listener): ) event.set() - def test_connection_failure(self): + @patch("socks.create_connection") + def test_socket_timeout(self, create_connection: Mock) -> None: + create_connection.side_effect = SocketTimeout() + proxy_url = f"socks5h://{self.host}:{self.port}" + with socks.SOCKSProxyManager(proxy_url) as pm: + with pytest.raises(ConnectTimeoutError, match="timed out"): + pm.request("GET", "http://example.com", retries=False) + + def test_connection_failure(self) -> None: event = threading.Event() - def request_handler(listener): + def request_handler(listener: socket.socket) -> None: listener.close() event.set() self._start_server(request_handler) - proxy_url = "socks5h://%s:%s" % (self.host, self.port) + proxy_url = f"socks5h://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: event.wait() with pytest.raises(NewConnectionError): pm.request("GET", "http://example.com", retries=False) - def test_proxy_rejection(self): + def test_proxy_rejection(self, monkeypatch: pytest.MonkeyPatch) -> None: + _set_up_fake_getaddrinfo(monkeypatch) evt = threading.Event() - def request_handler(listener): + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks5_negotiation(sock, negotiate=False) addr, port = next(handler) - handler.send(False) + with pytest.raises(StopIteration): + handler.send(False) evt.wait() sock.close() self._start_server(request_handler) - proxy_url = "socks5h://%s:%s" % (self.host, self.port) + proxy_url = f"socks5h://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: with pytest.raises(NewConnectionError): pm.request("GET", "http://example.com", retries=False) evt.set() - def test_socks_with_password(self): - def request_handler(listener): + def test_socks_with_password(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks5_negotiation( @@ -365,7 +416,8 @@ def request_handler(listener): assert addr == "16.17.18.19" assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) while True: buf = sock.recv(65535) @@ -381,7 +433,7 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks5://%s:%s" % (self.host, self.port) + proxy_url = f"socks5://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url, username="user", password="pass") as pm: response = pm.request("GET", "http://16.17.18.19") @@ -389,13 +441,13 @@ def request_handler(listener): assert response.data == b"" assert response.headers["Server"] == "SocksTestServer" - def test_socks_with_auth_in_url(self): + def test_socks_with_auth_in_url(self) -> None: """ Test when we have auth info in url, i.e. socks5://user:pass@host:port and no username/password as params """ - def request_handler(listener): + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks5_negotiation( @@ -405,7 +457,8 @@ def request_handler(listener): assert addr == "16.17.18.19" assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) while True: buf = sock.recv(65535) @@ -421,7 +474,7 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks5://user:pass@%s:%s" % (self.host, self.port) + proxy_url = f"socks5://user:pass@{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: response = pm.request("GET", "http://16.17.18.19") @@ -429,28 +482,32 @@ def request_handler(listener): assert response.data == b"" assert response.headers["Server"] == "SocksTestServer" - def test_socks_with_invalid_password(self): - def request_handler(listener): + def test_socks_with_invalid_password(self, monkeypatch: pytest.MonkeyPatch) -> None: + _set_up_fake_getaddrinfo(monkeypatch) + + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks5_negotiation( sock, negotiate=True, username=b"user", password=b"pass" ) - next(handler) + with pytest.raises(StopIteration): + next(handler) self._start_server(request_handler) - proxy_url = "socks5h://%s:%s" % (self.host, self.port) + proxy_url = f"socks5h://{self.host}:{self.port}" with socks.SOCKSProxyManager( proxy_url, username="user", password="badpass" ) as pm: - with pytest.raises(NewConnectionError) as e: + with pytest.raises( + NewConnectionError, match="SOCKS5 authentication failed" + ): pm.request("GET", "http://example.com", retries=False) - assert "SOCKS5 authentication failed" in str(e.value) - def test_source_address_works(self): + def test_source_address_works(self) -> None: expected_port = _get_free_port(self.host) - def request_handler(listener): + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] assert sock.getpeername()[0] == "127.0.0.1" assert sock.getpeername()[1] == expected_port @@ -460,7 +517,8 @@ def request_handler(listener): assert addr == "16.17.18.19" assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) while True: buf = sock.recv(65535) @@ -476,7 +534,7 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks5://%s:%s" % (self.host, self.port) + proxy_url = f"socks5://{self.host}:{self.port}" with socks.SOCKSProxyManager( proxy_url, source_address=("127.0.0.1", expected_port) ) as pm: @@ -492,8 +550,8 @@ class TestSOCKS4Proxy(IPV4SocketDummyServerTestCase): negotiation is done the two cases behave identically. """ - def test_basic_request(self): - def request_handler(listener): + def test_basic_request(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks4_negotiation(sock) @@ -501,7 +559,8 @@ def request_handler(listener): assert addr == "16.17.18.19" assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) while True: buf = sock.recv(65535) @@ -517,7 +576,7 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks4://%s:%s" % (self.host, self.port) + proxy_url = f"socks4://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: response = pm.request("GET", "http://16.17.18.19") @@ -525,8 +584,8 @@ def request_handler(listener): assert response.headers["Server"] == "SocksTestServer" assert response.data == b"" - def test_local_dns(self): - def request_handler(listener): + def test_local_dns(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks4_negotiation(sock) @@ -534,7 +593,8 @@ def request_handler(listener): assert addr == "127.0.0.1" assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) while True: buf = sock.recv(65535) @@ -550,7 +610,7 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks4://%s:%s" % (self.host, self.port) + proxy_url = f"socks4://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: response = pm.request("GET", "http://localhost") @@ -558,8 +618,8 @@ def request_handler(listener): assert response.headers["Server"] == "SocksTestServer" assert response.data == b"" - def test_correct_header_line(self): - def request_handler(listener): + def test_correct_header_line(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks4_negotiation(sock) @@ -567,7 +627,8 @@ def request_handler(listener): assert addr == b"example.com" assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) buf = b"" while True: @@ -587,33 +648,35 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks4a://%s:%s" % (self.host, self.port) + proxy_url = f"socks4a://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: response = pm.request("GET", "http://example.com") assert response.status == 200 - def test_proxy_rejection(self): + def test_proxy_rejection(self, monkeypatch: pytest.MonkeyPatch) -> None: + _set_up_fake_getaddrinfo(monkeypatch) evt = threading.Event() - def request_handler(listener): + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks4_negotiation(sock) addr, port = next(handler) - handler.send(False) + with pytest.raises(StopIteration): + handler.send(False) evt.wait() sock.close() self._start_server(request_handler) - proxy_url = "socks4a://%s:%s" % (self.host, self.port) + proxy_url = f"socks4a://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url) as pm: with pytest.raises(NewConnectionError): pm.request("GET", "http://example.com", retries=False) evt.set() - def test_socks4_with_username(self): - def request_handler(listener): + def test_socks4_with_username(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks4_negotiation(sock, username=b"user") @@ -621,7 +684,8 @@ def request_handler(listener): assert addr == "16.17.18.19" assert port == 80 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) while True: buf = sock.recv(65535) @@ -637,7 +701,7 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks4://%s:%s" % (self.host, self.port) + proxy_url = f"socks4://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url, username="user") as pm: response = pm.request("GET", "http://16.17.18.19") @@ -645,19 +709,18 @@ def request_handler(listener): assert response.data == b"" assert response.headers["Server"] == "SocksTestServer" - def test_socks_with_invalid_username(self): - def request_handler(listener): + def test_socks_with_invalid_username(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks4_negotiation(sock, username=b"user") - next(handler) + next(handler, None) self._start_server(request_handler) - proxy_url = "socks4a://%s:%s" % (self.host, self.port) + proxy_url = f"socks4a://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url, username="baduser") as pm: - with pytest.raises(NewConnectionError) as e: + with pytest.raises(NewConnectionError, match="different user-ids"): pm.request("GET", "http://example.com", retries=False) - assert "different user-ids" in str(e.value) class TestSOCKSWithTLS(IPV4SocketDummyServerTestCase): @@ -666,8 +729,8 @@ class TestSOCKSWithTLS(IPV4SocketDummyServerTestCase): """ @pytest.mark.skipif(not HAS_SSL, reason="No TLS available") - def test_basic_request(self): - def request_handler(listener): + def test_basic_request(self) -> None: + def request_handler(listener: socket.socket) -> None: sock = listener.accept()[0] handler = handle_socks5_negotiation(sock, negotiate=False) @@ -675,10 +738,11 @@ def request_handler(listener): assert addr == b"localhost" assert port == 443 - handler.send(True) + with pytest.raises(StopIteration): + handler.send(True) # Wrap in TLS - context = better_ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context = better_ssl.SSLContext(ssl.PROTOCOL_SSLv23) # type: ignore[misc] context.load_cert_chain(DEFAULT_CERTS["certfile"], DEFAULT_CERTS["keyfile"]) tls = context.wrap_socket(sock, server_side=True) buf = b"" @@ -700,7 +764,7 @@ def request_handler(listener): sock.close() self._start_server(request_handler) - proxy_url = "socks5h://%s:%s" % (self.host, self.port) + proxy_url = f"socks5h://{self.host}:{self.port}" with socks.SOCKSProxyManager(proxy_url, ca_certs=DEFAULT_CA) as pm: response = pm.request("GET", "https://localhost") diff --git a/test/port_helpers.py b/test/port_helpers.py index ae18cca..e8c9484 100644 --- a/test/port_helpers.py +++ b/test/port_helpers.py @@ -1,6 +1,8 @@ -# These helpers are copied from test_support.py in the Python 2.7 standard +# These helpers are copied from test/support/socket_helper.py in the Python 3.9 standard # library test suite. +from __future__ import annotations + import socket # Don't use "localhost", since resolving it uses the DNS under recent @@ -9,7 +11,10 @@ HOSTv6 = "::1" -def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): +def find_unused_port( + family: socket.AddressFamily = socket.AF_INET, + socktype: socket.SocketKind = socket.SOCK_STREAM, +) -> int: """Returns an unused port that should be suitable for binding. This is achieved by creating a temporary socket with the same family and type as the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to @@ -36,7 +41,7 @@ def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): the SO_REUSEADDR socket option having different semantics on Windows versus Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind, listen and then accept connections on identical host/ports. An EADDRINUSE - socket.error will be raised at some point (depending on the platform and + OSError will be raised at some point (depending on the platform and the order bind and listen were called on each socket). However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE @@ -63,15 +68,15 @@ def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): other process when we close and delete our temporary socket but before our calling code has a chance to bind the returned port. We can deal with this issue if/when we come across it.""" - tempsock = socket.socket(family, socktype) - port = bind_port(tempsock) - tempsock.close() + + with socket.socket(family, socktype) as tempsock: + port = bind_port(tempsock) del tempsock return port -def bind_port(sock, host=HOST): - """Bind the socket to a free port and return the port number. Relies on +def bind_port(sock: socket.socket, host: str = HOST) -> int: + """Bind the socket to a free port and return the port number. Relies on ephemeral ports in order to ensure we are using an unbound port. This is important as many tests may be running simultaneously, especially in a buildbot environment. This method raises an exception if the sock.family @@ -84,6 +89,7 @@ def bind_port(sock, host=HOST): on Windows), it will be set on the socket. This will prevent anyone else from bind()'ing to our host/port for the duration of the test. """ + if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM: if hasattr(socket, "SO_REUSEADDR"): if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1: @@ -92,14 +98,21 @@ def bind_port(sock, host=HOST): "socket option on TCP/IP sockets!" ) if hasattr(socket, "SO_REUSEPORT"): - if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1: - raise ValueError( - "tests should never set the SO_REUSEPORT " - "socket option on TCP/IP sockets!" - ) + try: + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1: + raise ValueError( + "tests should never set the SO_REUSEPORT " + "socket option on TCP/IP sockets!" + ) + except OSError: + # Python's socket module was compiled using modern headers + # thus defining SO_REUSEPORT but this process is running + # under an older kernel that does not support SO_REUSEPORT. + pass if hasattr(socket, "SO_EXCLUSIVEADDRUSE"): sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) sock.bind((host, 0)) port = sock.getsockname()[1] + assert isinstance(port, int) return port diff --git a/test/socketpair_helper.py b/test/socketpair_helper.py deleted file mode 100644 index 7ddb600..0000000 --- a/test/socketpair_helper.py +++ /dev/null @@ -1,63 +0,0 @@ -import socket - -# Figuring out what errors could come out of a socket. There are three -# different situations. Python 3 post-PEP3151 will define and use -# BlockingIOError and InterruptedError from sockets. For Python pre-PEP3151 -# both OSError and socket.error can be raised except on Windows where -# WindowsError can also be raised. We want to catch all of these possible -# exceptions so we catch WindowsError if it's defined. -try: - _CONNECT_ERROR = (BlockingIOError, InterruptedError) -except NameError: - try: - _CONNECT_ERROR = (WindowsError, OSError, socket.error) # noqa: F821 - except NameError: - _CONNECT_ERROR = (OSError, socket.error) - -if hasattr(socket, "socketpair"): - # Since Python 3.5, socket.socketpair() is now also available on Windows - socketpair = socket.socketpair -else: - # Replacement for socket.socketpair() - def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): - """A socket pair usable as a self-pipe, for Windows. - - Origin: https://gist.github.com/4325783, by Geert Jansen. - Public domain. - """ - if family == socket.AF_INET: - host = "127.0.0.1" - elif family == socket.AF_INET6: - host = "::1" - else: - raise ValueError( - "Only AF_INET and AF_INET6 socket address families are supported" - ) - if type != socket.SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with setblocking(0) - # that prevents us from having to create a thread. - lsock = socket.socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen(1) - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket.socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except _CONNECT_ERROR: - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except Exception: - csock.close() - raise - finally: - lsock.close() - return (ssock, csock) diff --git a/test/test_collections.py b/test/test_collections.py index 4b8624c..ae896e2 100644 --- a/test/test_collections.py +++ b/test/test_collections.py @@ -1,23 +1,23 @@ +from __future__ import annotations + +import typing + import pytest from urllib3._collections import HTTPHeaderDict from urllib3._collections import RecentlyUsedContainer as Container -from urllib3.exceptions import InvalidHeader -from urllib3.packages import six -xrange = six.moves.xrange +class TestLRUContainer: + def test_maxsize(self) -> None: + d: Container[int, str] = Container(5) -class TestLRUContainer(object): - def test_maxsize(self): - d = Container(5) - - for i in xrange(5): + for i in range(5): d[i] = str(i) assert len(d) == 5 - for i in xrange(5): + for i in range(5): assert d[i] == str(i) d[i + 1] = str(i + 1) @@ -26,49 +26,54 @@ def test_maxsize(self): assert 0 not in d assert (i + 1) in d - def test_expire(self): - d = Container(5) + def test_maxsize_0(self) -> None: + d: Container[int, int] = Container(0) + d[1] = 1 + assert len(d) == 0 + + def test_expire(self) -> None: + d: Container[int, str] = Container(5) - for i in xrange(5): + for i in range(5): d[i] = str(i) - for i in xrange(5): + for i in range(5): d.get(0) # Add one more entry d[5] = "5" # Check state - assert list(d.keys()) == [2, 3, 4, 0, 5] + assert list(d._container.keys()) == [2, 3, 4, 0, 5] - def test_same_key(self): - d = Container(5) + def test_same_key(self) -> None: + d: Container[str, int] = Container(5) - for i in xrange(10): + for i in range(10): d["foo"] = i - assert list(d.keys()) == ["foo"] + assert list(d._container.keys()) == ["foo"] assert len(d) == 1 - def test_access_ordering(self): - d = Container(5) + def test_access_ordering(self) -> None: + d: Container[int, bool] = Container(5) - for i in xrange(10): + for i in range(10): d[i] = True # Keys should be ordered by access time - assert list(d.keys()) == [5, 6, 7, 8, 9] + assert list(d._container.keys()) == [5, 6, 7, 8, 9] new_order = [7, 8, 6, 9, 5] for k in new_order: d[k] - assert list(d.keys()) == new_order + assert list(d._container.keys()) == new_order - def test_delete(self): - d = Container(5) + def test_delete(self) -> None: + d: Container[int, bool] = Container(5) - for i in xrange(5): + for i in range(5): d[i] = True del d[0] @@ -79,10 +84,10 @@ def test_delete(self): d.pop(1, None) - def test_get(self): - d = Container(5) + def test_get(self) -> None: + d: Container[int, bool | int] = Container(5) - for i in xrange(5): + for i in range(5): d[i] = True r = d.get(4) @@ -97,21 +102,21 @@ def test_get(self): with pytest.raises(KeyError): d[5] - def test_disposal(self): - evicted_items = [] + def test_disposal(self) -> None: + evicted_items: list[int] = [] - def dispose_func(arg): + def dispose_func(arg: int) -> None: # Save the evicted datum for inspection evicted_items.append(arg) - d = Container(5, dispose_func=dispose_func) - for i in xrange(5): + d: Container[int, int] = Container(5, dispose_func=dispose_func) + for i in range(5): d[i] = i - assert list(d.keys()) == list(xrange(5)) + assert list(d._container.keys()) == list(range(5)) assert evicted_items == [] # Nothing disposed d[5] = 5 - assert list(d.keys()) == list(xrange(1, 6)) + assert list(d._container.keys()) == list(range(1, 6)) assert evicted_items == [0] del d[1] @@ -120,49 +125,57 @@ def dispose_func(arg): d.clear() assert evicted_items == [0, 1, 2, 3, 4, 5] - def test_iter(self): - d = Container() + def test_iter(self) -> None: + d: Container[str, str] = Container() with pytest.raises(NotImplementedError): d.__iter__() -class NonMappingHeaderContainer(object): - def __init__(self, **kwargs): +class NonMappingHeaderContainer: + def __init__(self, **kwargs: str) -> None: self._data = {} self._data.update(kwargs) - def keys(self): - return self._data.keys() + def keys(self) -> typing.Iterator[str]: + return iter(self._data) - def __getitem__(self, key): + def __getitem__(self, key: str) -> str: return self._data[key] @pytest.fixture() -def d(): +def d() -> HTTPHeaderDict: header_dict = HTTPHeaderDict(Cookie="foo") header_dict.add("cookie", "bar") return header_dict -class TestHTTPHeaderDict(object): - def test_create_from_kwargs(self): - h = HTTPHeaderDict(ab=1, cd=2, ef=3, gh=4) +class TestHTTPHeaderDict: + def test_create_from_kwargs(self) -> None: + h = HTTPHeaderDict(ab="1", cd="2", ef="3", gh="4") assert len(h) == 4 assert "ab" in h - def test_create_from_dict(self): - h = HTTPHeaderDict(dict(ab=1, cd=2, ef=3, gh=4)) + def test_setdefault(self) -> None: + h = HTTPHeaderDict(a="1") + assert h.setdefault("A", "3") == "1" + assert h.setdefault("b", "2") == "2" + assert h.setdefault("c") == "" + assert h["c"] == "" + assert h["b"] == "2" + + def test_create_from_dict(self) -> None: + h = HTTPHeaderDict(dict(ab="1", cd="2", ef="3", gh="4")) assert len(h) == 4 assert "ab" in h - def test_create_from_iterator(self): + def test_create_from_iterator(self) -> None: teststr = "urllib3ontherocks" h = HTTPHeaderDict((c, c * 5) for c in teststr) assert len(h) == len(set(teststr)) - def test_create_from_list(self): + def test_create_from_list(self) -> None: headers = [ ("ab", "A"), ("cd", "B"), @@ -178,7 +191,7 @@ def test_create_from_list(self): assert clist[0] == "C" assert clist[-1] == "E" - def test_create_from_headerdict(self): + def test_create_from_headerdict(self) -> None: headers = [ ("ab", "A"), ("cd", "B"), @@ -197,54 +210,87 @@ def test_create_from_headerdict(self): assert h is not org assert h == org - def test_setitem(self, d): + def test_setitem(self, d: HTTPHeaderDict) -> None: d["Cookie"] = "foo" - assert d["cookie"] == "foo" + # The bytes value gets converted to str. The API is typed for str only, + # but the implementation continues supports bytes. + d[b"Cookie"] = "bar" # type: ignore[index] + assert d["cookie"] == "bar" d["cookie"] = "with, comma" assert d.getlist("cookie") == ["with, comma"] - def test_update(self, d): + def test_update(self, d: HTTPHeaderDict) -> None: d.update(dict(Cookie="foo")) assert d["cookie"] == "foo" d.update(dict(cookie="with, comma")) assert d.getlist("cookie") == ["with, comma"] - def test_delitem(self, d): + def test_delitem(self, d: HTTPHeaderDict) -> None: del d["cookie"] assert "cookie" not in d assert "COOKIE" not in d - def test_add_well_known_multiheader(self, d): + def test_add_well_known_multiheader(self, d: HTTPHeaderDict) -> None: d.add("COOKIE", "asdf") assert d.getlist("cookie") == ["foo", "bar", "asdf"] assert d["cookie"] == "foo, bar, asdf" - def test_add_comma_separated_multiheader(self, d): + def test_add_comma_separated_multiheader(self, d: HTTPHeaderDict) -> None: d.add("bar", "foo") - d.add("BAR", "bar") + # The bytes value gets converted to str. The API is typed for str only, + # but the implementation continues supports bytes. + d.add(b"BAR", "bar") # type: ignore[arg-type] d.add("Bar", "asdf") assert d.getlist("bar") == ["foo", "bar", "asdf"] assert d["bar"] == "foo, bar, asdf" - def test_extend_from_list(self, d): + def test_extend_from_list(self, d: HTTPHeaderDict) -> None: d.extend([("set-cookie", "100"), ("set-cookie", "200"), ("set-cookie", "300")]) assert d["set-cookie"] == "100, 200, 300" - def test_extend_from_dict(self, d): + def test_extend_from_dict(self, d: HTTPHeaderDict) -> None: d.extend(dict(cookie="asdf"), b="100") assert d["cookie"] == "foo, bar, asdf" assert d["b"] == "100" d.add("cookie", "with, comma") assert d.getlist("cookie") == ["foo", "bar", "asdf", "with, comma"] - def test_extend_from_container(self, d): + def test_extend_from_container(self, d: HTTPHeaderDict) -> None: h = NonMappingHeaderContainer(Cookie="foo", e="foofoo") d.extend(h) assert d["cookie"] == "foo, bar, foo" assert d["e"] == "foofoo" assert len(d) == 2 - def test_extend_from_headerdict(self, d): + def test_header_repeat(self, d: HTTPHeaderDict) -> None: + d["other-header"] = "hello" + d.add("other-header", "world") + + assert list(d.items()) == [ + ("Cookie", "foo"), + ("Cookie", "bar"), + ("other-header", "hello"), + ("other-header", "world"), + ] + + d.add("other-header", "!", combine=True) + expected_results = [ + ("Cookie", "foo"), + ("Cookie", "bar"), + ("other-header", "hello"), + ("other-header", "world, !"), + ] + + assert list(d.items()) == expected_results + # make sure the values persist over copies + assert list(d.copy().items()) == expected_results + + other_dict = HTTPHeaderDict() + # we also need for extensions to properly maintain results + other_dict.extend(d) + assert list(other_dict.items()) == expected_results + + def test_extend_from_headerdict(self, d: HTTPHeaderDict) -> None: h = HTTPHeaderDict(Cookie="foo", e="foofoo") d.extend(h) assert d["cookie"] == "foo, bar, foo" @@ -252,41 +298,48 @@ def test_extend_from_headerdict(self, d): assert len(d) == 2 @pytest.mark.parametrize("args", [(1, 2), (1, 2, 3, 4, 5)]) - def test_extend_with_wrong_number_of_args_is_typeerror(self, d, args): - with pytest.raises(TypeError) as err: - d.extend(*args) - assert "extend() takes at most 1 positional arguments" in err.value.args[0] - - def test_copy(self, d): + def test_extend_with_wrong_number_of_args_is_typeerror( + self, d: HTTPHeaderDict, args: tuple[int, ...] + ) -> None: + with pytest.raises( + TypeError, match=r"extend\(\) takes at most 1 positional arguments" + ): + d.extend(*args) # type: ignore[arg-type] + + def test_copy(self, d: HTTPHeaderDict) -> None: h = d.copy() assert d is not h assert d == h - def test_getlist(self, d): + def test_getlist(self, d: HTTPHeaderDict) -> None: assert d.getlist("cookie") == ["foo", "bar"] assert d.getlist("Cookie") == ["foo", "bar"] assert d.getlist("b") == [] d.add("b", "asdf") assert d.getlist("b") == ["asdf"] - def test_getlist_after_copy(self, d): + def test_getlist_after_copy(self, d: HTTPHeaderDict) -> None: assert d.getlist("cookie") == HTTPHeaderDict(d).getlist("cookie") - def test_equal(self, d): + def test_equal(self, d: HTTPHeaderDict) -> None: b = HTTPHeaderDict(cookie="foo, bar") c = NonMappingHeaderContainer(cookie="foo, bar") + e = [("cookie", "foo, bar")] assert d == b assert d == c + assert d == e assert d != 2 - def test_not_equal(self, d): + def test_not_equal(self, d: HTTPHeaderDict) -> None: b = HTTPHeaderDict(cookie="foo, bar") c = NonMappingHeaderContainer(cookie="foo, bar") + e = [("cookie", "foo, bar")] assert not (d != b) assert not (d != c) + assert not (d != e) assert d != 2 - def test_pop(self, d): + def test_pop(self, d: HTTPHeaderDict) -> None: key = "Cookie" a = d[key] b = d.pop(key) @@ -297,86 +350,105 @@ def test_pop(self, d): dummy = object() assert dummy is d.pop(key, dummy) - def test_discard(self, d): + def test_discard(self, d: HTTPHeaderDict) -> None: d.discard("cookie") assert "cookie" not in d d.discard("cookie") - def test_len(self, d): + def test_len(self, d: HTTPHeaderDict) -> None: assert len(d) == 1 d.add("cookie", "bla") d.add("asdf", "foo") # len determined by unique fieldnames assert len(d) == 2 - def test_repr(self, d): + def test_repr(self, d: HTTPHeaderDict) -> None: rep = "HTTPHeaderDict({'Cookie': 'foo, bar'})" assert repr(d) == rep - def test_items(self, d): + def test_items(self, d: HTTPHeaderDict) -> None: items = d.items() assert len(items) == 2 - assert items[0][0] == "Cookie" - assert items[0][1] == "foo" - assert items[1][0] == "Cookie" - assert items[1][1] == "bar" - - def test_dict_conversion(self, d): + assert list(items) == [ + ("Cookie", "foo"), + ("Cookie", "bar"), + ] + assert ("Cookie", "foo") in items + assert ("Cookie", "bar") in items + assert ("X-Some-Header", "foo") not in items + assert ("Cookie", "not_present") not in items + assert ("Cookie", 1) not in items # type: ignore[comparison-overlap] + assert "Cookie" not in items # type: ignore[comparison-overlap] + + def test_dict_conversion(self, d: HTTPHeaderDict) -> None: # Also tested in connectionpool, needs to preserve case hdict = { "Content-Length": "0", "Content-type": "text/plain", - "Server": "TornadoServer/1.2.3", + "Server": "Hypercorn/1.2.3", } h = dict(HTTPHeaderDict(hdict).items()) assert hdict == h assert hdict == dict(HTTPHeaderDict(hdict)) - def test_string_enforcement(self, d): + def test_string_enforcement(self, d: HTTPHeaderDict) -> None: # This currently throws AttributeError on key.lower(), should # probably be something nicer with pytest.raises(Exception): - d[3] = 5 + d[3] = "5" # type: ignore[index] with pytest.raises(Exception): - d.add(3, 4) + d.add(3, "4") # type: ignore[arg-type] with pytest.raises(Exception): - del d[3] + del d[3] # type: ignore[arg-type] with pytest.raises(Exception): - HTTPHeaderDict({3: 3}) - - @pytest.mark.skipif( - not six.PY2, reason="python3 has a different internal header implementation" - ) - def test_from_httplib_py2(self): - msg = """ -Server: nginx -Content-Type: text/html; charset=windows-1251 -Connection: keep-alive -X-Some-Multiline: asdf - asdf\t -\t asdf -Set-Cookie: bb_lastvisit=1348253375; expires=Sat, 21-Sep-2013 18:49:35 GMT; path=/ -Set-Cookie: bb_lastactivity=0; expires=Sat, 21-Sep-2013 18:49:35 GMT; path=/ -www-authenticate: asdf -www-authenticate: bla - -""" - buffer = six.moves.StringIO(msg.lstrip().replace("\n", "\r\n")) - msg = six.moves.http_client.HTTPMessage(buffer) - d = HTTPHeaderDict.from_httplib(msg) - assert d["server"] == "nginx" - cookies = d.getlist("set-cookie") - assert len(cookies) == 2 - assert cookies[0].startswith("bb_lastvisit") - assert cookies[1].startswith("bb_lastactivity") - assert d["x-some-multiline"] == "asdf asdf asdf" - assert d["www-authenticate"] == "asdf, bla" - assert d.getlist("www-authenticate") == ["asdf", "bla"] - with_invalid_multiline = """\tthis-is-not-a-header: but it has a pretend value -Authorization: Bearer 123 - -""" - buffer = six.moves.StringIO(with_invalid_multiline.replace("\n", "\r\n")) - msg = six.moves.http_client.HTTPMessage(buffer) - with pytest.raises(InvalidHeader): - HTTPHeaderDict.from_httplib(msg) + HTTPHeaderDict({3: 3}) # type: ignore[arg-type] + + def test_dunder_contains(self, d: HTTPHeaderDict) -> None: + """ + Test: + + HTTPHeaderDict.__contains__ returns True + - for matched string objects + - for case-similar string objects + HTTPHeaderDict.__contains__ returns False + - for non-similar strings + - for non-strings, even if they are keys + in the underlying datastructure + """ + assert "cookie" in d + assert "CoOkIe" in d + assert "Not a cookie" not in d + + marker = object() + d._container[marker] = ["some", "strings"] # type: ignore[index] + assert marker not in d + assert marker in d._container + + def test_union(self, d: HTTPHeaderDict) -> None: + to_merge = {"Cookie": "tim-tam"} + result = d | to_merge + assert result == HTTPHeaderDict({"Cookie": "foo, bar, tim-tam"}) + assert to_merge == {"Cookie": "tim-tam"} + assert d == HTTPHeaderDict({"Cookie": "foo, bar"}) + + def test_union_rhs(self, d: HTTPHeaderDict) -> None: + to_merge = {"Cookie": "tim-tam"} + result = to_merge | d + assert result == HTTPHeaderDict({"Cookie": "tim-tam, foo, bar"}) + assert to_merge == {"Cookie": "tim-tam"} + assert d == HTTPHeaderDict({"Cookie": "foo, bar"}) + + def test_inplace_union(self, d: HTTPHeaderDict) -> None: + to_merge = {"Cookie": "tim-tam"} + d |= to_merge + assert d == HTTPHeaderDict({"Cookie": "foo, bar, tim-tam"}) + + def test_union_with_unsupported_type(self, d: HTTPHeaderDict) -> None: + with pytest.raises(TypeError, match="unsupported operand type.*'int'"): + d | 42 + with pytest.raises(TypeError, match="unsupported operand type.*'float'"): + 3.14 | d + + def test_inplace_union_with_unsupported_type(self, d: HTTPHeaderDict) -> None: + with pytest.raises(TypeError, match="unsupported operand type.*'NoneType'"): + d |= None diff --git a/test/test_compatibility.py b/test/test_compatibility.py index 58a9ab5..95dea64 100644 --- a/test/test_compatibility.py +++ b/test/test_compatibility.py @@ -1,37 +1,19 @@ -import warnings +from __future__ import annotations + +import http.cookiejar +import urllib +from unittest import mock import pytest -from urllib3.connection import HTTPConnection -from urllib3.packages.six.moves import http_cookiejar, urllib +import urllib3.http2 from urllib3.response import HTTPResponse -class TestVersionCompatibility(object): - def test_connection_strict(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - - # strict=True is deprecated in Py33+ - HTTPConnection("localhost", 12345, strict=True) - - if w: - pytest.fail( - "HTTPConnection raised warning on strict=True: %r" % w[0].message - ) - - def test_connection_source_address(self): - try: - # source_address does not exist in Py26- - HTTPConnection("localhost", 12345, source_address="127.0.0.1") - except TypeError as e: - pytest.fail("HTTPConnection raised TypeError on source_address: %r" % e) - - -class TestCookiejar(object): - def test_extract(self): +class TestCookiejar: + def test_extract(self) -> None: request = urllib.request.Request("http://google.com") - cookiejar = http_cookiejar.CookieJar() + cookiejar = http.cookiejar.CookieJar() response = HTTPResponse() cookies = [ @@ -40,5 +22,27 @@ def test_extract(self): ] for c in cookies: response.headers.add("set-cookie", c) - cookiejar.extract_cookies(response, request) + cookiejar.extract_cookies(response, request) # type: ignore[arg-type] assert len(cookiejar) == len(cookies) + + +class TestInitialization: + @mock.patch("urllib3.http2.version") + def test_h2_version_check(self, mock_version: mock.MagicMock) -> None: + try: + mock_version.return_value = "4.1.0" + urllib3.http2.inject_into_urllib3() + + mock_version.return_value = "3.9.9" + with pytest.raises( + ImportError, match="urllib3 v2 supports h2 version 4.x.x.*" + ): + urllib3.http2.inject_into_urllib3() + + mock_version.return_value = "5.0.0" + with pytest.raises( + ImportError, match="urllib3 v2 supports h2 version 4.x.x.*" + ): + urllib3.http2.inject_into_urllib3() + finally: + urllib3.http2.extract_from_urllib3() diff --git a/test/test_connection.py b/test/test_connection.py index ac629ce..a4bd873 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -1,41 +1,58 @@ +from __future__ import annotations + import datetime +import socket +import typing +from http.client import ResponseNotReady +from unittest import mock -import mock import pytest -from urllib3.connection import RECENT_DATE, CertificateError, _match_hostname +from urllib3.connection import ( # type: ignore[attr-defined] + RECENT_DATE, + CertificateError, + HTTPConnection, + HTTPSConnection, + _match_hostname, + _url_from_connection, + _wrap_proxy_error, +) +from urllib3.exceptions import HTTPError, ProxyError, SSLError +from urllib3.util import ssl_ +from urllib3.util.request import SKIP_HEADER +from urllib3.util.ssl_match_hostname import ( + CertificateError as ImplementationCertificateError, +) +from urllib3.util.ssl_match_hostname import _dnsname_match, match_hostname + +if typing.TYPE_CHECKING: + from urllib3.util.ssl_ import _TYPE_PEER_CERT_RET_DICT -class TestConnection(object): +class TestConnection: """ Tests in this suite should not make any network requests or connections. """ - def test_match_hostname_no_cert(self): + def test_match_hostname_no_cert(self) -> None: cert = None asserted_hostname = "foo" with pytest.raises(ValueError): _match_hostname(cert, asserted_hostname) - def test_match_hostname_empty_cert(self): - cert = {} + def test_match_hostname_empty_cert(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = {} asserted_hostname = "foo" with pytest.raises(ValueError): _match_hostname(cert, asserted_hostname) - def test_match_hostname_match(self): - cert = {"subjectAltName": [("DNS", "foo")]} + def test_match_hostname_match(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = {"subjectAltName": (("DNS", "foo"),)} asserted_hostname = "foo" _match_hostname(cert, asserted_hostname) - def test_match_hostname_ipaddress_none(self): - cert = {"subjectAltName": [("DNS", "foo")]} - asserted_hostname = "foo" - with mock.patch("urllib3.util.ssl_match_hostname.ipaddress", None): - assert _match_hostname(cert, asserted_hostname) is None - - def test_match_hostname_mismatch(self): - cert = {"subjectAltName": [("DNS", "foo")]} + def test_match_hostname_mismatch(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = {"subjectAltName": (("DNS", "foo"),)} asserted_hostname = "bar" try: with mock.patch("urllib3.connection.log.warning") as mock_log: @@ -45,48 +62,72 @@ def test_match_hostname_mismatch(self): mock_log.assert_called_once_with( "Certificate did not match expected hostname: %s. Certificate: %s", "bar", - {"subjectAltName": [("DNS", "foo")]}, + {"subjectAltName": (("DNS", "foo"),)}, ) - assert e._peer_cert == cert + assert e._peer_cert == cert # type: ignore[attr-defined] - def test_match_hostname_ip_address_ipv6(self): - cert = {"subjectAltName": (("IP Address", "1:2::2:1"),)} - asserted_hostname = "1:2::2:2" + def test_match_hostname_no_dns(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = {"subjectAltName": (("DNS", ""),)} + asserted_hostname = "bar" try: with mock.patch("urllib3.connection.log.warning") as mock_log: _match_hostname(cert, asserted_hostname) except CertificateError as e: - assert "hostname '1:2::2:2' doesn't match '1:2::2:1'" in str(e) + assert "hostname 'bar' doesn't match ''" in str(e) mock_log.assert_called_once_with( "Certificate did not match expected hostname: %s. Certificate: %s", - "1:2::2:2", - {"subjectAltName": (("IP Address", "1:2::2:1"),)}, + "bar", + {"subjectAltName": (("DNS", ""),)}, ) - assert e._peer_cert == cert + assert e._peer_cert == cert # type: ignore[attr-defined] - def test_match_hostname_dns_with_brackets_doesnt_match(self): - cert = { - "subjectAltName": ( - ("DNS", "localhost"), - ("IP Address", "localhost"), - ) + def test_match_hostname_startwith_wildcard(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = {"subjectAltName": (("DNS", "*"),)} + asserted_hostname = "foo" + _match_hostname(cert, asserted_hostname) + + def test_match_hostname_dnsname(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = { + "subjectAltName": (("DNS", "xn--p1b6ci4b4b3a*.xn--11b5bs8d"),) } - asserted_hostname = "[localhost]" - with pytest.raises(CertificateError) as e: - _match_hostname(cert, asserted_hostname) - assert ( - "hostname '[localhost]' doesn't match either of 'localhost', 'localhost'" - in str(e.value) - ) + asserted_hostname = "xn--p1b6ci4b4b3a*.xn--11b5bs8d" + _match_hostname(cert, asserted_hostname) - def test_match_hostname_ip_address_ipv6_brackets(self): - cert = {"subjectAltName": (("IP Address", "1:2::2:1"),)} - asserted_hostname = "[1:2::2:1]" - # Assert no error is raised + def test_match_hostname_include_wildcard(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = {"subjectAltName": (("DNS", "foo*"),)} + asserted_hostname = "foobar" _match_hostname(cert, asserted_hostname) - def test_match_hostname_ip_address(self): - cert = {"subjectAltName": [("IP Address", "1.1.1.1")]} + def test_match_hostname_more_than_one_dnsname_error(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = { + "subjectAltName": (("DNS", "foo*"), ("DNS", "fo*")) + } + asserted_hostname = "bar" + with pytest.raises(CertificateError, match="doesn't match either of"): + _match_hostname(cert, asserted_hostname) + + def test_dnsname_match_include_more_than_one_wildcard_error(self) -> None: + with pytest.raises(CertificateError, match="too many wildcards in certificate"): + _dnsname_match("foo**", "foobar") + + def test_match_hostname_ignore_common_name(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = {"subject": ((("commonName", "foo"),),)} + asserted_hostname = "foo" + with pytest.raises( + ImplementationCertificateError, + match="no appropriate subjectAltName fields were found", + ): + match_hostname(cert, asserted_hostname) + + def test_match_hostname_check_common_name(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = {"subject": ((("commonName", "foo"),),)} + asserted_hostname = "foo" + match_hostname(cert, asserted_hostname, True) + + def test_match_hostname_ip_address(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = { + "subjectAltName": (("IP Address", "1.1.1.1"),) + } asserted_hostname = "1.1.1.2" try: with mock.patch("urllib3.connection.log.warning") as mock_log: @@ -96,44 +137,189 @@ def test_match_hostname_ip_address(self): mock_log.assert_called_once_with( "Certificate did not match expected hostname: %s. Certificate: %s", "1.1.1.2", - {"subjectAltName": [("IP Address", "1.1.1.1")]}, + {"subjectAltName": (("IP Address", "1.1.1.1"),)}, ) - assert e._peer_cert == cert + assert e._peer_cert == cert # type: ignore[attr-defined] - def test_match_hostname_no_dns(self): - cert = {"subjectAltName": [("DNS", "")]} - asserted_hostname = "bar" + @pytest.mark.parametrize( + ["asserted_hostname", "san_ip"], + [ + ("1:2::3:4", "1:2:0:0:0:0:3:4"), + ("1:2:0:0::3:4", "1:2:0:0:0:0:3:4"), + ("::0.1.0.2", "0:0:0:0:0:0:1:2"), + ("::1%42", "0:0:0:0:0:0:0:1"), + ("::2%iface", "0:0:0:0:0:0:0:2"), + ], + ) + def test_match_hostname_ip_address_ipv6( + self, asserted_hostname: str, san_ip: str + ) -> None: + """Check that hostname matches follow RFC 9110 rules for IPv6.""" + cert: _TYPE_PEER_CERT_RET_DICT = {"subjectAltName": (("IP Address", san_ip),)} + match_hostname(cert, asserted_hostname) + + def test_match_hostname_ip_address_ipv6_doesnt_match(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = { + "subjectAltName": (("IP Address", "1:2::2:1"),) + } + asserted_hostname = "1:2::2:2" try: with mock.patch("urllib3.connection.log.warning") as mock_log: _match_hostname(cert, asserted_hostname) except CertificateError as e: - assert "hostname 'bar' doesn't match ''" in str(e) + assert "hostname '1:2::2:2' doesn't match '1:2::2:1'" in str(e) mock_log.assert_called_once_with( "Certificate did not match expected hostname: %s. Certificate: %s", - "bar", - {"subjectAltName": [("DNS", "")]}, + "1:2::2:2", + {"subjectAltName": (("IP Address", "1:2::2:1"),)}, ) - assert e._peer_cert == cert + assert e._peer_cert == cert # type: ignore[attr-defined] - def test_match_hostname_startwith_wildcard(self): - cert = {"subjectAltName": [("DNS", "*")]} - asserted_hostname = "foo" - _match_hostname(cert, asserted_hostname) - - def test_match_hostname_dnsname(self): - cert = {"subjectAltName": [("DNS", "xn--p1b6ci4b4b3a*.xn--11b5bs8d")]} - asserted_hostname = "xn--p1b6ci4b4b3a*.xn--11b5bs8d" - _match_hostname(cert, asserted_hostname) + def test_match_hostname_dns_with_brackets_doesnt_match(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = { + "subjectAltName": ( + ("DNS", "localhost"), + ("IP Address", "localhost"), + ) + } + asserted_hostname = "[localhost]" + with pytest.raises(CertificateError) as e: + _match_hostname(cert, asserted_hostname) + assert ( + "hostname '[localhost]' doesn't match either of 'localhost', 'localhost'" + in str(e.value) + ) - def test_match_hostname_include_wildcard(self): - cert = {"subjectAltName": [("DNS", "foo*")]} - asserted_hostname = "foobar" + def test_match_hostname_ip_address_ipv6_brackets(self) -> None: + cert: _TYPE_PEER_CERT_RET_DICT = { + "subjectAltName": (("IP Address", "1:2::2:1"),) + } + asserted_hostname = "[1:2::2:1]" + # Assert no error is raised _match_hostname(cert, asserted_hostname) - def test_recent_date(self): + def test_recent_date(self) -> None: # This test is to make sure that the RECENT_DATE value # doesn't get too far behind what the current date is. # When this test fails update urllib3.connection.RECENT_DATE # according to the rules defined in that file. two_years = datetime.timedelta(days=365 * 2) assert RECENT_DATE > (datetime.datetime.today() - two_years).date() + + def test_HTTPSConnection_default_socket_options(self) -> None: + conn = HTTPSConnection("not.a.real.host", port=443) + assert conn.socket_options == [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + + @pytest.mark.parametrize( + "proxy_scheme, err_part", + [ + ("http", "Unable to connect to proxy"), + ( + "https", + "Unable to connect to proxy. Your proxy appears to only use HTTP and not HTTPS", + ), + ], + ) + def test_wrap_proxy_error(self, proxy_scheme: str, err_part: str) -> None: + new_err = _wrap_proxy_error(HTTPError("unknown protocol"), proxy_scheme) + assert isinstance(new_err, ProxyError) is True + assert err_part in new_err.args[0] + + def test_url_from_pool(self) -> None: + conn = HTTPConnection("google.com", port=80) + + path = "path?query=foo" + assert f"http://google.com:80/{path}" == _url_from_connection(conn, path) + + def test_getresponse_requires_reponseoptions(self) -> None: + conn = HTTPConnection("google.com", port=80) + + # Should error if a request has not been sent + with pytest.raises(ResponseNotReady): + conn.getresponse() + + def test_assert_fingerprint_closes_socket(self) -> None: + context = mock.create_autospec(ssl_.SSLContext) + context.wrap_socket.return_value.getpeercert.return_value = b"fake cert" + conn = HTTPSConnection( + "google.com", + port=443, + assert_fingerprint="AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA", + ssl_context=context, + ) + with mock.patch.object(conn, "_new_conn"): + with pytest.raises(SSLError): + conn.connect() + + context.wrap_socket.return_value.close.assert_called_once_with() + + def test_assert_hostname_closes_socket(self) -> None: + context = mock.create_autospec(ssl_.SSLContext) + context.wrap_socket.return_value.getpeercert.return_value = { + "subjectAltName": (("DNS", "google.com"),) + } + conn = HTTPSConnection( + "google.com", port=443, assert_hostname="example.com", ssl_context=context + ) + with mock.patch.object(conn, "_new_conn"): + with pytest.raises(ImplementationCertificateError): + conn.connect() + + context.wrap_socket.return_value.close.assert_called_once_with() + + @pytest.mark.parametrize( + "accept_encoding", + [ + "Accept-Encoding", + "accept-encoding", + b"Accept-Encoding", + b"accept-encoding", + None, + ], + ) + @pytest.mark.parametrize("host", ["Host", "host", b"Host", b"host", None]) + @pytest.mark.parametrize( + "user_agent", ["User-Agent", "user-agent", b"User-Agent", b"user-agent", None] + ) + @pytest.mark.parametrize("chunked", [True, False]) + def test_skip_header( + self, + accept_encoding: str | None, + host: str | None, + user_agent: str | None, + chunked: bool, + ) -> None: + headers = {} + if accept_encoding is not None: + headers[accept_encoding] = SKIP_HEADER + if host is not None: + headers[host] = SKIP_HEADER + if user_agent is not None: + headers[user_agent] = SKIP_HEADER + + # When dropping support for Python 3.9, this can be rewritten to parenthesized + # context managers + with mock.patch("urllib3.util.connection.create_connection"): + with mock.patch( + "urllib3.connection._HTTPConnection.putheader" + ) as http_client_putheader: + conn = HTTPConnection("") + conn.request("GET", "/headers", headers=headers, chunked=chunked) + + request_headers = {} + for call in http_client_putheader.call_args_list: + header, value = call.args + request_headers[header] = value + + if accept_encoding is None: + assert "Accept-Encoding" in request_headers + else: + assert accept_encoding not in request_headers + if host is None: + assert "Host" in request_headers + else: + assert host not in request_headers + if user_agent is None: + assert "User-Agent" in request_headers + else: + assert user_agent not in request_headers diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index 872d01c..176fed4 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -1,44 +1,48 @@ -from __future__ import absolute_import +from __future__ import annotations +import http.client as httplib import ssl +import typing +from http.client import HTTPException +from queue import Empty from socket import error as SocketError from ssl import SSLError as BaseSSLError from test import SHORT_TIMEOUT +from unittest.mock import Mock, patch import pytest -from mock import Mock -from dummyserver.server import DEFAULT_CA -from urllib3._collections import HTTPHeaderDict +from dummyserver.socketserver import DEFAULT_CA +from urllib3 import Retry +from urllib3.connection import HTTPConnection from urllib3.connectionpool import ( - HTTPConnection, HTTPConnectionPool, HTTPSConnectionPool, + _url_from_pool, connection_from_url, ) from urllib3.exceptions import ( ClosedPoolError, EmptyPoolError, + FullPoolError, HostChangedError, LocationValueError, MaxRetryError, ProtocolError, + ReadTimeoutError, SSLError, TimeoutError, ) -from urllib3.packages.six.moves import http_client as httplib -from urllib3.packages.six.moves.http_client import HTTPException -from urllib3.packages.six.moves.queue import Empty from urllib3.response import HTTPResponse from urllib3.util.ssl_match_hostname import CertificateError -from urllib3.util.timeout import Timeout +from urllib3.util.timeout import _DEFAULT_TIMEOUT, Timeout from .test_response import MockChunkedEncodingResponse, MockSock class HTTPUnixConnection(HTTPConnection): - def __init__(self, host, timeout=60, **kwargs): - super(HTTPUnixConnection, self).__init__("localhost") + def __init__(self, host: str, timeout: int = 60, **kwargs: typing.Any) -> None: + super().__init__("localhost") self.unix_socket = host self.timeout = timeout self.sock = None @@ -49,7 +53,7 @@ class HTTPUnixConnectionPool(HTTPConnectionPool): ConnectionCls = HTTPUnixConnection -class TestConnectionPool(object): +class TestConnectionPool: """ Tests in this suite should exercise the ConnectionPool functionality without actually making any network requests or connections. @@ -83,7 +87,7 @@ class TestConnectionPool(object): ), ], ) - def test_same_host(self, a, b): + def test_same_host(self, a: str, b: str) -> None: with connection_from_url(a) as c: assert c.is_same_host(b) @@ -109,7 +113,7 @@ def test_same_host(self, a, b): ("http://[dead::beef]", "https://[dead::beef%en5]/"), ], ) - def test_not_same_host(self, a, b): + def test_not_same_host(self, a: str, b: str) -> None: with connection_from_url(a) as c: assert not c.is_same_host(b) @@ -127,7 +131,7 @@ def test_not_same_host(self, a, b): ("google.com", "http://google.com:80/abracadabra"), ], ) - def test_same_host_no_port_http(self, a, b): + def test_same_host_no_port_http(self, a: str, b: str) -> None: # This test was introduced in #801 to deal with the fact that urllib3 # never initializes ConnectionPool objects with port=None. with HTTPConnectionPool(a) as c: @@ -144,7 +148,7 @@ def test_same_host_no_port_http(self, a, b): ("google.com", "https://google.com:443/abracadabra"), ], ) - def test_same_host_no_port_https(self, a, b): + def test_same_host_no_port_https(self, a: str, b: str) -> None: # This test was introduced in #801 to deal with the fact that urllib3 # never initializes ConnectionPool objects with port=None. with HTTPSConnectionPool(a) as c: @@ -159,7 +163,7 @@ def test_same_host_no_port_https(self, a, b): ("google.com", "http://google.com./"), ], ) - def test_not_same_host_no_port_http(self, a, b): + def test_not_same_host_no_port_http(self, a: str, b: str) -> None: with HTTPConnectionPool(a) as c: assert not c.is_same_host(b) @@ -175,7 +179,7 @@ def test_not_same_host_no_port_http(self, a, b): ("google.com", "https://google.com./"), ], ) - def test_not_same_host_no_port_https(self, a, b): + def test_not_same_host_no_port_https(self, a: str, b: str) -> None: with HTTPSConnectionPool(a) as c: assert not c.is_same_host(b) @@ -196,7 +200,7 @@ def test_not_same_host_no_port_https(self, a, b): ("%2Ftmp%2FTEST.sock", "http+unix://%2Ftmp%2FTEST.sock/abracadabra"), ], ) - def test_same_host_custom_protocol(self, a, b): + def test_same_host_custom_protocol(self, a: str, b: str) -> None: with HTTPUnixConnectionPool(a) as c: assert c.is_same_host(b) @@ -209,11 +213,11 @@ def test_same_host_custom_protocol(self, a, b): ("%2Fvar%2Frun%2Fdocker.sock", "http+unix://%2Ftmp%2FTEST.sock"), ], ) - def test_not_same_host_custom_protocol(self, a, b): + def test_not_same_host_custom_protocol(self, a: str, b: str) -> None: with HTTPUnixConnectionPool(a) as c: assert not c.is_same_host(b) - def test_max_connections(self): + def test_max_connections(self) -> None: with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool: pool._get_conn(timeout=SHORT_TIMEOUT) @@ -225,13 +229,26 @@ def test_max_connections(self): assert pool.num_connections == 1 - def test_pool_edgecases(self, caplog): + def test_put_conn_when_pool_is_full_nonblocking( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """ + If maxsize = n and we _put_conn n + 1 conns, the n + 1th conn will + get closed and will not get added to the pool. + """ with HTTPConnectionPool(host="localhost", maxsize=1, block=False) as pool: conn1 = pool._get_conn() - conn2 = pool._get_conn() # New because block=False + # pool.pool is empty because we popped the one None that pool.pool was initialized with + # but this pool._get_conn call will not raise EmptyPoolError because block is False + conn2 = pool._get_conn() - pool._put_conn(conn1) - pool._put_conn(conn2) # Should be discarded + with patch.object(conn1, "close") as conn1_close: + with patch.object(conn2, "close") as conn2_close: + pool._put_conn(conn1) + pool._put_conn(conn2) + + assert conn1_close.called is False + assert conn2_close.called is True assert conn1 == pool._get_conn() assert conn2 != pool._get_conn() @@ -240,13 +257,47 @@ def test_pool_edgecases(self, caplog): assert "Connection pool is full, discarding connection" in caplog.text assert "Connection pool size: 1" in caplog.text - def test_exception_str(self): + def test_put_conn_when_pool_is_full_blocking(self) -> None: + """ + If maxsize = n and we _put_conn n + 1 conns, the n + 1th conn will + cause a FullPoolError. + """ + with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool: + conn1 = pool._get_conn() + conn2 = pool._new_conn() + + with patch.object(conn1, "close") as conn1_close: + with patch.object(conn2, "close") as conn2_close: + pool._put_conn(conn1) + with pytest.raises(FullPoolError): + pool._put_conn(conn2) + + assert conn1_close.called is False + assert conn2_close.called is True + + assert conn1 == pool._get_conn() + + def test_put_conn_closed_pool(self) -> None: + with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool: + conn1 = pool._get_conn() + with patch.object(conn1, "close") as conn1_close: + pool.close() + + assert pool.pool is None + + # Accessing pool.pool will raise AttributeError, which will get + # caught and will close conn1 + pool._put_conn(conn1) + + assert conn1_close.called is True + + def test_exception_str(self) -> None: assert ( str(EmptyPoolError(HTTPConnectionPool(host="localhost"), "Test.")) == "HTTPConnectionPool(host='localhost', port=None): Test." ) - def test_retry_exception_str(self): + def test_retry_exception_str(self) -> None: assert ( str(MaxRetryError(HTTPConnectionPool(host="localhost"), "Test.", None)) == "HTTPConnectionPool(host='localhost', port=None): " @@ -264,21 +315,23 @@ def test_retry_exception_str(self): "(Caused by %r)" % err ) - def test_pool_size(self): + def test_pool_size(self) -> None: POOL_SIZE = 1 with HTTPConnectionPool( host="localhost", maxsize=POOL_SIZE, block=True ) as pool: - def _raise(ex): - raise ex() - - def _test(exception, expect, reason=None): - pool._make_request = lambda *args, **kwargs: _raise(exception) - with pytest.raises(expect) as excinfo: - pool.request("GET", "/") + def _test( + exception: type[BaseException], + expect: type[BaseException], + reason: type[BaseException] | None = None, + ) -> None: + with patch.object(pool, "_make_request", side_effect=exception()): + with pytest.raises(expect) as excinfo: + pool.request("GET", "/") if reason is not None: - assert isinstance(excinfo.value.reason, reason) + assert isinstance(excinfo.value.reason, reason) # type: ignore[attr-defined] + assert pool.pool is not None assert pool.pool.qsize() == POOL_SIZE # Make sure that all of the exceptions return the connection @@ -290,26 +343,33 @@ def _test(exception, expect, reason=None): # being raised, a retry will be triggered, but that retry will # fail, eventually raising MaxRetryError, not EmptyPoolError # See: https://github.com/urllib3/urllib3/issues/76 - pool._make_request = lambda *args, **kwargs: _raise(HTTPException) - with pytest.raises(MaxRetryError): - pool.request("GET", "/", retries=1, pool_timeout=SHORT_TIMEOUT) + with patch.object(pool, "_make_request", side_effect=HTTPException()): + with pytest.raises(MaxRetryError): + pool.request("GET", "/", retries=1, pool_timeout=SHORT_TIMEOUT) + assert pool.pool is not None assert pool.pool.qsize() == POOL_SIZE - def test_empty_does_not_put_conn(self): + def test_empty_does_not_put_conn(self) -> None: """Do not put None back in the pool if the pool was empty""" with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool: - pool._get_conn = Mock(side_effect=EmptyPoolError(pool, "Pool is empty")) - pool._put_conn = Mock(side_effect=AssertionError("Unexpected _put_conn")) - with pytest.raises(EmptyPoolError): - pool.request("GET", "/") - - def test_assert_same_host(self): + with patch.object( + pool, "_get_conn", side_effect=EmptyPoolError(pool, "Pool is empty") + ): + with patch.object( + pool, + "_put_conn", + side_effect=AssertionError("Unexpected _put_conn"), + ): + with pytest.raises(EmptyPoolError): + pool.request("GET", "/") + + def test_assert_same_host(self) -> None: with connection_from_url("http://google.com:80") as c: with pytest.raises(HostChangedError): c.request("GET", "http://yahoo.com:80", assert_same_host=True) - def test_pool_close(self): + def test_pool_close(self) -> None: pool = connection_from_url("http://google.com:80") # Populate with some connections @@ -333,9 +393,10 @@ def test_pool_close(self): pool._get_conn() with pytest.raises(Empty): + assert old_pool_queue is not None old_pool_queue.get(block=False) - def test_pool_close_twice(self): + def test_pool_close_twice(self) -> None: pool = connection_from_url("http://google.com:80") # Populate with some connections @@ -352,13 +413,13 @@ def test_pool_close_twice(self): except AttributeError: pytest.fail("Pool of the ConnectionPool is None and has no attribute get.") - def test_pool_timeouts(self): + def test_pool_timeouts(self) -> None: with HTTPConnectionPool(host="localhost") as pool: conn = pool._new_conn() assert conn.__class__ == HTTPConnection assert pool.timeout.__class__ == Timeout - assert pool.timeout._read == Timeout.DEFAULT_TIMEOUT - assert pool.timeout._connect == Timeout.DEFAULT_TIMEOUT + assert pool.timeout._read == _DEFAULT_TIMEOUT + assert pool.timeout._connect == _DEFAULT_TIMEOUT assert pool.timeout.total is None pool = HTTPConnectionPool(host="localhost", timeout=SHORT_TIMEOUT) @@ -366,11 +427,11 @@ def test_pool_timeouts(self): assert pool.timeout._connect == SHORT_TIMEOUT assert pool.timeout.total is None - def test_no_host(self): + def test_no_host(self) -> None: with pytest.raises(LocationValueError): - HTTPConnectionPool(None) + HTTPConnectionPool(None) # type: ignore[arg-type] - def test_contextmanager(self): + def test_contextmanager(self) -> None: with connection_from_url("http://google.com:80") as pool: # Populate with some connections conn1 = pool._get_conn() @@ -389,20 +450,20 @@ def test_contextmanager(self): with pytest.raises(ClosedPoolError): pool._get_conn() with pytest.raises(Empty): + assert old_pool_queue is not None old_pool_queue.get(block=False) - def test_absolute_url(self): - with connection_from_url("http://google.com:80") as c: - assert "http://google.com:80/path?query=foo" == c._absolute_url( - "path?query=foo" - ) + def test_url_from_pool(self) -> None: + with connection_from_url("http://google.com:80") as pool: + path = "path?query=foo" + assert f"http://google.com:80/{path}" == _url_from_pool(pool, path) - def test_ca_certs_default_cert_required(self): + def test_ca_certs_default_cert_required(self) -> None: with connection_from_url("https://google.com:80", ca_certs=DEFAULT_CA) as pool: conn = pool._get_conn() - assert conn.cert_reqs == ssl.CERT_REQUIRED + assert conn.cert_reqs == ssl.CERT_REQUIRED # type: ignore[attr-defined] - def test_cleanup_on_extreme_connection_error(self): + def test_cleanup_on_extreme_connection_error(self) -> None: """ This test validates that we clean up properly even on exceptions that we'd not otherwise catch, i.e. those that inherit from BaseException @@ -412,25 +473,25 @@ def test_cleanup_on_extreme_connection_error(self): class RealBad(BaseException): pass - def kaboom(*args, **kwargs): + def kaboom(*args: typing.Any, **kwargs: typing.Any) -> None: raise RealBad() with connection_from_url("http://localhost:80") as c: - c._make_request = kaboom + with patch.object(c, "_make_request", kaboom): + assert c.pool is not None + initial_pool_size = c.pool.qsize() - initial_pool_size = c.pool.qsize() - - try: - # We need to release_conn this way or we'd put it away - # regardless. - c.urlopen("GET", "/", release_conn=False) - except RealBad: - pass + try: + # We need to release_conn this way or we'd put it away + # regardless. + c.urlopen("GET", "/", release_conn=False) + except RealBad: + pass new_pool_size = c.pool.qsize() assert initial_pool_size == new_pool_size - def test_release_conn_param_is_respected_after_http_error_retry(self): + def test_release_conn_param_is_respected_after_http_error_retry(self) -> None: """For successful ```urlopen(release_conn=False)```, the connection isn't released, even after a retry. @@ -441,40 +502,73 @@ def test_release_conn_param_is_respected_after_http_error_retry(self): [1] """ - class _raise_once_make_request_function(object): + class _raise_once_make_request_function: """Callable that can mimic `_make_request()`. Raises the given exception on its first call, but returns a successful response on subsequent calls. """ - def __init__(self, ex): - super(_raise_once_make_request_function, self).__init__() - self._ex = ex - - def __call__(self, *args, **kwargs): + def __init__( + self, ex: type[BaseException], pool: HTTPConnectionPool + ) -> None: + super().__init__() + self._ex: type[BaseException] | None = ex + self._pool = pool + + def __call__( + self, + conn: HTTPConnection, + method: str, + url: str, + *args: typing.Any, + retries: Retry, + **kwargs: typing.Any, + ) -> HTTPResponse: if self._ex: ex, self._ex = self._ex, None raise ex() - response = httplib.HTTPResponse(MockSock) - response.fp = MockChunkedEncodingResponse([b"f", b"o", b"o"]) - response.headers = response.msg = HTTPHeaderDict() + httplib_response = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + httplib_response.fp = MockChunkedEncodingResponse([b"f", b"o", b"o"]) # type: ignore[assignment] + httplib_response.headers = httplib_response.msg = httplib.HTTPMessage() + + response_conn: HTTPConnection | None = kwargs.get("response_conn") + + response = HTTPResponse( + body=httplib_response, + headers=httplib_response.headers, # type: ignore[arg-type] + status=httplib_response.status, + version=httplib_response.version, + reason=httplib_response.reason, + original_response=httplib_response, + retries=retries, + request_method=method, + request_url=url, + preload_content=False, + connection=response_conn, + pool=self._pool, + ) return response - def _test(exception): + def _test(exception: type[BaseException]) -> None: with HTTPConnectionPool(host="localhost", maxsize=1, block=True) as pool: # Verify that the request succeeds after two attempts, and that the # connection is left on the response object, instead of being # released back into the pool. - pool._make_request = _raise_once_make_request_function(exception) - response = pool.urlopen( - "GET", - "/", - retries=1, - release_conn=False, - preload_content=False, - chunked=True, - ) + with patch.object( + pool, + "_make_request", + _raise_once_make_request_function(exception, pool), + ): + response = pool.urlopen( + "GET", + "/", + retries=1, + release_conn=False, + preload_content=False, + chunked=True, + ) + assert pool.pool is not None assert pool.pool.qsize() == 0 assert pool.num_connections == 2 assert response.connection is not None @@ -489,21 +583,12 @@ def _test(exception): _test(SocketError) _test(ProtocolError) - def test_custom_http_response_class(self): - class CustomHTTPResponse(HTTPResponse): - pass - - class CustomConnectionPool(HTTPConnectionPool): - ResponseCls = CustomHTTPResponse - - def _make_request(self, *args, **kwargs): - httplib_response = httplib.HTTPResponse(MockSock) - httplib_response.fp = MockChunkedEncodingResponse([b"f", b"o", b"o"]) - httplib_response.headers = httplib_response.msg = HTTPHeaderDict() - return httplib_response - - with CustomConnectionPool(host="localhost", maxsize=1, block=True) as pool: - response = pool.request( - "GET", "/", retries=False, chunked=True, preload_content=False - ) - assert isinstance(response, CustomHTTPResponse) + def test_read_timeout_0_does_not_raise_bad_status_line_error(self) -> None: + with HTTPConnectionPool(host="localhost", maxsize=1) as pool: + conn = Mock(spec=HTTPConnection) + # Needed to tell the pool that the connection is alive. + conn.is_closed = False + with patch.object(Timeout, "read_timeout", 0): + timeout = Timeout(1, 1, 1) + with pytest.raises(ReadTimeoutError): + pool._make_request(conn, "", "", timeout=timeout) diff --git a/test/test_exceptions.py b/test/test_exceptions.py index 9fd0eb0..ef103d1 100644 --- a/test/test_exceptions.py +++ b/test/test_exceptions.py @@ -1,7 +1,13 @@ +from __future__ import annotations + import pickle +import socket +from email.errors import MessageDefect +from test import DUMMY_POOL import pytest +from urllib3.connection import HTTPConnection from urllib3.connectionpool import HTTPConnectionPool from urllib3.exceptions import ( ClosedPoolError, @@ -12,36 +18,56 @@ HTTPError, LocationParseError, MaxRetryError, + NameResolutionError, + NewConnectionError, ReadTimeoutError, ) -class TestPickle(object): +class TestPickle: @pytest.mark.parametrize( "exception", [ HTTPError(None), - MaxRetryError(None, None, None), - LocationParseError(None), + MaxRetryError(DUMMY_POOL, "", None), + LocationParseError(""), ConnectTimeoutError(None), HTTPError("foo"), HTTPError("foo", IOError("foo")), MaxRetryError(HTTPConnectionPool("localhost"), "/", None), LocationParseError("fake location"), - ClosedPoolError(HTTPConnectionPool("localhost"), None), - EmptyPoolError(HTTPConnectionPool("localhost"), None), - HostChangedError(HTTPConnectionPool("localhost"), "/", None), - ReadTimeoutError(HTTPConnectionPool("localhost"), "/", None), + ClosedPoolError(HTTPConnectionPool("localhost"), ""), + EmptyPoolError(HTTPConnectionPool("localhost"), ""), + HostChangedError(HTTPConnectionPool("localhost"), "/", 0), + ReadTimeoutError(HTTPConnectionPool("localhost"), "/", ""), + NewConnectionError(HTTPConnection("localhost"), ""), + NameResolutionError("", HTTPConnection("localhost"), socket.gaierror()), ], ) - def test_exceptions(self, exception): + def test_exceptions(self, exception: Exception) -> None: result = pickle.loads(pickle.dumps(exception)) assert isinstance(result, type(exception)) -class TestFormat(object): - def test_header_parsing_errors(self): - hpe = HeaderParsingError("defects", "unparsed_data") +class TestFormat: + def test_header_parsing_errors(self) -> None: + hpe = HeaderParsingError([MessageDefect("defects")], "unparsed_data") assert "defects" in str(hpe) assert "unparsed_data" in str(hpe) + + +class TestNewConnectionError: + def test_pool_property_deprecation_warning(self) -> None: + err = NewConnectionError(HTTPConnection("localhost"), "test") + with pytest.warns(DeprecationWarning) as records: + err_pool = err.pool + + assert err_pool is err.conn + msg = ( + "The 'pool' property is deprecated and will be removed " + "in urllib3 v2.1.0. Use 'conn' instead." + ) + record = records[0] + assert isinstance(record.message, Warning) + assert record.message.args[0] == msg diff --git a/test/test_fields.py b/test/test_fields.py index 98ce17c..faf74a3 100644 --- a/test/test_fields.py +++ b/test/test_fields.py @@ -1,10 +1,18 @@ +from __future__ import annotations + import pytest -from urllib3.fields import RequestField, format_header_param_rfc2231, guess_content_type -from urllib3.packages.six import u +from urllib3.fields import ( + RequestField, + format_header_param, + format_header_param_html5, + format_header_param_rfc2231, + format_multipart_header_param, + guess_content_type, +) -class TestRequestField(object): +class TestRequestField: @pytest.mark.parametrize( "filename, content_types", [ @@ -13,18 +21,22 @@ class TestRequestField(object): (None, ["application/octet-stream"]), ], ) - def test_guess_content_type(self, filename, content_types): + def test_guess_content_type( + self, filename: str | None, content_types: list[str] + ) -> None: assert guess_content_type(filename) in content_types - def test_create(self): + def test_create(self) -> None: simple_field = RequestField("somename", "data") assert simple_field.render_headers() == "\r\n" filename_field = RequestField("somename", "data", filename="somefile.txt") assert filename_field.render_headers() == "\r\n" - headers_field = RequestField("somename", "data", headers={"Content-Length": 4}) + headers_field = RequestField( + "somename", "data", headers={"Content-Length": "4"} + ) assert headers_field.render_headers() == "Content-Length: 4\r\n\r\n" - def test_make_multipart(self): + def test_make_multipart(self) -> None: field = RequestField("somename", "data") field.make_multipart(content_type="image/jpg", content_location="/test") assert ( @@ -35,7 +47,7 @@ def test_make_multipart(self): "\r\n" ) - def test_make_multipart_empty_filename(self): + def test_make_multipart_empty_filename(self) -> None: field = RequestField("somename", "data", "") field.make_multipart(content_type="application/octet-stream") assert ( @@ -45,7 +57,7 @@ def test_make_multipart_empty_filename(self): "\r\n" ) - def test_render_parts(self): + def test_render_parts(self) -> None: field = RequestField("somename", "data") parts = field._render_parts({"name": "value", "filename": "value"}) assert 'name="value"' in parts @@ -53,50 +65,56 @@ def test_render_parts(self): parts = field._render_parts([("name", "value"), ("filename", "value")]) assert parts == 'name="value"; filename="value"' - def test_render_part_rfc2231_unicode(self): - field = RequestField( - "somename", "data", header_formatter=format_header_param_rfc2231 - ) - param = field._render_part("filename", u("n\u00e4me")) - assert param == "filename*=utf-8''n%C3%A4me" + @pytest.mark.parametrize( + ("value", "expect"), + [("näme", "filename*=utf-8''n%C3%A4me"), (b"name", 'filename="name"')], + ) + def test_format_header_param_rfc2231_deprecated( + self, value: bytes | str, expect: str + ) -> None: + with pytest.deprecated_call(match=r"urllib3 v2\.1\.0"): + param = format_header_param_rfc2231("filename", value) - def test_render_part_rfc2231_ascii(self): - field = RequestField( - "somename", "data", header_formatter=format_header_param_rfc2231 - ) - param = field._render_part("filename", b"name") - assert param == 'filename="name"' + assert param == expect - def test_render_part_html5_unicode(self): - field = RequestField("somename", "data") - param = field._render_part("filename", u("n\u00e4me")) - assert param == u('filename="n\u00e4me"') + def test_format_header_param_html5_deprecated(self) -> None: + with pytest.deprecated_call(match=r"urllib3 v2\.1\.0"): + param2 = format_header_param_html5("filename", "name") - def test_render_part_html5_ascii(self): - field = RequestField("somename", "data") - param = field._render_part("filename", b"name") - assert param == 'filename="name"' + with pytest.deprecated_call(match=r"urllib3 v2\.1\.0"): + param1 = format_header_param("filename", "name") - def test_render_part_html5_unicode_escape(self): - field = RequestField("somename", "data") - param = field._render_part("filename", u("hello\\world\u0022")) - assert param == u('filename="hello\\\\world%22"') + assert param1 == param2 - def test_render_part_html5_unicode_with_control_character(self): - field = RequestField("somename", "data") - param = field._render_part("filename", u("hello\x1A\x1B\x1C")) - assert param == u('filename="hello%1A\x1B%1C"') - - def test_from_tuples_rfc2231(self): - field = RequestField.from_tuples( - u("fieldname"), - (u("filen\u00e4me"), "data"), - header_formatter=format_header_param_rfc2231, - ) + @pytest.mark.parametrize( + ("value", "expect"), + [ + ("name", "name"), + ("näme", "näme"), + (b"n\xc3\xa4me", "näme"), + ("ski ⛷.txt", "ski ⛷.txt"), + ("control \x1A\x1B\x1C", "control \x1A\x1B\x1C"), + ("backslash \\", "backslash \\"), + ("quotes '\"", "quotes '%22"), + ("newline \n\r", "newline %0A%0D"), + ], + ) + def test_format_multipart_header_param( + self, value: bytes | str, expect: str + ) -> None: + param = format_multipart_header_param("filename", value) + assert param == f'filename="{expect}"' + + def test_from_tuples(self) -> None: + field = RequestField.from_tuples("file", ("スキー旅行.txt", "data")) cd = field.headers["Content-Disposition"] - assert cd == u("form-data; name=\"fieldname\"; filename*=utf-8''filen%C3%A4me") + assert cd == 'form-data; name="file"; filename="スキー旅行.txt"' + + def test_from_tuples_rfc2231(self) -> None: + with pytest.deprecated_call(match=r"urllib3 v2\.1\.0"): + field = RequestField.from_tuples( + "file", ("näme", "data"), header_formatter=format_header_param_rfc2231 + ) - def test_from_tuples_html5(self): - field = RequestField.from_tuples(u("fieldname"), (u("filen\u00e4me"), "data")) cd = field.headers["Content-Disposition"] - assert cd == u('form-data; name="fieldname"; filename="filen\u00e4me"') + assert cd == "form-data; name=\"file\"; filename*=utf-8''n%C3%A4me" diff --git a/test/test_filepost.py b/test/test_filepost.py index 5b0cfe1..b6da4b9 100644 --- a/test/test_filepost.py +++ b/test/test_filepost.py @@ -1,112 +1,100 @@ +from __future__ import annotations + import pytest from urllib3.fields import RequestField -from urllib3.filepost import encode_multipart_formdata, iter_fields -from urllib3.packages.six import b, u +from urllib3.filepost import _TYPE_FIELDS, encode_multipart_formdata BOUNDARY = "!! test boundary !!" +BOUNDARY_BYTES = BOUNDARY.encode() -class TestIterfields(object): - def test_dict(self): - for fieldname, value in iter_fields(dict(a="b")): - assert (fieldname, value) == ("a", "b") - - assert list(sorted(iter_fields(dict(a="b", c="d")))) == [("a", "b"), ("c", "d")] - - def test_tuple_list(self): - for fieldname, value in iter_fields([("a", "b")]): - assert (fieldname, value) == ("a", "b") - - assert list(iter_fields([("a", "b"), ("c", "d")])) == [("a", "b"), ("c", "d")] - - -class TestMultipartEncoding(object): +class TestMultipartEncoding: @pytest.mark.parametrize( "fields", [dict(k="v", k2="v2"), [("k", "v"), ("k2", "v2")]] ) - def test_input_datastructures(self, fields): + def test_input_datastructures(self, fields: _TYPE_FIELDS) -> None: encoded, _ = encode_multipart_formdata(fields, boundary=BOUNDARY) - assert encoded.count(b(BOUNDARY)) == 3 + assert encoded.count(BOUNDARY_BYTES) == 3 @pytest.mark.parametrize( "fields", [ [("k", "v"), ("k2", "v2")], - [("k", b"v"), (u("k2"), b"v2")], - [("k", b"v"), (u("k2"), "v2")], + [("k", b"v"), ("k2", b"v2")], + [("k", b"v"), ("k2", "v2")], ], ) - def test_field_encoding(self, fields): + def test_field_encoding(self, fields: _TYPE_FIELDS) -> None: encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) expected = ( - b"--" + b(BOUNDARY) + b"\r\n" + b"--" + BOUNDARY_BYTES + b"\r\n" b'Content-Disposition: form-data; name="k"\r\n' b"\r\n" b"v\r\n" - b"--" + b(BOUNDARY) + b"\r\n" + b"--" + BOUNDARY_BYTES + b"\r\n" b'Content-Disposition: form-data; name="k2"\r\n' b"\r\n" b"v2\r\n" - b"--" + b(BOUNDARY) + b"--\r\n" + b"--" + BOUNDARY_BYTES + b"--\r\n" ) assert encoded == expected assert content_type == "multipart/form-data; boundary=" + str(BOUNDARY) - def test_filename(self): + def test_filename(self) -> None: fields = [("k", ("somename", b"v"))] encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) expected = ( - b"--" + b(BOUNDARY) + b"\r\n" + b"--" + BOUNDARY_BYTES + b"\r\n" b'Content-Disposition: form-data; name="k"; filename="somename"\r\n' b"Content-Type: application/octet-stream\r\n" b"\r\n" b"v\r\n" - b"--" + b(BOUNDARY) + b"--\r\n" + b"--" + BOUNDARY_BYTES + b"--\r\n" ) assert encoded == expected assert content_type == "multipart/form-data; boundary=" + str(BOUNDARY) - def test_textplain(self): + def test_textplain(self) -> None: fields = [("k", ("somefile.txt", b"v"))] encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) expected = ( - b"--" + b(BOUNDARY) + b"\r\n" + b"--" + BOUNDARY_BYTES + b"\r\n" b'Content-Disposition: form-data; name="k"; filename="somefile.txt"\r\n' b"Content-Type: text/plain\r\n" b"\r\n" b"v\r\n" - b"--" + b(BOUNDARY) + b"--\r\n" + b"--" + BOUNDARY_BYTES + b"--\r\n" ) assert encoded == expected assert content_type == "multipart/form-data; boundary=" + str(BOUNDARY) - def test_explicit(self): + def test_explicit(self) -> None: fields = [("k", ("somefile.txt", b"v", "image/jpeg"))] encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) expected = ( - b"--" + b(BOUNDARY) + b"\r\n" + b"--" + BOUNDARY_BYTES + b"\r\n" b'Content-Disposition: form-data; name="k"; filename="somefile.txt"\r\n' b"Content-Type: image/jpeg\r\n" b"\r\n" b"v\r\n" - b"--" + b(BOUNDARY) + b"--\r\n" + b"--" + BOUNDARY_BYTES + b"--\r\n" ) assert encoded == expected assert content_type == "multipart/form-data; boundary=" + str(BOUNDARY) - def test_request_fields(self): + def test_request_fields(self) -> None: fields = [ RequestField( "k", @@ -118,11 +106,11 @@ def test_request_fields(self): encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) expected = ( - b"--" + b(BOUNDARY) + b"\r\n" + b"--" + BOUNDARY_BYTES + b"\r\n" b"Content-Type: image/jpeg\r\n" b"\r\n" b"v\r\n" - b"--" + b(BOUNDARY) + b"--\r\n" + b"--" + BOUNDARY_BYTES + b"--\r\n" ) assert encoded == expected diff --git a/test/test_http2_connection.py b/test/test_http2_connection.py new file mode 100644 index 0000000..52c754e --- /dev/null +++ b/test/test_http2_connection.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import socket +from unittest import mock + +import pytest + +from urllib3.connection import _get_default_user_agent +from urllib3.exceptions import ConnectionError +from urllib3.http2.connection import ( + HTTP2Connection, + _is_illegal_header_value, + _is_legal_header_name, +) + +# [1] https://httpwg.org/specs/rfc9113.html#n-field-validity + + +class TestHTTP2Connection: + def test__is_legal_header_name(self) -> None: + assert _is_legal_header_name(b"foo"), "foo" + assert _is_legal_header_name(b"foo-bar"), "foo-bar" + assert _is_legal_header_name(b"foo-bar-baz"), "foo-bar-baz" + + # A field name MUST NOT contain characters in the ranges 0x00-0x20, + # 0x41-0x5a, or 0x7f-0xff (all ranges inclusive). [1] + for i in range(0x00, 0x20): + assert not _is_legal_header_name( + f"foo{chr(i)}bar".encode() + ), f"foo\\x{i}bar" + for i in range(0x41, 0x5A): + assert not _is_legal_header_name( + f"foo{chr(i)}bar".encode() + ), f"foo\\x{i}bar" + for i in range(0x7F, 0xFF): + assert not _is_legal_header_name( + f"foo{chr(i)}bar".encode() + ), f"foo\\x{i}bar" + + # This specifically excludes all non-visible ASCII characters, ASCII SP + # (0x20), and uppercase characters ('A' to 'Z', ASCII 0x41 to 0x5a). [1] + assert not _is_legal_header_name(b"foo bar"), "foo bar" + assert not _is_legal_header_name(b"foo\x20bar"), "foo\\x20bar" + assert not _is_legal_header_name(b"Foo-Bar"), "Foo-Bar" + + # With the exception of pseudo-header fields (Section 8.3), which have a + # name that starts with a single colon, field names MUST NOT include a + # colon (ASCII COLON, 0x3a). [1] + assert not _is_legal_header_name(b":foo"), ":foo" + assert not _is_legal_header_name(b"foo:bar"), "foo:bar" + assert not _is_legal_header_name(b"foo:"), "foo:" + + def test__is_illegal_header_value(self) -> None: + assert not _is_illegal_header_value(b"foo"), "foo" + assert not _is_illegal_header_value(b"foo bar"), "foo bar" + assert not _is_illegal_header_value(b"foo\tbar"), "foo\\tbar" + + # A field value MUST NOT contain the zero value (ASCII NUL, 0x00), line + # feed (ASCII LF, 0x0a), or carriage return (ASCII CR, 0x0d) at any + # position. [1] + assert _is_illegal_header_value(b"foo\0bar"), "foo\\0bar" + assert _is_illegal_header_value(b"foo\x00bar"), "foo\\x00bar" + assert _is_illegal_header_value(b"foo\x0abar"), "foo\\x0abar" + assert _is_illegal_header_value(b"foo\x0dbar"), "foo\\x0dbar" + assert _is_illegal_header_value(b"foo\rbar"), "foo\\rbar" + assert _is_illegal_header_value(b"foo\nbar"), "foo\\nbar" + assert _is_illegal_header_value(b"foo\r\nbar"), "foo\\r\\nbar" + + # A field value MUST NOT start or end with an ASCII whitespace character + # (ASCII SP or HTAB, 0x20 or 0x09). [1] + assert _is_illegal_header_value(b" foo"), " foo" + assert _is_illegal_header_value(b"foo "), "foo " + assert _is_illegal_header_value(b"foo\x20"), "foo\\x20" + assert _is_illegal_header_value(b"\tfoo"), "\\tfoo" + assert _is_illegal_header_value(b"foo\t"), "foo\\t" + assert _is_illegal_header_value(b"foo\x09"), "foo\\x09" + + def test_default_socket_options(self) -> None: + conn = HTTP2Connection("example.com") + assert conn.socket_options == [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + assert conn.port == 443 + + def test_putheader(self) -> None: + conn = HTTP2Connection("example.com") + conn.putheader("foo", "bar") + assert conn._headers == [(b"foo", b"bar")] + + def test_request_putheader(self) -> None: + conn = HTTP2Connection("example.com") + conn.sock = mock.MagicMock( + sendall=mock.Mock(return_value=None), + ) + conn.putheader = mock.MagicMock(return_value=None) # type: ignore[method-assign] + conn.request("GET", "/", headers={"foo": "bar"}) + conn.putheader.assert_has_calls( + [ + mock.call("foo", "bar"), + mock.call(b"user-agent", _get_default_user_agent()), + ] + ) + + def test_putheader_ValueError(self) -> None: + conn = HTTP2Connection("example.com") + with pytest.raises(ValueError): + conn.putheader("foo\0bar", "baz") + with pytest.raises(ValueError): + conn.putheader("foo", "foo\r\nbar") + + def test_endheaders_ConnectionError(self) -> None: + conn = HTTP2Connection("example.com") + with pytest.raises(ConnectionError): + conn.endheaders() + + def test_send_ConnectionError(self) -> None: + conn = HTTP2Connection("example.com") + with pytest.raises(ConnectionError): + conn.send(b"foo") + + def test_send_bytes(self) -> None: + conn = HTTP2Connection("example.com") + conn.sock = mock.MagicMock( + sendall=mock.Mock(return_value=None), + ) + conn._h2_conn._obj.data_to_send = mock.Mock(return_value=b"bar") + conn._h2_conn._obj.send_data = mock.Mock(return_value=None) + conn._h2_conn._obj.get_next_available_stream_id = mock.Mock(return_value=1) + + conn.putrequest("GET", "/") + conn.endheaders() + conn.send(b"foo") + + conn._h2_conn._obj.data_to_send.assert_called_with() + conn.sock.sendall.assert_called_with(b"bar") + conn._h2_conn._obj.send_data.assert_called_with(1, b"foo", end_stream=True) + + def test_send_str(self) -> None: + conn = HTTP2Connection("example.com") + conn.sock = mock.MagicMock( + sendall=mock.Mock(return_value=None), + ) + conn._h2_conn._obj.data_to_send = mock.Mock(return_value=b"bar") + conn._h2_conn._obj.send_data = mock.Mock(return_value=None) + conn._h2_conn._obj.get_next_available_stream_id = mock.Mock(return_value=1) + + conn.putrequest("GET", "/") + conn.endheaders(message_body=b"foo") + conn.send("foo") + + conn._h2_conn._obj.data_to_send.assert_called_with() + conn.sock.sendall.assert_called_with(b"bar") + conn._h2_conn._obj.send_data.assert_called_with(1, b"foo", end_stream=True) + + def test_send_iter(self) -> None: + conn = HTTP2Connection("example.com") + conn.sock = mock.MagicMock( + sendall=mock.Mock(return_value=None), + ) + conn._h2_conn._obj.data_to_send = mock.Mock(return_value=b"baz") + conn._h2_conn._obj.send_data = mock.Mock(return_value=None) + conn._h2_conn._obj.get_next_available_stream_id = mock.Mock(return_value=1) + conn._h2_conn._obj.end_stream = mock.Mock(return_value=None) + + conn.putrequest("GET", "/") + conn.endheaders(message_body=[b"foo", b"bar"]) + conn.send([b"foo", b"bar"]) + + conn._h2_conn._obj.data_to_send.assert_has_calls( + [ + mock.call(), + mock.call(), + ] + ) + conn.sock.sendall.assert_has_calls( + [ + mock.call(b"baz"), + mock.call(b"baz"), + ] + ) + conn._h2_conn._obj.send_data.assert_has_calls( + [ + mock.call(1, b"foo", end_stream=False), + mock.call(1, b"bar", end_stream=False), + ] + ) + conn._h2_conn._obj.end_stream.assert_called_with(1) + + def test_send_file_str(self) -> None: + conn = HTTP2Connection("example.com") + mock_open = mock.mock_open(read_data="foo\r\nbar\r\n") + with mock.patch("builtins.open", mock_open): + conn.sock = mock.MagicMock( + sendall=mock.Mock(return_value=None), + ) + conn._h2_conn._obj.data_to_send = mock.Mock(return_value=b"foo") + conn._h2_conn._obj.send_data = mock.Mock(return_value=None) + conn._h2_conn._obj.get_next_available_stream_id = mock.Mock(return_value=1) + conn._h2_conn._obj.end_stream = mock.Mock(return_value=None) + + with open("foo") as body: + conn.putrequest("GET", "/") + conn.endheaders(message_body=body) + conn.send(body) + + conn._h2_conn._obj.data_to_send.assert_called_with() + conn.sock.sendall.assert_called_with(b"foo") + conn._h2_conn._obj.send_data.assert_called_with( + 1, b"foo\r\nbar\r\n", end_stream=False + ) + conn._h2_conn._obj.end_stream.assert_called_with(1) + + def test_send_file_bytes(self) -> None: + conn = HTTP2Connection("example.com") + mock_open = mock.mock_open(read_data=b"foo\r\nbar\r\n") + with mock.patch("builtins.open", mock_open): + conn.sock = mock.MagicMock( + sendall=mock.Mock(return_value=None), + ) + conn._h2_conn._obj.data_to_send = mock.Mock(return_value=b"foo") + conn._h2_conn._obj.send_data = mock.Mock(return_value=None) + conn._h2_conn._obj.get_next_available_stream_id = mock.Mock(return_value=1) + conn._h2_conn._obj.end_stream = mock.Mock(return_value=None) + + body = open("foo", "rb") + conn.putrequest("GET", "/") + conn.endheaders(message_body=body) + conn.send(body) + + conn._h2_conn._obj.data_to_send.assert_called_with() + conn.sock.sendall.assert_called_with(b"foo") + conn._h2_conn._obj.send_data.assert_called_with( + 1, b"foo\r\nbar\r\n", end_stream=False + ) + conn._h2_conn._obj.end_stream.assert_called_with(1) + + def test_send_invalid_type(self) -> None: + conn = HTTP2Connection("example.com") + conn.putrequest("GET", "/") + with pytest.raises(TypeError): + conn.send(1) + + def test_request_GET(self) -> None: + conn = HTTP2Connection("example.com") + conn.sock = mock.MagicMock( + sendall=mock.Mock(return_value=None), + ) + sendall = conn.sock.sendall + data_to_send = conn._h2_conn._obj.data_to_send = mock.Mock(return_value=b"foo") + send_headers = conn._h2_conn._obj.send_headers = mock.Mock(return_value=None) + conn._h2_conn._obj.send_data = mock.Mock(return_value=None) + conn._h2_conn._obj.get_next_available_stream_id = mock.Mock(return_value=1) + close_connection = conn._h2_conn._obj.close_connection = mock.Mock( + return_value=None + ) + + conn.request("GET", "/") + conn.close() + + data_to_send.assert_called_with() + sendall.assert_called_with(b"foo") + send_headers.assert_called_with( + stream_id=1, + headers=[ + (b":scheme", b"https"), + (b":method", b"GET"), + (b":authority", b"example.com:443"), + (b":path", b"/"), + (b"user-agent", _get_default_user_agent().encode()), + ], + end_stream=True, + ) + + close_connection.assert_called_with() + + def test_request_POST(self) -> None: + conn = HTTP2Connection("example.com") + conn.sock = mock.MagicMock( + sendall=mock.Mock(return_value=None), + ) + sendall = conn.sock.sendall + data_to_send = conn._h2_conn._obj.data_to_send = mock.Mock(return_value=b"foo") + send_headers = conn._h2_conn._obj.send_headers = mock.Mock(return_value=None) + send_data = conn._h2_conn._obj.send_data = mock.Mock(return_value=None) + conn._h2_conn._obj.get_next_available_stream_id = mock.Mock(return_value=1) + close_connection = conn._h2_conn._obj.close_connection = mock.Mock( + return_value=None + ) + + conn.request("POST", "/", body=b"foo") + conn.close() + + data_to_send.assert_called_with() + sendall.assert_called_with(b"foo") + send_headers.assert_called_with( + stream_id=1, + headers=[ + (b":scheme", b"https"), + (b":method", b"POST"), + (b":authority", b"example.com:443"), + (b":path", b"/"), + (b"user-agent", _get_default_user_agent().encode()), + ], + end_stream=False, + ) + send_data.assert_called_with(1, b"foo", end_stream=True) + close_connection.assert_called_with() + + def test_close(self) -> None: + conn = HTTP2Connection("example.com") + conn.sock = mock.MagicMock( + sendall=mock.Mock(side_effect=Exception("foo")), + ) + sendall = conn.sock.sendall + data_to_send = conn._h2_conn._obj.data_to_send = mock.Mock(return_value=b"foo") + close_connection = conn._h2_conn._obj.close_connection = mock.Mock( + return_value=None + ) + + try: + conn.close() + except Exception: + assert False, "Exception was raised" + + close_connection.assert_called_with() + data_to_send.assert_called_with() + sendall.assert_called_with(b"foo") + assert conn._h2_stream is None + assert conn._headers == [] + + def test_request_ignore_chunked(self) -> None: + conn = HTTP2Connection("example.com") + conn.sock = mock.MagicMock( + sendall=mock.Mock(return_value=None), + ) + sendall = conn.sock.sendall + data_to_send = conn._h2_conn._obj.data_to_send = mock.Mock(return_value=b"foo") + send_headers = conn._h2_conn._obj.send_headers = mock.Mock(return_value=None) + conn._h2_conn._obj.send_data = mock.Mock(return_value=None) + conn._h2_conn._obj.get_next_available_stream_id = mock.Mock(return_value=1) + close_connection = conn._h2_conn._obj.close_connection = mock.Mock( + return_value=None + ) + + conn.request("GET", "/", headers={"Transfer-Encoding": "chunked"}, chunked=True) + conn.close() + + data_to_send.assert_called_with() + sendall.assert_called_with(b"foo") + send_headers.assert_called_with( + stream_id=1, + headers=[ + (b":scheme", b"https"), + (b":method", b"GET"), + (b":authority", b"example.com:443"), + (b":path", b"/"), + (b"user-agent", _get_default_user_agent().encode()), + ], + end_stream=True, + ) + + close_connection.assert_called_with() diff --git a/test/test_no_ssl.py b/test/test_no_ssl.py index 7cf6260..e793f79 100644 --- a/test/test_no_ssl.py +++ b/test/test_no_ssl.py @@ -5,83 +5,36 @@ * HTTPS requests must fail with an error that points at the ssl module """ +from __future__ import annotations + import sys +from test import ImportBlocker, ModuleStash import pytest - -class ImportBlocker(object): - """ - Block Imports - - To be placed on ``sys.meta_path``. This ensures that the modules - specified cannot be imported, even if they are a builtin. - """ - - def __init__(self, *namestoblock): - self.namestoblock = namestoblock - - def find_module(self, fullname, path=None): - if fullname in self.namestoblock: - return self - return None - - def load_module(self, fullname): - raise ImportError("import of {0} is blocked".format(fullname)) - - -class ModuleStash(object): - """ - Stashes away previously imported modules - - If we reimport a module the data from coverage is lost, so we reuse the old - modules - """ - - def __init__(self, namespace, modules=sys.modules): - self.namespace = namespace - self.modules = modules - self._data = {} - - def stash(self): - self._data[self.namespace] = self.modules.pop(self.namespace, None) - - for module in list(self.modules.keys()): - if module.startswith(self.namespace + "."): - self._data[module] = self.modules.pop(module) - - def pop(self): - self.modules.pop(self.namespace, None) - - for module in list(self.modules.keys()): - if module.startswith(self.namespace + "."): - self.modules.pop(module) - - self.modules.update(self._data) - - ssl_blocker = ImportBlocker("ssl", "_ssl") module_stash = ModuleStash("urllib3") -class TestWithoutSSL(object): +class TestWithoutSSL: @classmethod - def setup_class(cls): + def setup_class(cls) -> None: sys.modules.pop("ssl", None) sys.modules.pop("_ssl", None) module_stash.stash() sys.meta_path.insert(0, ssl_blocker) - def teardown_class(cls): + @classmethod + def teardown_class(cls) -> None: sys.meta_path.remove(ssl_blocker) module_stash.pop() class TestImportWithoutSSL(TestWithoutSSL): - def test_cannot_import_ssl(self): + def test_cannot_import_ssl(self) -> None: with pytest.raises(ImportError): import ssl # noqa: F401 - def test_import_urllib3(self): + def test_import_urllib3(self) -> None: import urllib3 # noqa: F401 diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py index 61715e9..ab5f203 100644 --- a/test/test_poolmanager.py +++ b/test/test_poolmanager.py @@ -1,19 +1,29 @@ +from __future__ import annotations + import gc import socket from test import resolvesLocalhostFQDN +from unittest import mock +from unittest.mock import MagicMock, patch import pytest -from mock import patch from urllib3 import connection_from_url +from urllib3.connectionpool import HTTPSConnectionPool from urllib3.exceptions import LocationValueError -from urllib3.poolmanager import PoolKey, PoolManager, key_fn_by_scheme +from urllib3.poolmanager import ( + _DEFAULT_BLOCKSIZE, + PoolKey, + PoolManager, + key_fn_by_scheme, +) from urllib3.util import retry, timeout +from urllib3.util.url import Url -class TestPoolManager(object): - @resolvesLocalhostFQDN - def test_same_url(self): +class TestPoolManager: + @resolvesLocalhostFQDN() + def test_same_url(self) -> None: # Convince ourselves that normally we don't get the same object conn1 = connection_from_url("http://localhost:8081/foo") conn2 = connection_from_url("http://localhost:8081/bar") @@ -36,7 +46,7 @@ def test_same_url(self): assert conn1 != conn2 - def test_many_urls(self): + def test_many_urls(self) -> None: urls = [ "http://localhost:8081/foo", "http://www.google.com/mail", @@ -58,7 +68,7 @@ def test_many_urls(self): assert len(connections) == 5 - def test_manager_clear(self): + def test_manager_clear(self) -> None: p = PoolManager(5) p.connection_from_url("http://google.com") @@ -68,26 +78,26 @@ def test_manager_clear(self): assert len(p.pools) == 0 @pytest.mark.parametrize("url", ["http://@", None]) - def test_nohost(self, url): + def test_nohost(self, url: str | None) -> None: p = PoolManager(5) with pytest.raises(LocationValueError): - p.connection_from_url(url=url) + p.connection_from_url(url=url) # type: ignore[arg-type] - def test_contextmanager(self): + def test_contextmanager(self) -> None: with PoolManager(1) as p: p.connection_from_url("http://google.com") assert len(p.pools) == 1 assert len(p.pools) == 0 - def test_http_pool_key_fields(self): + def test_http_pool_key_fields(self) -> None: """Assert the HTTPPoolKey fields are honored when selecting a pool.""" connection_pool_kw = { "timeout": timeout.Timeout(3.14), "retries": retry.Retry(total=6, connect=2), "block": True, - "strict": True, "source_address": "127.0.0.1", + "blocksize": _DEFAULT_BLOCKSIZE + 1, } p = PoolManager() conn_pools = [ @@ -108,19 +118,19 @@ def test_http_pool_key_fields(self): ) assert all(isinstance(key, PoolKey) for key in p.pools.keys()) - def test_https_pool_key_fields(self): + def test_https_pool_key_fields(self) -> None: """Assert the HTTPSPoolKey fields are honored when selecting a pool.""" connection_pool_kw = { "timeout": timeout.Timeout(3.14), "retries": retry.Retry(total=6, connect=2), "block": True, - "strict": True, "source_address": "127.0.0.1", "key_file": "/root/totally_legit.key", "cert_file": "/root/totally_legit.crt", "cert_reqs": "CERT_REQUIRED", "ca_certs": "/root/path_to_pem", "ssl_version": "SSLv23_METHOD", + "blocksize": _DEFAULT_BLOCKSIZE + 1, } p = PoolManager() conn_pools = [ @@ -146,13 +156,13 @@ def test_https_pool_key_fields(self): assert all(pool in conn_pools for pool in dup_pools) assert all(isinstance(key, PoolKey) for key in p.pools.keys()) - def test_default_pool_key_funcs_copy(self): + def test_default_pool_key_funcs_copy(self) -> None: """Assert each PoolManager gets a copy of ``pool_keys_by_scheme``.""" p = PoolManager() assert p.key_fn_by_scheme == p.key_fn_by_scheme assert p.key_fn_by_scheme is not key_fn_by_scheme - def test_pools_keyed_with_from_host(self): + def test_pools_keyed_with_from_host(self) -> None: """Assert pools are still keyed correctly with connection_from_host.""" ssl_kw = { "key_file": "/root/totally_legit.key", @@ -161,7 +171,7 @@ def test_pools_keyed_with_from_host(self): "ca_certs": "/root/path_to_pem", "ssl_version": "SSLv23_METHOD", } - p = PoolManager(5, **ssl_kw) + p = PoolManager(5, **ssl_kw) # type: ignore[arg-type] conns = [p.connection_from_host("example.com", 443, scheme="https")] for k in ssl_kw: @@ -175,7 +185,7 @@ def test_pools_keyed_with_from_host(self): if i != j ) - def test_https_connection_from_url_case_insensitive(self): + def test_https_connection_from_url_case_insensitive(self) -> None: """Assert scheme case is ignored when pooling HTTPS connections.""" p = PoolManager() pool = p.connection_from_url("https://example.com/") @@ -185,7 +195,7 @@ def test_https_connection_from_url_case_insensitive(self): assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys()) - def test_https_connection_from_host_case_insensitive(self): + def test_https_connection_from_host_case_insensitive(self) -> None: """Assert scheme case is ignored when getting the https key class.""" p = PoolManager() pool = p.connection_from_host("example.com", scheme="https") @@ -195,7 +205,7 @@ def test_https_connection_from_host_case_insensitive(self): assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys()) - def test_https_connection_from_context_case_insensitive(self): + def test_https_connection_from_context_case_insensitive(self) -> None: """Assert scheme case is ignored when getting the https key class.""" p = PoolManager() context = {"scheme": "https", "host": "example.com", "port": "443"} @@ -207,7 +217,7 @@ def test_https_connection_from_context_case_insensitive(self): assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys()) - def test_http_connection_from_url_case_insensitive(self): + def test_http_connection_from_url_case_insensitive(self) -> None: """Assert scheme case is ignored when pooling HTTP connections.""" p = PoolManager() pool = p.connection_from_url("http://example.com/") @@ -217,7 +227,7 @@ def test_http_connection_from_url_case_insensitive(self): assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys()) - def test_http_connection_from_host_case_insensitive(self): + def test_http_connection_from_host_case_insensitive(self) -> None: """Assert scheme case is ignored when getting the https key class.""" p = PoolManager() pool = p.connection_from_host("example.com", scheme="http") @@ -227,16 +237,17 @@ def test_http_connection_from_host_case_insensitive(self): assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys()) - def test_assert_hostname_and_fingerprint_flag(self): + def test_assert_hostname_and_fingerprint_flag(self) -> None: """Assert that pool manager can accept hostname and fingerprint flags.""" fingerprint = "92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A" p = PoolManager(assert_hostname=True, assert_fingerprint=fingerprint) pool = p.connection_from_url("https://example.com/") assert 1 == len(p.pools) + assert isinstance(pool, HTTPSConnectionPool) assert pool.assert_hostname assert fingerprint == pool.assert_fingerprint - def test_http_connection_from_context_case_insensitive(self): + def test_http_connection_from_context_case_insensitive(self) -> None: """Assert scheme case is ignored when getting the https key class.""" p = PoolManager() context = {"scheme": "http", "host": "example.com", "port": "8080"} @@ -248,11 +259,61 @@ def test_http_connection_from_context_case_insensitive(self): assert pool is other_pool assert all(isinstance(key, PoolKey) for key in p.pools.keys()) - def test_custom_pool_key(self): + @patch("urllib3.poolmanager.PoolManager.connection_from_host") + def test_deprecated_no_scheme(self, connection_from_host: mock.MagicMock) -> None: + # Don't actually make a network connection, just verify the DeprecationWarning + connection_from_host.side_effect = ConnectionError("Not attempting connection") + p = PoolManager() + + with pytest.warns(DeprecationWarning) as records: + with pytest.raises(ConnectionError): + p.request(method="GET", url="evil.com://good.com") + + msg = ( + "URLs without a scheme (ie 'https://') are deprecated and will raise an error " + "in a future version of urllib3. To avoid this DeprecationWarning ensure all URLs " + "start with 'https://' or 'http://'. Read more in this issue: " + "https://github.com/urllib3/urllib3/issues/2920" + ) + + assert len(records) == 1 + assert isinstance(records[0].message, DeprecationWarning) + assert records[0].message.args[0] == msg + + @patch("urllib3.poolmanager.PoolManager.connection_from_pool_key") + def test_connection_from_context_strict_param( + self, connection_from_pool_key: mock.MagicMock + ) -> None: + p = PoolManager() + context = { + "scheme": "http", + "host": "example.com", + "port": 8080, + "strict": True, + } + with pytest.warns(DeprecationWarning) as records: + p.connection_from_context(context) + + msg = ( + "The 'strict' parameter is no longer needed on Python 3+. " + "This will raise an error in urllib3 v2.1.0." + ) + record = records[0] + assert isinstance(record.message, Warning) + assert record.message.args[0] == msg + + _, kwargs = connection_from_pool_key.call_args + assert kwargs["request_context"] == { + "scheme": "http", + "host": "example.com", + "port": 8080, + } + + def test_custom_pool_key(self) -> None: """Assert it is possible to define a custom key function.""" p = PoolManager(10) - p.key_fn_by_scheme["http"] = lambda x: tuple(x["key"]) + p.key_fn_by_scheme["http"] = lambda x: tuple(x["key"]) # type: ignore[assignment] pool1 = p.connection_from_url( "http://example.com", pool_kwargs={"key": "value"} ) @@ -267,43 +328,39 @@ def test_custom_pool_key(self): assert pool1 is pool3 assert pool1 is not pool2 - def test_override_pool_kwargs_url(self): + def test_override_pool_kwargs_url(self) -> None: """Assert overriding pool kwargs works with connection_from_url.""" - p = PoolManager(strict=True) - pool_kwargs = {"strict": False, "retries": 100, "block": True} + p = PoolManager() + pool_kwargs = {"retries": 100, "block": True} default_pool = p.connection_from_url("http://example.com/") override_pool = p.connection_from_url( "http://example.com/", pool_kwargs=pool_kwargs ) - assert default_pool.strict assert retry.Retry.DEFAULT == default_pool.retries assert not default_pool.block - assert not override_pool.strict assert 100 == override_pool.retries assert override_pool.block - def test_override_pool_kwargs_host(self): + def test_override_pool_kwargs_host(self) -> None: """Assert overriding pool kwargs works with connection_from_host""" - p = PoolManager(strict=True) - pool_kwargs = {"strict": False, "retries": 100, "block": True} + p = PoolManager() + pool_kwargs = {"retries": 100, "block": True} default_pool = p.connection_from_host("example.com", scheme="http") override_pool = p.connection_from_host( "example.com", scheme="http", pool_kwargs=pool_kwargs ) - assert default_pool.strict assert retry.Retry.DEFAULT == default_pool.retries assert not default_pool.block - assert not override_pool.strict assert 100 == override_pool.retries assert override_pool.block - def test_pool_kwargs_socket_options(self): + def test_pool_kwargs_socket_options(self) -> None: """Assert passing socket options works with connection_from_host""" p = PoolManager(socket_options=[]) override_opts = [ @@ -320,37 +377,57 @@ def test_pool_kwargs_socket_options(self): assert default_pool.conn_kw["socket_options"] == [] assert override_pool.conn_kw["socket_options"] == override_opts - def test_merge_pool_kwargs(self): + def test_merge_pool_kwargs(self) -> None: """Assert _merge_pool_kwargs works in the happy case""" - p = PoolManager(strict=True) + p = PoolManager(retries=100) merged = p._merge_pool_kwargs({"new_key": "value"}) - assert {"strict": True, "new_key": "value"} == merged + assert {"retries": 100, "new_key": "value"} == merged - def test_merge_pool_kwargs_none(self): + def test_merge_pool_kwargs_none(self) -> None: """Assert false-y values to _merge_pool_kwargs result in defaults""" - p = PoolManager(strict=True) + p = PoolManager(retries=100) merged = p._merge_pool_kwargs({}) assert p.connection_pool_kw == merged merged = p._merge_pool_kwargs(None) assert p.connection_pool_kw == merged - def test_merge_pool_kwargs_remove_key(self): + def test_merge_pool_kwargs_remove_key(self) -> None: """Assert keys can be removed with _merge_pool_kwargs""" - p = PoolManager(strict=True) - merged = p._merge_pool_kwargs({"strict": None}) - assert "strict" not in merged + p = PoolManager(retries=100) + merged = p._merge_pool_kwargs({"retries": None}) + assert "retries" not in merged - def test_merge_pool_kwargs_invalid_key(self): + def test_merge_pool_kwargs_invalid_key(self) -> None: """Assert removing invalid keys with _merge_pool_kwargs doesn't break""" - p = PoolManager(strict=True) + p = PoolManager(retries=100) merged = p._merge_pool_kwargs({"invalid_key": None}) assert p.connection_pool_kw == merged - def test_pool_manager_no_url_absolute_form(self): + def test_pool_manager_no_url_absolute_form(self) -> None: """Valides we won't send a request with absolute form without a proxy""" - p = PoolManager(strict=True) - assert p._proxy_requires_url_absolute_form("http://example.com") is False - assert p._proxy_requires_url_absolute_form("https://example.com") is False + p = PoolManager() + assert p._proxy_requires_url_absolute_form(Url("http://example.com")) is False + assert p._proxy_requires_url_absolute_form(Url("https://example.com")) is False + + @pytest.mark.parametrize( + "input_blocksize,expected_blocksize", + [ + (_DEFAULT_BLOCKSIZE, _DEFAULT_BLOCKSIZE), + (None, _DEFAULT_BLOCKSIZE), + (8192, 8192), + ], + ) + def test_poolmanager_blocksize( + self, input_blocksize: int, expected_blocksize: int + ) -> None: + """Assert PoolManager sets blocksize properly""" + p = PoolManager() + + pool_blocksize = p.connection_from_url( + "http://example.com", {"blocksize": input_blocksize} + ) + assert pool_blocksize.conn_kw["blocksize"] == expected_blocksize + assert pool_blocksize._get_conn().blocksize == expected_blocksize @pytest.mark.parametrize( "url", @@ -362,7 +439,9 @@ def test_pool_manager_no_url_absolute_form(self): ], ) @patch("urllib3.util.connection.create_connection") - def test_e2e_connect_to_ipv6_scoped(self, create_connection, url): + def test_e2e_connect_to_ipv6_scoped( + self, create_connection: MagicMock, url: str + ) -> None: """Checks that IPv6 scoped addresses are properly handled end-to-end. This is not strictly speaking a pool manager unit test - this test @@ -376,7 +455,21 @@ def test_e2e_connect_to_ipv6_scoped(self, create_connection, url): assert create_connection.call_args[0][0] == ("a::b%zone", 80) - def test_thread_safty(self): + @patch("urllib3.connection.ssl_wrap_socket") + @patch("urllib3.util.connection.create_connection") + def test_e2e_connect_to_ipv6_scoped_tls( + self, create_connection: MagicMock, ssl_wrap_socket: MagicMock + ) -> None: + p = PoolManager() + conn_pool = p.connection_from_url( + "https://[a::b%zone]", pool_kwargs={"assert_hostname": False} + ) + conn = conn_pool._get_conn() + conn.connect() + + assert ssl_wrap_socket.call_args[1]["server_hostname"] == "a::b" + + def test_thread_safty(self) -> None: pool_manager = PoolManager(num_pools=2) # thread 1 gets a pool for host x diff --git a/test/test_proxymanager.py b/test/test_proxymanager.py index 7f1c396..140ca9f 100644 --- a/test/test_proxymanager.py +++ b/test/test_proxymanager.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from urllib3.exceptions import MaxRetryError, NewConnectionError, ProxyError @@ -8,11 +10,11 @@ from .port_helpers import find_unused_port -class TestProxyManager(object): +class TestProxyManager: @pytest.mark.parametrize("proxy_scheme", ["http", "https"]) - def test_proxy_headers(self, proxy_scheme): + def test_proxy_headers(self, proxy_scheme: str) -> None: url = "http://pypi.org/project/urllib3/" - proxy_url = "{}://something:1234".format(proxy_scheme) + proxy_url = f"{proxy_scheme}://something:1234" with ProxyManager(proxy_url) as p: # Verify default headers default_headers = {"Accept": "*/*", "Host": "pypi.org"} @@ -39,19 +41,21 @@ def test_proxy_headers(self, proxy_scheme): assert headers == expected_headers - def test_default_port(self): + def test_default_port(self) -> None: with ProxyManager("http://something") as p: + assert p.proxy is not None assert p.proxy.port == 80 with ProxyManager("https://something") as p: + assert p.proxy is not None assert p.proxy.port == 443 - def test_invalid_scheme(self): + def test_invalid_scheme(self) -> None: with pytest.raises(AssertionError): ProxyManager("invalid://host/p") with pytest.raises(ValueError): ProxyManager("invalid://host/p") - def test_proxy_tunnel(self): + def test_proxy_tunnel(self) -> None: http_url = parse_url("http://example.com") https_url = parse_url("https://example.com") with ProxyManager("http://proxy:8080") as p: @@ -66,16 +70,18 @@ def test_proxy_tunnel(self): assert p._proxy_requires_url_absolute_form(http_url) assert p._proxy_requires_url_absolute_form(https_url) - def test_proxy_connect_retry(self): + def test_proxy_connect_retry(self) -> None: retry = Retry(total=None, connect=False) port = find_unused_port() - with ProxyManager("http://localhost:{}".format(port)) as p: + with ProxyManager(f"http://localhost:{port}") as p: with pytest.raises(ProxyError) as ei: p.urlopen("HEAD", url="http://localhost/", retries=retry) assert isinstance(ei.value.original_error, NewConnectionError) retry = Retry(total=None, connect=2) - with ProxyManager("http://localhost:{}".format(port)) as p: - with pytest.raises(MaxRetryError) as ei: + with ProxyManager(f"http://localhost:{port}") as p: + with pytest.raises(MaxRetryError) as ei1: p.urlopen("HEAD", url="http://localhost/", retries=retry) - assert isinstance(ei.value.reason.original_error, NewConnectionError) + assert ei1.value.reason is not None + assert isinstance(ei1.value.reason, ProxyError) + assert isinstance(ei1.value.reason.original_error, NewConnectionError) diff --git a/test/test_queue_monkeypatch.py b/test/test_queue_monkeypatch.py index f8420a0..508136d 100644 --- a/test/test_queue_monkeypatch.py +++ b/test/test_queue_monkeypatch.py @@ -1,11 +1,12 @@ -from __future__ import absolute_import +from __future__ import annotations + +import queue +from unittest import mock -import mock import pytest from urllib3 import HTTPConnectionPool from urllib3.exceptions import EmptyPoolError -from urllib3.packages.six.moves import queue class BadError(Exception): @@ -13,16 +14,14 @@ class BadError(Exception): This should not be raised. """ - pass - -class TestMonkeypatchResistance(object): +class TestMonkeypatchResistance: """ Test that connection pool works even with a monkey patched Queue module, see obspy/obspy#1599, psf/requests#3742, urllib3/urllib3#1061. """ - def test_queue_monkeypatching(self): + def test_queue_monkeypatching(self) -> None: with mock.patch.object(queue, "Empty", BadError): with HTTPConnectionPool(host="localhost", block=True) as http: http._get_conn() diff --git a/test/test_request.py b/test/test_request.py deleted file mode 100644 index 1db819d..0000000 --- a/test/test_request.py +++ /dev/null @@ -1,26 +0,0 @@ -import types - -import pytest - -import urllib3 -from urllib3.packages import six - - -@pytest.mark.skipif( - six.PY2, - reason="This behaviour isn't added when running urllib3 in Python 2", -) -class TestRequestImport(object): - def test_request_import_error(self): - """Ensure an appropriate error is raised to the user - if they try and run urllib3.request()""" - with pytest.raises(TypeError) as exc_info: - urllib3.request(1, a=2) - assert "urllib3 v2" in exc_info.value.args[0] - - def test_request_module_properties(self): - """Ensure properties of the overridden request module - are still present""" - assert isinstance(urllib3.request, types.ModuleType) - expected_attrs = {"RequestMethods", "encode_multipart_formdata", "urlencode"} - assert set(dir(urllib3.request)).issuperset(expected_attrs) diff --git a/test/test_response.py b/test/test_response.py index e09e385..3eec0ab 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -1,20 +1,22 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations import contextlib -import re +import http.client as httplib import socket import ssl +import typing import zlib from base64 import b64decode +from http.client import IncompleteRead as httplib_IncompleteRead from io import BufferedReader, BytesIO, TextIOWrapper -from test import onlyBrotlipy +from test import onlyBrotli, onlyZstd +from unittest import mock -import mock import pytest -import six -from urllib3._collections import HTTPHeaderDict +from urllib3 import HTTPHeaderDict from urllib3.exceptions import ( + BodyNotHttplibCompatible, DecodeError, IncompleteRead, InvalidChunkLength, @@ -22,15 +24,102 @@ ProtocolError, ResponseNotChunked, SSLError, - httplib_IncompleteRead, ) -from urllib3.packages.six.moves import http_client as httplib -from urllib3.response import HTTPResponse, brotli +from urllib3.response import ( # type: ignore[attr-defined] + BaseHTTPResponse, + BytesQueueBuffer, + HTTPResponse, + brotli, +) from urllib3.util.response import is_fp_closed from urllib3.util.retry import RequestHistory, Retry + +class TestBytesQueueBuffer: + def test_single_chunk(self) -> None: + buffer = BytesQueueBuffer() + assert len(buffer) == 0 + with pytest.raises(RuntimeError, match="buffer is empty"): + assert buffer.get(10) + + assert buffer.get(0) == b"" + + buffer.put(b"foo") + with pytest.raises(ValueError, match="n should be > 0"): + buffer.get(-1) + + assert buffer.get(1) == b"f" + assert buffer.get(2) == b"oo" + with pytest.raises(RuntimeError, match="buffer is empty"): + assert buffer.get(10) + + def test_read_too_much(self) -> None: + buffer = BytesQueueBuffer() + buffer.put(b"foo") + assert buffer.get(100) == b"foo" + + def test_multiple_chunks(self) -> None: + buffer = BytesQueueBuffer() + buffer.put(b"foo") + buffer.put(b"bar") + buffer.put(b"baz") + assert len(buffer) == 9 + + assert buffer.get(1) == b"f" + assert len(buffer) == 8 + assert buffer.get(4) == b"ooba" + assert len(buffer) == 4 + assert buffer.get(4) == b"rbaz" + assert len(buffer) == 0 + + def test_get_all_empty(self) -> None: + q = BytesQueueBuffer() + assert q.get_all() == b"" + assert len(q) == 0 + + def test_get_all_single(self) -> None: + q = BytesQueueBuffer() + q.put(b"a") + assert q.get_all() == b"a" + assert len(q) == 0 + + def test_get_all_many(self) -> None: + q = BytesQueueBuffer() + q.put(b"a") + q.put(b"b") + q.put(b"c") + assert q.get_all() == b"abc" + assert len(q) == 0 + + @pytest.mark.parametrize( + "get_func", + (lambda b: b.get(len(b)), lambda b: b.get_all()), + ids=("get", "get_all"), + ) + @pytest.mark.limit_memory( + "12.5 MB", current_thread_only=True + ) # assert that we're not doubling memory usagelimit_mem + def test_memory_usage( + self, get_func: typing.Callable[[BytesQueueBuffer], str] + ) -> None: + # Allocate 10 1MiB chunks + buffer = BytesQueueBuffer() + for i in range(10): + # This allocates 2MiB, putting the max at around 12MiB. Not sure why. + buffer.put(bytes(2**20)) + + assert len(get_func(buffer)) == 10 * 2**20 + + @pytest.mark.limit_memory("10.01 MB", current_thread_only=True) + def test_get_all_memory_usage_single_chunk(self) -> None: + buffer = BytesQueueBuffer() + chunk = bytes(10 * 2**20) # 10 MiB + buffer.put(chunk) + assert buffer.get_all() is chunk + + # A known random (i.e, not-too-compressible) payload generated with: -# "".join(random.choice(string.printable) for i in xrange(512)) +# "".join(random.choice(string.printable) for i in range(512)) # .encode("zlib").encode("base64") # Randomness in tests == bad, and fixing a seed may not be sufficient. ZLIB_PAYLOAD = b64decode( @@ -48,14 +137,14 @@ @pytest.fixture -def sock(): +def sock() -> typing.Generator[socket.socket]: s = socket.socket() yield s s.close() -class TestLegacyResponse(object): - def test_getheaders(self): +class TestLegacyResponse: + def test_getheaders(self) -> None: headers = {"host": "example.com"} r = HTTPResponse(headers=headers) with pytest.warns( @@ -64,7 +153,7 @@ def test_getheaders(self): ): assert r.getheaders() == HTTPHeaderDict(headers) - def test_getheader(self): + def test_getheader(self) -> None: headers = {"host": "example.com"} r = HTTPResponse(headers=headers) with pytest.warns( @@ -74,21 +163,31 @@ def test_getheader(self): assert r.getheader("host") == "example.com" -class TestResponse(object): - def test_cache_content(self): - r = HTTPResponse("foo") - assert r.data == "foo" - assert r._body == "foo" +class TestResponse: + def test_cache_content(self) -> None: + r = HTTPResponse(b"foo") + assert r._body == b"foo" + assert r.data == b"foo" + assert r._body == b"foo" + + def test_cache_content_preload_false(self) -> None: + fp = BytesIO(b"foo") + r = HTTPResponse(fp, preload_content=False) + + assert not r._body + assert r.data == b"foo" + assert r._body == b"foo" # type: ignore[comparison-overlap] + assert r.data == b"foo" - def test_default(self): + def test_default(self) -> None: r = HTTPResponse() assert r.data is None - def test_none(self): - r = HTTPResponse(None) + def test_none(self) -> None: + r = HTTPResponse(None) # type: ignore[arg-type] assert r.data is None - def test_preload(self): + def test_preload(self) -> None: fp = BytesIO(b"foo") r = HTTPResponse(fp, preload_content=True) @@ -96,7 +195,7 @@ def test_preload(self): assert fp.tell() == len(b"foo") assert r.data == b"foo" - def test_no_preload(self): + def test_no_preload(self) -> None: fp = BytesIO(b"foo") r = HTTPResponse(fp, preload_content=False) @@ -105,21 +204,76 @@ def test_no_preload(self): assert r.data == b"foo" assert fp.tell() == len(b"foo") - def test_decode_bad_data(self): + def test_no_shutdown(self) -> None: + r = HTTPResponse() + with pytest.raises( + ValueError, match="Cannot shutdown socket as self._sock_shutdown is not set" + ): + r.shutdown() + + def test_decode_bad_data(self) -> None: fp = BytesIO(b"\x00" * 10) with pytest.raises(DecodeError): HTTPResponse(fp, headers={"content-encoding": "deflate"}) - def test_reference_read(self): + def test_reference_read(self) -> None: fp = BytesIO(b"foo") r = HTTPResponse(fp, preload_content=False) + assert r.read(0) == b"" assert r.read(1) == b"f" assert r.read(2) == b"oo" assert r.read() == b"" assert r.read() == b"" - def test_decode_deflate(self): + @pytest.mark.parametrize("read_args", ((), (None,), (-1,))) + def test_reference_read_until_eof(self, read_args: tuple[typing.Any, ...]) -> None: + fp = BytesIO(b"foo") + r = HTTPResponse(fp, preload_content=False) + assert r.read(*read_args) == b"foo" + + def test_reference_read1(self) -> None: + fp = BytesIO(b"foobar") + r = HTTPResponse(fp, preload_content=False) + + assert r.read1(0) == b"" + assert r.read1(1) == b"f" + assert r.read1(2) == b"oo" + assert r.read1() == b"bar" + assert r.read1() == b"" + + @pytest.mark.parametrize("read1_args", ((), (None,), (-1,))) + def test_reference_read1_without_limit( + self, read1_args: tuple[typing.Any, ...] + ) -> None: + fp = BytesIO(b"foo") + r = HTTPResponse(fp, preload_content=False) + assert r.read1(*read1_args) == b"foo" + + def test_reference_read1_nodecode(self) -> None: + fp = BytesIO(b"foobar") + r = HTTPResponse(fp, preload_content=False, decode_content=False) + + assert r.read1(0) == b"" + assert r.read1(1) == b"f" + assert r.read1(2) == b"oo" + assert r.read1() == b"bar" + assert r.read1() == b"" + + def test_decoding_read1(self) -> None: + data = zlib.compress(b"foobar") + + fp = BytesIO(data) + r = HTTPResponse( + fp, headers={"content-encoding": "deflate"}, preload_content=False + ) + + assert r.read1(1) == b"f" + assert r.read1(2) == b"oo" + assert r.read1() == b"bar" + assert r.read1() == b"" + + def test_decode_deflate(self) -> None: data = zlib.compress(b"foo") fp = BytesIO(data) @@ -127,7 +281,7 @@ def test_decode_deflate(self): assert r.data == b"foo" - def test_decode_deflate_case_insensitve(self): + def test_decode_deflate_case_insensitve(self) -> None: data = zlib.compress(b"foo") fp = BytesIO(data) @@ -135,7 +289,7 @@ def test_decode_deflate_case_insensitve(self): assert r.data == b"foo" - def test_chunked_decoding_deflate(self): + def test_chunked_decoding_deflate(self) -> None: data = zlib.compress(b"foo") fp = BytesIO(data) @@ -143,17 +297,12 @@ def test_chunked_decoding_deflate(self): fp, headers={"content-encoding": "deflate"}, preload_content=False ) - assert r.read(3) == b"" - # Buffer in case we need to switch to the raw stream - assert r._decoder._data is not None assert r.read(1) == b"f" - # Now that we've decoded data, we just stream through the decoder - assert r._decoder._data is None assert r.read(2) == b"oo" assert r.read() == b"" assert r.read() == b"" - def test_chunked_decoding_deflate2(self): + def test_chunked_decoding_deflate2(self) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS) data = compress.compress(b"foo") data += compress.flush() @@ -163,31 +312,28 @@ def test_chunked_decoding_deflate2(self): fp, headers={"content-encoding": "deflate"}, preload_content=False ) - assert r.read(1) == b"" assert r.read(1) == b"f" - # Once we've decoded data, we just stream to the decoder; no buffering - assert r._decoder._data is None assert r.read(2) == b"oo" assert r.read() == b"" assert r.read() == b"" - def test_chunked_decoding_gzip(self): + @pytest.mark.parametrize("content_encoding", ["gzip", "x-gzip"]) + def test_chunked_decoding_gzip(self, content_encoding: str) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(b"foo") data += compress.flush() fp = BytesIO(data) r = HTTPResponse( - fp, headers={"content-encoding": "gzip"}, preload_content=False + fp, headers={"content-encoding": content_encoding}, preload_content=False ) - assert r.read(11) == b"" assert r.read(1) == b"f" assert r.read(2) == b"oo" assert r.read() == b"" assert r.read() == b"" - def test_decode_gzip_multi_member(self): + def test_decode_gzip_multi_member(self) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(b"foo") data += compress.flush() @@ -198,12 +344,12 @@ def test_decode_gzip_multi_member(self): assert r.data == b"foofoofoo" - def test_decode_gzip_error(self): + def test_decode_gzip_error(self) -> None: fp = BytesIO(b"foo") with pytest.raises(DecodeError): HTTPResponse(fp, headers={"content-encoding": "gzip"}) - def test_decode_gzip_swallow_garbage(self): + def test_decode_gzip_swallow_garbage(self) -> None: # When data comes from multiple calls to read(), data after # the first zlib error (here triggered by garbage) should be # ignored. @@ -224,7 +370,7 @@ def test_decode_gzip_swallow_garbage(self): assert ret == b"foofoofoo" - def test_chunked_decoding_gzip_swallow_garbage(self): + def test_chunked_decoding_gzip_swallow_garbage(self) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(b"foo") data += compress.flush() @@ -235,16 +381,16 @@ def test_chunked_decoding_gzip_swallow_garbage(self): assert r.data == b"foofoofoo" - @onlyBrotlipy() - def test_decode_brotli(self): + @onlyBrotli() + def test_decode_brotli(self) -> None: data = brotli.compress(b"foo") fp = BytesIO(data) r = HTTPResponse(fp, headers={"content-encoding": "br"}) assert r.data == b"foo" - @onlyBrotlipy() - def test_chunked_decoding_brotli(self): + @onlyBrotli() + def test_chunked_decoding_brotli(self) -> None: data = brotli.compress(b"foobarbaz") fp = BytesIO(data) @@ -257,13 +403,145 @@ def test_chunked_decoding_brotli(self): break assert ret == b"foobarbaz" - @onlyBrotlipy() - def test_decode_brotli_error(self): + @onlyBrotli() + def test_decode_brotli_error(self) -> None: fp = BytesIO(b"foo") with pytest.raises(DecodeError): HTTPResponse(fp, headers={"content-encoding": "br"}) - def test_multi_decoding_deflate_deflate(self): + @onlyZstd() + def test_decode_zstd(self) -> None: + import zstandard as zstd + + data = zstd.compress(b"foo") + + fp = BytesIO(data) + r = HTTPResponse(fp, headers={"content-encoding": "zstd"}) + assert r.data == b"foo" + + @onlyZstd() + def test_decode_multiframe_zstd(self) -> None: + import zstandard as zstd + + data = ( + # Zstandard frame + zstd.compress(b"foo") + # skippable frame (must be ignored) + + bytes.fromhex( + "50 2A 4D 18" # Magic_Number (little-endian) + "07 00 00 00" # Frame_Size (little-endian) + "00 00 00 00 00 00 00" # User_Data + ) + # Zstandard frame + + zstd.compress(b"bar") + ) + + fp = BytesIO(data) + r = HTTPResponse(fp, headers={"content-encoding": "zstd"}) + assert r.data == b"foobar" + + @onlyZstd() + def test_chunked_decoding_zstd(self) -> None: + import zstandard as zstd + + data = zstd.compress(b"foobarbaz") + + fp = BytesIO(data) + r = HTTPResponse( + fp, headers={"content-encoding": "zstd"}, preload_content=False + ) + + ret = b"" + + for _ in range(100): + ret += r.read(1) + if r.closed: + break + assert ret == b"foobarbaz" + + decode_param_set = [ + b"foo", + b"x" * 100, + ] + + @onlyZstd() + @pytest.mark.parametrize("data", decode_param_set) + def test_decode_zstd_error(self, data: bytes) -> None: + fp = BytesIO(data) + + with pytest.raises(DecodeError): + HTTPResponse(fp, headers={"content-encoding": "zstd"}) + + @onlyZstd() + @pytest.mark.parametrize("data", decode_param_set) + def test_decode_zstd_incomplete_preload_content(self, data: bytes) -> None: + import zstandard as zstd + + data = zstd.compress(data) + fp = BytesIO(data[:-1]) + + with pytest.raises(DecodeError): + HTTPResponse(fp, headers={"content-encoding": "zstd"}) + + @onlyZstd() + @pytest.mark.parametrize("data", decode_param_set) + def test_decode_zstd_incomplete_read(self, data: bytes) -> None: + import zstandard as zstd + + data = zstd.compress(data) + fp = BytesIO(data[:-1]) # shorten the data to trigger DecodeError + + # create response object without(!) reading/decoding the content + r = HTTPResponse( + fp, headers={"content-encoding": "zstd"}, preload_content=False + ) + + # read/decode, expecting DecodeError + with pytest.raises(DecodeError): + r.read(decode_content=True) + + @onlyZstd() + @pytest.mark.parametrize("data", decode_param_set) + def test_decode_zstd_incomplete_read1(self, data: bytes) -> None: + import zstandard as zstd + + data = zstd.compress(data) + fp = BytesIO(data[:-1]) + + r = HTTPResponse( + fp, headers={"content-encoding": "zstd"}, preload_content=False + ) + + # read/decode via read1(!), expecting DecodeError + with pytest.raises(DecodeError): + amt_decoded = 0 + # loop, as read1() may return just partial data + while amt_decoded < len(data): + part = r.read1(decode_content=True) + amt_decoded += len(part) + + @onlyZstd() + @pytest.mark.parametrize("data", decode_param_set) + def test_decode_zstd_read1(self, data: bytes) -> None: + import zstandard as zstd + + encoded_data = zstd.compress(data) + fp = BytesIO(encoded_data) + + r = HTTPResponse( + fp, headers={"content-encoding": "zstd"}, preload_content=False + ) + + amt_decoded = 0 + decoded_data = b"" + # loop, as read1() may return just partial data + while amt_decoded < len(data): + part = r.read1(decode_content=True) + amt_decoded += len(part) + decoded_data += part + assert decoded_data == data + + def test_multi_decoding_deflate_deflate(self) -> None: data = zlib.compress(zlib.compress(b"foo")) fp = BytesIO(data) @@ -271,7 +549,7 @@ def test_multi_decoding_deflate_deflate(self): assert r.data == b"foo" - def test_multi_decoding_deflate_gzip(self): + def test_multi_decoding_deflate_gzip(self) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(zlib.compress(b"foo")) data += compress.flush() @@ -281,7 +559,7 @@ def test_multi_decoding_deflate_gzip(self): assert r.data == b"foo" - def test_multi_decoding_gzip_gzip(self): + def test_multi_decoding_gzip_gzip(self) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(b"foo") data += compress.flush() @@ -295,12 +573,49 @@ def test_multi_decoding_gzip_gzip(self): assert r.data == b"foo" - def test_body_blob(self): + def test_read_multi_decoding_deflate_deflate(self) -> None: + msg = b"foobarbaz" * 42 + data = zlib.compress(zlib.compress(msg)) + + fp = BytesIO(data) + r = HTTPResponse( + fp, headers={"content-encoding": "deflate, deflate"}, preload_content=False + ) + + assert r.read(3) == b"foo" + assert r.read(3) == b"bar" + assert r.read(3) == b"baz" + assert r.read(9) == b"foobarbaz" + assert r.read(9 * 3) == b"foobarbaz" * 3 + assert r.read(9 * 37) == b"foobarbaz" * 37 + assert r.read() == b"" + + def test_body_blob(self) -> None: resp = HTTPResponse(b"foo") assert resp.data == b"foo" assert resp.closed - def test_io(self, sock): + @pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") + def test_base_io(self) -> None: + resp = BaseHTTPResponse( + status=200, + version=11, + version_string="HTTP/1.1", + reason=None, + decode_content=False, + request_url=None, + ) + + assert not resp.closed + assert not resp.readable() + assert not resp.writable() + + with pytest.raises(NotImplementedError): + resp.read() + with pytest.raises(NotImplementedError): + resp.close() + + def test_io(self, sock: socket.socket) -> None: fp = BytesIO(b"foo") resp = HTTPResponse(fp, preload_content=False) @@ -336,14 +651,15 @@ def test_io(self, sock): with pytest.raises(IOError): resp3.fileno() - def test_io_closed_consistently(self, sock): + def test_io_closed_consistently_by_read(self, sock: socket.socket) -> None: try: hlr = httplib.HTTPResponse(sock) - hlr.fp = BytesIO(b"foo") - hlr.chunked = 0 + hlr.fp = BytesIO(b"foo") # type: ignore[assignment] + hlr.chunked = 0 # type: ignore[assignment] hlr.length = 3 with HTTPResponse(hlr, preload_content=False) as resp: assert not resp.closed + assert resp._fp is not None assert not resp._fp.isclosed() assert not is_fp_closed(resp._fp) assert not resp.isclosed() @@ -355,10 +671,103 @@ def test_io_closed_consistently(self, sock): finally: hlr.close() - def test_io_bufferedreader(self): + @pytest.mark.parametrize("read_amt", (None, 3)) + @pytest.mark.parametrize("length_known", (True, False)) + def test_io_closed_consistently_by_read1( + self, sock: socket.socket, length_known: bool, read_amt: int | None + ) -> None: + with httplib.HTTPResponse(sock) as hlr: + hlr.fp = BytesIO(b"foo") # type: ignore[assignment] + hlr.chunked = 0 # type: ignore[assignment] + hlr.length = 3 if length_known else None + with HTTPResponse(hlr, preload_content=False) as resp: + if length_known: + resp.length_remaining = 3 + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + resp.read1(read_amt) + # If content length is unknown, IO is not closed until + # the next read returning zero bytes. + if not length_known: + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + resp.read1(read_amt) + assert resp.closed + assert resp._fp.isclosed() + assert is_fp_closed(resp._fp) + assert resp.isclosed() + + @pytest.mark.parametrize("length_known", (True, False)) + def test_io_not_closed_until_all_data_is_read( + self, sock: socket.socket, length_known: bool + ) -> None: + with httplib.HTTPResponse(sock) as hlr: + hlr.fp = BytesIO(b"foo") # type: ignore[assignment] + hlr.chunked = 0 # type: ignore[assignment] + length_remaining = 3 + hlr.length = length_remaining if length_known else None + with HTTPResponse(hlr, preload_content=False) as resp: + if length_known: + resp.length_remaining = length_remaining + while length_remaining: + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + data = resp.read(1) + assert len(data) == 1 + length_remaining -= 1 + # If content length is unknown, IO is not closed until + # the next read returning zero bytes. + if not length_known: + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + data = resp.read(1) + assert len(data) == 0 + assert resp.closed + assert resp._fp.isclosed() # type: ignore[union-attr] + assert is_fp_closed(resp._fp) + assert resp.isclosed() + + @pytest.mark.parametrize("length_known", (True, False)) + def test_io_not_closed_after_requesting_0_bytes( + self, sock: socket.socket, length_known: bool + ) -> None: + with httplib.HTTPResponse(sock) as hlr: + hlr.fp = BytesIO(b"foo") # type: ignore[assignment] + hlr.chunked = 0 # type: ignore[assignment] + length_remaining = 3 + hlr.length = length_remaining if length_known else None + with HTTPResponse(hlr, preload_content=False) as resp: + if length_known: + resp.length_remaining = length_remaining + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + data = resp.read(0) + assert data == b"" + assert not resp.closed + assert resp._fp is not None + assert not resp._fp.isclosed() + assert not is_fp_closed(resp._fp) + assert not resp.isclosed() + + def test_io_bufferedreader(self) -> None: fp = BytesIO(b"foo") resp = HTTPResponse(fp, preload_content=False) - br = BufferedReader(resp) + br = BufferedReader(resp) # type: ignore[arg-type] assert br.read() == b"foo" @@ -369,14 +778,13 @@ def test_io_bufferedreader(self): # https://github.com/urllib3/urllib3/issues/1305 fp = BytesIO(b"hello\nworld") resp = HTTPResponse(fp, preload_content=False) - with pytest.raises(ValueError) as ctx: - list(BufferedReader(resp)) - assert str(ctx.value) == "readline of closed file" + with pytest.raises(ValueError, match="readline of closed file"): + list(BufferedReader(resp)) # type: ignore[arg-type] b = b"fooandahalf" fp = BytesIO(b) resp = HTTPResponse(fp, preload_content=False) - br = BufferedReader(resp, 5) + br = BufferedReader(resp, 5) # type: ignore[arg-type] br.read(1) # sets up the buffer, reading 5 assert len(fp.read()) == (len(b) - 5) @@ -386,10 +794,10 @@ def test_io_bufferedreader(self): while not br.closed: br.read(5) - def test_io_not_autoclose_bufferedreader(self): + def test_io_not_autoclose_bufferedreader(self) -> None: fp = BytesIO(b"hello\nworld") resp = HTTPResponse(fp, preload_content=False, auto_close=False) - reader = BufferedReader(resp) + reader = BufferedReader(resp) # type: ignore[arg-type] assert list(reader) == [b"hello\n", b"world"] assert not reader.closed @@ -400,16 +808,15 @@ def test_io_not_autoclose_bufferedreader(self): reader.close() assert reader.closed assert resp.closed - with pytest.raises(ValueError) as ctx: + with pytest.raises(ValueError, match="readline of closed file"): next(reader) - assert str(ctx.value) == "readline of closed file" - def test_io_textiowrapper(self): + def test_io_textiowrapper(self) -> None: fp = BytesIO(b"\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f") resp = HTTPResponse(fp, preload_content=False) - br = TextIOWrapper(resp, encoding="utf8") + br = TextIOWrapper(resp, encoding="utf8") # type: ignore[type-var] - assert br.read() == u"äöüß" + assert br.read() == "äöüß" br.close() assert resp.closed @@ -420,25 +827,16 @@ def test_io_textiowrapper(self): b"\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f\n\xce\xb1\xce\xb2\xce\xb3\xce\xb4" ) resp = HTTPResponse(fp, preload_content=False) - with pytest.raises(ValueError) as ctx: - if six.PY2: - # py2's implementation of TextIOWrapper requires `read1` - # method which is provided by `BufferedReader` wrapper - resp = BufferedReader(resp) - list(TextIOWrapper(resp)) - assert re.match("I/O operation on closed file.?", str(ctx.value)) - - def test_io_not_autoclose_textiowrapper(self): + with pytest.raises(ValueError, match="I/O operation on closed file.?"): + list(TextIOWrapper(resp)) # type: ignore[type-var] + + def test_io_not_autoclose_textiowrapper(self) -> None: fp = BytesIO( b"\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f\n\xce\xb1\xce\xb2\xce\xb3\xce\xb4" ) resp = HTTPResponse(fp, preload_content=False, auto_close=False) - if six.PY2: - # py2's implementation of TextIOWrapper requires `read1` - # method which is provided by `BufferedReader` wrapper - resp = BufferedReader(resp) - reader = TextIOWrapper(resp, encoding="utf8") - assert list(reader) == [u"äöüß\n", u"αβγδ"] + reader = TextIOWrapper(resp, encoding="utf8") # type: ignore[type-var] + assert list(reader) == ["äöüß\n", "αβγδ"] assert not reader.closed assert not resp.closed @@ -448,11 +846,79 @@ def test_io_not_autoclose_textiowrapper(self): reader.close() assert reader.closed assert resp.closed - with pytest.raises(ValueError) as ctx: + with pytest.raises(ValueError, match="I/O operation on closed file.?"): next(reader) - assert re.match("I/O operation on closed file.?", str(ctx.value)) - def test_streaming(self): + def test_read_with_illegal_mix_decode_toggle(self) -> None: + data = zlib.compress(b"foo") + + fp = BytesIO(data) + + resp = HTTPResponse( + fp, headers={"content-encoding": "deflate"}, preload_content=False + ) + + assert resp.read(1) == b"f" + + with pytest.raises( + RuntimeError, + match=( + r"Calling read\(decode_content=False\) is not supported after " + r"read\(decode_content=True\) was called" + ), + ): + resp.read(1, decode_content=False) + + with pytest.raises( + RuntimeError, + match=( + r"Calling read\(decode_content=False\) is not supported after " + r"read\(decode_content=True\) was called" + ), + ): + resp.read(decode_content=False) + + def test_read1_with_illegal_mix_decode_toggle(self) -> None: + data = zlib.compress(b"foo") + + fp = BytesIO(data) + + resp = HTTPResponse( + fp, headers={"content-encoding": "deflate"}, preload_content=False + ) + + assert resp.read1(1) == b"f" + + with pytest.raises( + RuntimeError, + match=( + r"Calling read1\(decode_content=False\) is not supported after " + r"read1\(decode_content=True\) was called" + ), + ): + resp.read1(1, decode_content=False) + + with pytest.raises( + RuntimeError, + match=( + r"Calling read1\(decode_content=False\) is not supported after " + r"read1\(decode_content=True\) was called" + ), + ): + resp.read1(decode_content=False) + + def test_read_with_mix_decode_toggle(self) -> None: + data = zlib.compress(b"foo") + + fp = BytesIO(data) + + resp = HTTPResponse( + fp, headers={"content-encoding": "deflate"}, preload_content=False + ) + assert resp.read(2, decode_content=False) is not None + assert resp.read(1, decode_content=True) == b"f" + + def test_streaming(self) -> None: fp = BytesIO(b"foo") resp = HTTPResponse(fp, preload_content=False) stream = resp.stream(2, decode_content=False) @@ -462,7 +928,7 @@ def test_streaming(self): with pytest.raises(StopIteration): next(stream) - def test_streaming_tell(self): + def test_streaming_tell(self) -> None: fp = BytesIO(b"foo") resp = HTTPResponse(fp, preload_content=False) stream = resp.stream(2, decode_content=False) @@ -480,7 +946,7 @@ def test_streaming_tell(self): with pytest.raises(StopIteration): next(stream) - def test_gzipped_streaming(self): + def test_gzipped_streaming(self) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(b"foo") data += compress.flush() @@ -491,12 +957,12 @@ def test_gzipped_streaming(self): ) stream = resp.stream(2) - assert next(stream) == b"f" - assert next(stream) == b"oo" + assert next(stream) == b"fo" + assert next(stream) == b"o" with pytest.raises(StopIteration): next(stream) - def test_gzipped_streaming_tell(self): + def test_gzipped_streaming_tell(self) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) uncompressed_data = b"foo" data = compress.compress(uncompressed_data) @@ -517,10 +983,11 @@ def test_gzipped_streaming_tell(self): with pytest.raises(StopIteration): next(stream) - def test_deflate_streaming_tell_intermediate_point(self): + def test_deflate_streaming_tell_intermediate_point(self) -> None: # Ensure that ``tell()`` returns the correct number of bytes when # part-way through streaming compressed content. NUMBER_OF_READS = 10 + PART_SIZE = 64 class MockCompressedDataReading(BytesIO): """ @@ -528,7 +995,7 @@ class MockCompressedDataReading(BytesIO): calls to ``read``. """ - def __init__(self, payload, payload_part_size): + def __init__(self, payload: bytes, payload_part_size: int) -> None: self.payloads = [ payload[i * payload_part_size : (i + 1) * payload_part_size] for i in range(NUMBER_OF_READS + 1) @@ -536,12 +1003,15 @@ def __init__(self, payload, payload_part_size): assert b"".join(self.payloads) == payload - def read(self, _): + def read(self, _: int) -> bytes: # type: ignore[override] # Amount is unused. if len(self.payloads) > 0: return self.payloads.pop(0) return b"" + def read1(self, amt: int) -> bytes: # type: ignore[override] + return self.read(amt) + uncompressed_data = zlib.decompress(ZLIB_PAYLOAD) payload_part_size = len(ZLIB_PAYLOAD) // NUMBER_OF_READS @@ -549,7 +1019,7 @@ def read(self, _): resp = HTTPResponse( fp, headers={"content-encoding": "deflate"}, preload_content=False ) - stream = resp.stream() + stream = resp.stream(PART_SIZE) parts_positions = [(part, resp.tell()) for part in stream] end_of_stream = resp.tell() @@ -564,13 +1034,29 @@ def read(self, _): assert uncompressed_data == payload # Check that the positions in the stream are correct - expected = [(i + 1) * payload_part_size for i in range(NUMBER_OF_READS)] - assert expected == list(positions) + # It is difficult to determine programmatically what the positions + # returned by `tell` will be because the `HTTPResponse.read` method may + # call socket `read` a couple of times if it doesn't have enough data + # in the buffer or not call socket `read` at all if it has enough. All + # this depends on the message, how it was compressed, what is + # `PART_SIZE` and `payload_part_size`. + # So for simplicity the expected values are hardcoded. + expected = (92, 184, 230, 276, 322, 368, 414, 460) + assert expected == positions # Check that the end of the stream is in the correct place assert len(ZLIB_PAYLOAD) == end_of_stream - def test_deflate_streaming(self): + # Check that all parts have expected length + expected_last_part_size = len(uncompressed_data) % PART_SIZE + whole_parts = len(uncompressed_data) // PART_SIZE + if expected_last_part_size == 0: + expected_lengths = [PART_SIZE] * whole_parts + else: + expected_lengths = [PART_SIZE] * whole_parts + [expected_last_part_size] + assert expected_lengths == [len(part) for part in parts] + + def test_deflate_streaming(self) -> None: data = zlib.compress(b"foo") fp = BytesIO(data) @@ -579,12 +1065,12 @@ def test_deflate_streaming(self): ) stream = resp.stream(2) - assert next(stream) == b"f" - assert next(stream) == b"oo" + assert next(stream) == b"fo" + assert next(stream) == b"o" with pytest.raises(StopIteration): next(stream) - def test_deflate2_streaming(self): + def test_deflate2_streaming(self) -> None: compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS) data = compress.compress(b"foo") data += compress.flush() @@ -595,12 +1081,12 @@ def test_deflate2_streaming(self): ) stream = resp.stream(2) - assert next(stream) == b"f" - assert next(stream) == b"oo" + assert next(stream) == b"fo" + assert next(stream) == b"o" with pytest.raises(StopIteration): next(stream) - def test_empty_stream(self): + def test_empty_stream(self) -> None: fp = BytesIO(b"") resp = HTTPResponse(fp, preload_content=False) stream = resp.stream(2, decode_content=False) @@ -608,19 +1094,63 @@ def test_empty_stream(self): with pytest.raises(StopIteration): next(stream) - def test_length_no_header(self): + @pytest.mark.parametrize( + "preload_content, amt, read_meth", + [ + (True, None, "read"), + (False, None, "read"), + (False, 10 * 2**20, "read"), + (False, None, "read1"), + (False, 10 * 2**20, "read1"), + ], + ) + @pytest.mark.limit_memory("25 MB", current_thread_only=True) + def test_buffer_memory_usage_decode_one_chunk( + self, preload_content: bool, amt: int, read_meth: str + ) -> None: + content_length = 10 * 2**20 # 10 MiB + fp = BytesIO(zlib.compress(bytes(content_length))) + resp = HTTPResponse( + fp, + preload_content=preload_content, + headers={"content-encoding": "deflate"}, + ) + data = resp.data if preload_content else getattr(resp, read_meth)(amt) + assert len(data) == content_length + + @pytest.mark.parametrize( + "preload_content, amt, read_meth", + [ + (True, None, "read"), + (False, None, "read"), + (False, 10 * 2**20, "read"), + (False, None, "read1"), + (False, 10 * 2**20, "read1"), + ], + ) + @pytest.mark.limit_memory("10.5 MB", current_thread_only=True) + def test_buffer_memory_usage_no_decoding( + self, preload_content: bool, amt: int, read_meth: str + ) -> None: + content_length = 10 * 2**20 # 10 MiB + fp = BytesIO(bytes(content_length)) + resp = HTTPResponse(fp, preload_content=preload_content, decode_content=False) + data = resp.data if preload_content else getattr(resp, read_meth)(amt) + assert len(data) == content_length + + def test_length_no_header(self) -> None: fp = BytesIO(b"12345") resp = HTTPResponse(fp, preload_content=False) assert resp.length_remaining is None - def test_length_w_valid_header(self): + def test_length_w_valid_header(self) -> None: headers = {"content-length": "5"} fp = BytesIO(b"12345") resp = HTTPResponse(fp, headers=headers, preload_content=False) assert resp.length_remaining == 5 - def test_length_w_bad_header(self): + def test_length_w_bad_header(self) -> None: garbage = {"content-length": "foo"} fp = BytesIO(b"12345") @@ -631,7 +1161,7 @@ def test_length_w_bad_header(self): resp = HTTPResponse(fp, headers=garbage, preload_content=False) assert resp.length_remaining is None - def test_length_when_chunked(self): + def test_length_when_chunked(self) -> None: # This is expressly forbidden in RFC 7230 sec 3.3.2 # We fall back to chunked in this case and try to # handle response ignoring content length. @@ -641,7 +1171,7 @@ def test_length_when_chunked(self): resp = HTTPResponse(fp, headers=headers, preload_content=False) assert resp.length_remaining is None - def test_length_with_multiple_content_lengths(self): + def test_length_with_multiple_content_lengths(self) -> None: headers = {"content-length": "5, 5, 5"} garbage = {"content-length": "5, 42"} fp = BytesIO(b"abcde") @@ -652,7 +1182,7 @@ def test_length_with_multiple_content_lengths(self): with pytest.raises(InvalidHeader): HTTPResponse(fp, headers=garbage, preload_content=False) - def test_length_after_read(self): + def test_length_after_read(self) -> None: headers = {"content-length": "5"} # Test no defined length @@ -674,27 +1204,32 @@ def test_length_after_read(self): next(data) assert resp.length_remaining == 3 - def test_mock_httpresponse_stream(self): + def test_mock_httpresponse_stream(self) -> None: # Mock out a HTTP Request that does enough to make it through urllib3's # read() and close() calls, and also exhausts and underlying file # object. - class MockHTTPRequest(object): - self.fp = None + class MockHTTPRequest: + def __init__(self) -> None: + self.fp: BytesIO | None = None - def read(self, amt): + def read(self, amt: int) -> bytes: + assert self.fp is not None data = self.fp.read(amt) if not data: self.fp = None return data - def close(self): + def read1(self, amt: int) -> bytes: + return self.read(1) + + def close(self) -> None: self.fp = None bio = BytesIO(b"foo") fp = MockHTTPRequest() fp.fp = bio - resp = HTTPResponse(fp, preload_content=False) + resp = HTTPResponse(fp, preload_content=False) # type: ignore[arg-type] stream = resp.stream(2) assert next(stream) == b"fo" @@ -702,11 +1237,11 @@ def close(self): with pytest.raises(StopIteration): next(stream) - def test_mock_transfer_encoding_chunked(self): + def test_mock_transfer_encoding_chunked(self) -> None: stream = [b"fo", b"o", b"bar"] fp = MockChunkedEncodingResponse(stream) - r = httplib.HTTPResponse(MockSock) - r.fp = fp + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] resp = HTTPResponse( r, preload_content=False, headers={"transfer-encoding": "chunked"} ) @@ -714,10 +1249,10 @@ def test_mock_transfer_encoding_chunked(self): for i, c in enumerate(resp.stream()): assert c == stream[i] - def test_mock_gzipped_transfer_encoding_chunked_decoded(self): + def test_mock_gzipped_transfer_encoding_chunked_decoded(self) -> None: """Show that we can decode the gzipped and chunked body.""" - def stream(): + def stream() -> typing.Generator[bytes]: # Set up a generator to chunk the gzipped body compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(b"foobar") @@ -726,8 +1261,8 @@ def stream(): yield data[i : i + 2] fp = MockChunkedEncodingResponse(list(stream())) - r = httplib.HTTPResponse(MockSock) - r.fp = fp + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] headers = {"transfer-encoding": "chunked", "content-encoding": "gzip"} resp = HTTPResponse(r, preload_content=False, headers=headers) @@ -737,11 +1272,11 @@ def stream(): assert b"foobar" == data - def test_mock_transfer_encoding_chunked_custom_read(self): + def test_mock_transfer_encoding_chunked_custom_read(self) -> None: stream = [b"foooo", b"bbbbaaaaar"] fp = MockChunkedEncodingResponse(stream) - r = httplib.HTTPResponse(MockSock) - r.fp = fp + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] r.chunked = True r.chunk_left = None resp = HTTPResponse( @@ -751,26 +1286,38 @@ def test_mock_transfer_encoding_chunked_custom_read(self): response = list(resp.read_chunked(2)) assert expected_response == response - def test_mock_transfer_encoding_chunked_unlmtd_read(self): + @pytest.mark.parametrize("read_chunked_args", ((), (None,), (-1,))) + def test_mock_transfer_encoding_chunked_unlmtd_read( + self, read_chunked_args: tuple[typing.Any, ...] + ) -> None: stream = [b"foooo", b"bbbbaaaaar"] fp = MockChunkedEncodingResponse(stream) - r = httplib.HTTPResponse(MockSock) - r.fp = fp + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] r.chunked = True r.chunk_left = None resp = HTTPResponse( r, preload_content=False, headers={"transfer-encoding": "chunked"} ) - assert stream == list(resp.read_chunked()) + assert stream == list(resp.read_chunked(*read_chunked_args)) - def test_read_not_chunked_response_as_chunks(self): + def test_read_not_chunked_response_as_chunks(self) -> None: fp = BytesIO(b"foo") resp = HTTPResponse(fp, preload_content=False) r = resp.read_chunked() with pytest.raises(ResponseNotChunked): next(r) - def test_buggy_incomplete_read(self): + def test_read_chunked_not_supported(self) -> None: + fp = BytesIO(b"foo") + resp = HTTPResponse( + fp, preload_content=False, headers={"transfer-encoding": "chunked"} + ) + r = resp.read_chunked() + with pytest.raises(BodyNotHttplibCompatible): + next(r) + + def test_buggy_incomplete_read(self) -> None: # Simulate buggy versions of Python (<2.7.4) # See http://bugs.python.org/issue16298 content_length = 1337 @@ -789,11 +1336,11 @@ def test_buggy_incomplete_read(self): assert orig_ex.partial == 0 assert orig_ex.expected == content_length - def test_incomplete_chunk(self): + def test_incomplete_chunk(self) -> None: stream = [b"foooo", b"bbbbaaaaar"] fp = MockChunkedIncompleteRead(stream) - r = httplib.HTTPResponse(MockSock) - r.fp = fp + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] r.chunked = True r.chunk_left = None resp = HTTPResponse( @@ -805,11 +1352,11 @@ def test_incomplete_chunk(self): orig_ex = ctx.value.args[1] assert isinstance(orig_ex, httplib_IncompleteRead) - def test_invalid_chunk_length(self): + def test_invalid_chunk_length(self) -> None: stream = [b"foooo", b"bbbbaaaaar"] fp = MockChunkedInvalidChunkLength(stream) - r = httplib.HTTPResponse(MockSock) - r.fp = fp + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] r.chunked = True r.chunk_left = None resp = HTTPResponse( @@ -819,14 +1366,34 @@ def test_invalid_chunk_length(self): next(resp.read_chunked()) orig_ex = ctx.value.args[1] + msg = ( + "(\"Connection broken: InvalidChunkLength(got length b'ZZZ\\\\r\\\\n', 0 bytes read)\", " + "InvalidChunkLength(got length b'ZZZ\\r\\n', 0 bytes read))" + ) + assert str(ctx.value) == msg assert isinstance(orig_ex, InvalidChunkLength) - assert orig_ex.length == six.b(fp.BAD_LENGTH_LINE) + assert orig_ex.length == fp.BAD_LENGTH_LINE.encode() - def test_chunked_response_without_crlf_on_end(self): + def test_truncated_before_chunk(self) -> None: + stream = [b"foooo", b"bbbbaaaaar"] + fp = MockChunkedNoChunks(stream) + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] + r.chunked = True + r.chunk_left = None + resp = HTTPResponse( + r, preload_content=False, headers={"transfer-encoding": "chunked"} + ) + with pytest.raises(ProtocolError) as ctx: + next(resp.read_chunked()) + + assert str(ctx.value) == "Response ended prematurely" + + def test_chunked_response_without_crlf_on_end(self) -> None: stream = [b"foo", b"bar", b"baz"] fp = MockChunkedEncodingWithoutCRLFOnEnd(stream) - r = httplib.HTTPResponse(MockSock) - r.fp = fp + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] r.chunked = True r.chunk_left = None resp = HTTPResponse( @@ -834,11 +1401,11 @@ def test_chunked_response_without_crlf_on_end(self): ) assert stream == list(resp.stream()) - def test_chunked_response_with_extensions(self): + def test_chunked_response_with_extensions(self) -> None: stream = [b"foo", b"bar"] fp = MockChunkedEncodingWithExtensions(stream) - r = httplib.HTTPResponse(MockSock) - r.fp = fp + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] r.chunked = True r.chunk_left = None resp = HTTPResponse( @@ -846,8 +1413,8 @@ def test_chunked_response_with_extensions(self): ) assert stream == list(resp.stream()) - def test_chunked_head_response(self): - r = httplib.HTTPResponse(MockSock, method="HEAD") + def test_chunked_head_response(self) -> None: + r = httplib.HTTPResponse(MockSock, method="HEAD") # type: ignore[arg-type] r.chunked = True r.chunk_left = None resp = HTTPResponse( @@ -858,19 +1425,19 @@ def test_chunked_head_response(self): ) assert resp.chunked is True - resp.supports_chunked_reads = lambda: True - resp.release_conn = mock.Mock() + setattr(resp, "supports_chunked_reads", lambda: True) + setattr(resp, "release_conn", mock.Mock()) for _ in resp.stream(): continue - resp.release_conn.assert_called_once_with() + resp.release_conn.assert_called_once_with() # type: ignore[attr-defined] - def test_get_case_insensitive_headers(self): + def test_get_case_insensitive_headers(self) -> None: headers = {"host": "example.com"} r = HTTPResponse(headers=headers) assert r.headers.get("host") == "example.com" assert r.headers.get("Host") == "example.com" - def test_retries(self): + def test_retries(self) -> None: fp = BytesIO(b"") resp = HTTPResponse(fp) assert resp.retries is None @@ -878,16 +1445,24 @@ def test_retries(self): resp = HTTPResponse(fp, retries=retry) assert resp.retries == retry - def test_geturl(self): + def test_geturl(self) -> None: fp = BytesIO(b"") request_url = "https://example.com" resp = HTTPResponse(fp, request_url=request_url) assert resp.geturl() == request_url - def test_geturl_retries(self): + def test_url(self) -> None: + fp = BytesIO(b"") + request_url = "https://example.com" + resp = HTTPResponse(fp, request_url=request_url) + assert resp.url == request_url + resp.url = "https://anotherurl.com" + assert resp.url == "https://anotherurl.com" + + def test_geturl_retries(self) -> None: fp = BytesIO(b"") resp = HTTPResponse(fp, request_url="http://example.com") - request_histories = [ + request_histories = ( RequestHistory( method="GET", url="http://example.com", @@ -902,7 +1477,7 @@ def test_geturl_retries(self): status=301, redirect_location="https://www.example.com", ), - ] + ) retry = Retry(history=request_histories) resp = HTTPResponse(fp, retries=retry) assert resp.geturl() == "https://www.example.com" @@ -917,15 +1492,15 @@ def test_geturl_retries(self): (b"Hello\nworld\n\n\n!", [b"Hello\n", b"world\n", b"\n", b"\n", b"!"]), ], ) - def test__iter__(self, payload, expected_stream): + def test__iter__(self, payload: bytes, expected_stream: list[bytes]) -> None: actual_stream = [] for chunk in HTTPResponse(BytesIO(payload), preload_content=False): actual_stream.append(chunk) assert actual_stream == expected_stream - def test__iter__decode_content(self): - def stream(): + def test__iter__decode_content(self) -> None: + def stream() -> typing.Generator[bytes]: # Set up a generator to chunk the gzipped body compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(b"foo\nbar") @@ -934,8 +1509,8 @@ def stream(): yield data[i : i + 2] fp = MockChunkedEncodingResponse(list(stream())) - r = httplib.HTTPResponse(MockSock) - r.fp = fp + r = httplib.HTTPResponse(MockSock) # type: ignore[arg-type] + r.fp = fp # type: ignore[assignment] headers = {"transfer-encoding": "chunked", "content-encoding": "gzip"} resp = HTTPResponse(r, preload_content=False, headers=headers) @@ -945,13 +1520,13 @@ def stream(): assert b"foo\nbar" == data - def test_non_timeout_ssl_error_on_read(self): + def test_non_timeout_ssl_error_on_read(self) -> None: mac_error = ssl.SSLError( "SSL routines", "ssl3_get_record", "decryption failed or bad record mac" ) @contextlib.contextmanager - def make_bad_mac_fp(): + def make_bad_mac_fp() -> typing.Generator[BytesIO]: fp = BytesIO(b"") with mock.patch.object(fp, "read") as fp_read: # mac/decryption error @@ -969,9 +1544,30 @@ def make_bad_mac_fp(): resp.read() assert e.value.args[0] == mac_error - -class MockChunkedEncodingResponse(object): - def __init__(self, content): + def test_unexpected_body(self) -> None: + with pytest.raises(ProtocolError) as excinfo: + fp = BytesIO(b"12345") + headers = {"content-length": "5"} + resp = HTTPResponse(fp, status=204, headers=headers) + resp.read(16) + assert "Response may not contain content" in str(excinfo.value) + + with pytest.raises(ProtocolError): + fp = BytesIO(b"12345") + headers = {"content-length": "0"} + resp = HTTPResponse(fp, status=204, headers=headers) + resp.read(16) + assert "Response may not contain content" in str(excinfo.value) + + with pytest.raises(ProtocolError): + fp = BytesIO(b"12345") + resp = HTTPResponse(fp, status=204) + resp.read(16) + assert "Response may not contain content" in str(excinfo.value) + + +class MockChunkedEncodingResponse: + def __init__(self, content: list[bytes]) -> None: """ content: collection of str, each str is a chunk in response """ @@ -981,13 +1577,12 @@ def __init__(self, content): self.cur_chunk = b"" self.chunks_exhausted = False - @staticmethod - def _encode_chunk(chunk): + def _encode_chunk(self, chunk: bytes) -> bytes: # In the general case, we can't decode the chunk to unicode - length = "%X\r\n" % len(chunk) + length = f"{len(chunk):X}\r\n" return length.encode() + chunk + b"\r\n" - def _pop_new_chunk(self): + def _pop_new_chunk(self) -> bytes: if self.chunks_exhausted: return b"" try: @@ -1000,9 +1595,10 @@ def _pop_new_chunk(self): chunk = self._encode_chunk(chunk) if not isinstance(chunk, bytes): chunk = chunk.encode() + assert isinstance(chunk, bytes) return chunk - def pop_current_chunk(self, amt=-1, till_crlf=False): + def pop_current_chunk(self, amt: int = -1, till_crlf: bool = False) -> bytes: if amt > 0 and till_crlf: raise ValueError("Can't specify amt and till_crlf.") if len(self.cur_chunk) <= 0: @@ -1032,47 +1628,55 @@ def pop_current_chunk(self, amt=-1, till_crlf=False): self.cur_chunk = self.cur_chunk[amt:] return chunk_part - def readline(self): + def readline(self) -> bytes: return self.pop_current_chunk(till_crlf=True) - def read(self, amt=-1): + def read(self, amt: int = -1) -> bytes: + return self.pop_current_chunk(amt) + + def read1(self, amt: int = -1) -> bytes: return self.pop_current_chunk(amt) - def flush(self): + def flush(self) -> None: # Python 3 wants this method. pass - def close(self): + def close(self) -> None: self.closed = True class MockChunkedIncompleteRead(MockChunkedEncodingResponse): - def _encode_chunk(self, chunk): - return "9999\r\n%s\r\n" % chunk.decode() + def _encode_chunk(self, chunk: bytes) -> bytes: + return f"9999\r\n{chunk.decode()}\r\n".encode() class MockChunkedInvalidChunkLength(MockChunkedEncodingResponse): BAD_LENGTH_LINE = "ZZZ\r\n" - def _encode_chunk(self, chunk): - return "%s%s\r\n" % (self.BAD_LENGTH_LINE, chunk.decode()) + def _encode_chunk(self, chunk: bytes) -> bytes: + return f"{self.BAD_LENGTH_LINE}{chunk.decode()}\r\n".encode() class MockChunkedEncodingWithoutCRLFOnEnd(MockChunkedEncodingResponse): - def _encode_chunk(self, chunk): - return "%X\r\n%s%s" % ( + def _encode_chunk(self, chunk: bytes) -> bytes: + return "{:X}\r\n{}{}".format( len(chunk), chunk.decode(), "\r\n" if len(chunk) > 0 else "", - ) + ).encode() class MockChunkedEncodingWithExtensions(MockChunkedEncodingResponse): - def _encode_chunk(self, chunk): - return "%X;asd=qwe\r\n%s\r\n" % (len(chunk), chunk.decode()) + def _encode_chunk(self, chunk: bytes) -> bytes: + return f"{len(chunk):X};asd=qwe\r\n{chunk.decode()}\r\n".encode() + + +class MockChunkedNoChunks(MockChunkedEncodingResponse): + def _encode_chunk(self, chunk: bytes) -> bytes: + return b"" -class MockSock(object): +class MockSock: @classmethod - def makefile(cls, *args, **kwargs): + def makefile(cls, *args: typing.Any, **kwargs: typing.Any) -> None: return diff --git a/test/test_retry.py b/test/test_retry.py index 95a33e7..85206b9 100644 --- a/test/test_retry.py +++ b/test/test_retry.py @@ -1,6 +1,9 @@ -import warnings +from __future__ import annotations + +import datetime +from test import DUMMY_POOL +from unittest import mock -import mock import pytest from urllib3.exceptions import ( @@ -11,21 +14,12 @@ ResponseError, SSLError, ) -from urllib3.packages import six -from urllib3.packages.six.moves import xrange from urllib3.response import HTTPResponse from urllib3.util.retry import RequestHistory, Retry -@pytest.fixture(scope="function", autouse=True) -def no_retry_deprecations(): - with warnings.catch_warnings(record=True) as w: - yield - assert len([str(x.message) for x in w if "Retry" in str(x.message)]) == 0 - - -class TestRetry(object): - def test_string(self): +class TestRetry: + def test_string(self) -> None: """Retry string representation looks the way we expect""" retry = Retry() assert ( @@ -39,7 +33,7 @@ def test_string(self): == "Retry(total=7, connect=None, read=None, redirect=None, status=None)" ) - def test_retry_both_specified(self): + def test_retry_both_specified(self) -> None: """Total can win if it's lower than the connect value""" error = ConnectTimeoutError() retry = Retry(connect=3, total=2) @@ -49,7 +43,7 @@ def test_retry_both_specified(self): retry.increment(error=error) assert e.value.reason == error - def test_retry_higher_total_loses(self): + def test_retry_higher_total_loses(self) -> None: """A lower connect timeout than the total is honored""" error = ConnectTimeoutError() retry = Retry(connect=2, total=3) @@ -58,16 +52,16 @@ def test_retry_higher_total_loses(self): with pytest.raises(MaxRetryError): retry.increment(error=error) - def test_retry_higher_total_loses_vs_read(self): + def test_retry_higher_total_loses_vs_read(self) -> None: """A lower read timeout than the total is honored""" - error = ReadTimeoutError(None, "/", "read timed out") + error = ReadTimeoutError(DUMMY_POOL, "/", "read timed out") retry = Retry(read=2, total=3) retry = retry.increment(method="GET", error=error) retry = retry.increment(method="GET", error=error) with pytest.raises(MaxRetryError): retry.increment(method="GET", error=error) - def test_retry_total_none(self): + def test_retry_total_none(self) -> None: """if Total is none, connect error should take precedence""" error = ConnectTimeoutError() retry = Retry(connect=2, total=None) @@ -77,14 +71,14 @@ def test_retry_total_none(self): retry.increment(error=error) assert e.value.reason == error - error = ReadTimeoutError(None, "/", "read timed out") + timeout_error = ReadTimeoutError(DUMMY_POOL, "/", "read timed out") retry = Retry(connect=2, total=None) - retry = retry.increment(method="GET", error=error) - retry = retry.increment(method="GET", error=error) - retry = retry.increment(method="GET", error=error) + retry = retry.increment(method="GET", error=timeout_error) + retry = retry.increment(method="GET", error=timeout_error) + retry = retry.increment(method="GET", error=timeout_error) assert not retry.is_exhausted() - def test_retry_default(self): + def test_retry_default(self) -> None: """If no value is specified, should retry connects 3 times""" retry = Retry() assert retry.total == 10 @@ -106,7 +100,7 @@ def test_retry_default(self): assert Retry(0).raise_on_redirect assert not Retry(False).raise_on_redirect - def test_retry_other(self): + def test_retry_other(self) -> None: """If an unexpected error is raised, should retry other times""" other_error = SSLError() retry = Retry(connect=1) @@ -120,26 +114,24 @@ def test_retry_other(self): retry.increment(error=other_error) assert e.value.reason == other_error - def test_retry_read_zero(self): + def test_retry_read_zero(self) -> None: """No second chances on read timeouts, by default""" - error = ReadTimeoutError(None, "/", "read timed out") + error = ReadTimeoutError(DUMMY_POOL, "/", "read timed out") retry = Retry(read=0) with pytest.raises(MaxRetryError) as e: retry.increment(method="GET", error=error) assert e.value.reason == error - def test_status_counter(self): + def test_status_counter(self) -> None: resp = HTTPResponse(status=400) retry = Retry(status=2) retry = retry.increment(response=resp) retry = retry.increment(response=resp) - with pytest.raises(MaxRetryError) as e: + msg = ResponseError.SPECIFIC_ERROR.format(status_code=400) + with pytest.raises(MaxRetryError, match=msg): retry.increment(response=resp) - assert str(e.value.reason) == ResponseError.SPECIFIC_ERROR.format( - status_code=400 - ) - def test_backoff(self): + def test_backoff(self) -> None: """Backoff is computed correctly""" max_backoff = Retry.DEFAULT_BACKOFF_MAX @@ -160,19 +152,72 @@ def test_backoff(self): retry = retry.increment(method="GET") assert retry.get_backoff_time() == 1.6 - for _ in xrange(10): + for _ in range(10): retry = retry.increment(method="GET") assert retry.get_backoff_time() == max_backoff - def test_zero_backoff(self): + def test_configurable_backoff_max(self) -> None: + """Configurable backoff is computed correctly""" + max_backoff = 1 + + retry = Retry(total=100, backoff_factor=0.2, backoff_max=max_backoff) + assert retry.get_backoff_time() == 0 # First request + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == 0 # First retry + + retry = retry.increment(method="GET") + assert retry.backoff_factor == 0.2 + assert retry.total == 98 + assert retry.get_backoff_time() == 0.4 # Start backoff + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == 0.8 + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == max_backoff + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == max_backoff + + def test_backoff_jitter(self) -> None: + """Backoff with jitter is computed correctly""" + max_backoff = 1 + jitter = 0.4 + retry = Retry( + total=100, + backoff_factor=0.2, + backoff_max=max_backoff, + backoff_jitter=jitter, + ) + assert retry.get_backoff_time() == 0 # First request + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == 0 # First retry + + retry = retry.increment(method="GET") + assert retry.backoff_factor == 0.2 + assert retry.total == 98 + assert 0.4 <= retry.get_backoff_time() <= 0.8 # Start backoff + + retry = retry.increment(method="GET") + assert 0.8 <= retry.get_backoff_time() <= max_backoff + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == max_backoff + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == max_backoff + + def test_zero_backoff(self) -> None: retry = Retry() assert retry.get_backoff_time() == 0 retry = retry.increment(method="GET") retry = retry.increment(method="GET") assert retry.get_backoff_time() == 0 - def test_backoff_reset_after_redirect(self): + def test_backoff_reset_after_redirect(self) -> None: retry = Retry(total=100, redirect=5, backoff_factor=0.2) retry = retry.increment(method="GET") retry = retry.increment(method="GET") @@ -184,15 +229,15 @@ def test_backoff_reset_after_redirect(self): retry = retry.increment(method="GET") assert retry.get_backoff_time() == 0.4 - def test_sleep(self): + def test_sleep(self) -> None: # sleep a very small amount of time so our code coverage is happy retry = Retry(backoff_factor=0.0001) retry = retry.increment(method="GET") retry = retry.increment(method="GET") retry.sleep() - def test_status_forcelist(self): - retry = Retry(status_forcelist=xrange(500, 600)) + def test_status_forcelist(self) -> None: + retry = Retry(status_forcelist=range(500, 600)) assert not retry.is_retry("GET", status_code=200) assert not retry.is_retry("GET", status_code=400) assert retry.is_retry("GET", status_code=500) @@ -202,10 +247,10 @@ def test_status_forcelist(self): assert retry.is_retry("GET", status_code=418) # String status codes are not matched. - retry = Retry(total=1, status_forcelist=["418"]) + retry = Retry(total=1, status_forcelist=["418"]) # type: ignore[list-item] assert not retry.is_retry("GET", status_code=418) - def test_allowed_methods_with_status_forcelist(self): + def test_allowed_methods_with_status_forcelist(self) -> None: # Falsey allowed_methods means to retry on any method. retry = Retry(status_forcelist=[500], allowed_methods=None) assert retry.is_retry("GET", status_code=500) @@ -216,92 +261,92 @@ def test_allowed_methods_with_status_forcelist(self): assert not retry.is_retry("GET", status_code=500) assert retry.is_retry("POST", status_code=500) - def test_exhausted(self): + def test_exhausted(self) -> None: assert not Retry(0).is_exhausted() assert Retry(-1).is_exhausted() assert Retry(1).increment(method="GET").total == 0 @pytest.mark.parametrize("total", [-1, 0]) - def test_disabled(self, total): + def test_disabled(self, total: int) -> None: with pytest.raises(MaxRetryError): Retry(total).increment(method="GET") - def test_error_message(self): + def test_error_message(self) -> None: retry = Retry(total=0) - with pytest.raises(MaxRetryError) as e: + with pytest.raises(MaxRetryError, match="read timed out") as e: retry = retry.increment( - method="GET", error=ReadTimeoutError(None, "/", "read timed out") + method="GET", error=ReadTimeoutError(DUMMY_POOL, "/", "read timed out") ) assert "Caused by redirect" not in str(e.value) - assert str(e.value.reason) == "None: read timed out" retry = Retry(total=1) - with pytest.raises(MaxRetryError) as e: - retry = retry.increment("POST", "/") + retry = retry.increment("POST", "/") + with pytest.raises(MaxRetryError, match=ResponseError.GENERIC_ERROR) as e: retry = retry.increment("POST", "/") assert "Caused by redirect" not in str(e.value) assert isinstance(e.value.reason, ResponseError) - assert str(e.value.reason) == ResponseError.GENERIC_ERROR retry = Retry(total=1) response = HTTPResponse(status=500) - with pytest.raises(MaxRetryError) as e: - retry = retry.increment("POST", "/", response=response) + msg = ResponseError.SPECIFIC_ERROR.format(status_code=500) + retry = retry.increment("POST", "/", response=response) + with pytest.raises(MaxRetryError, match=msg) as e: retry = retry.increment("POST", "/", response=response) assert "Caused by redirect" not in str(e.value) - msg = ResponseError.SPECIFIC_ERROR.format(status_code=500) - assert str(e.value.reason) == msg retry = Retry(connect=1) - with pytest.raises(MaxRetryError) as e: - retry = retry.increment(error=ConnectTimeoutError("conntimeout")) + retry = retry.increment(error=ConnectTimeoutError("conntimeout")) + with pytest.raises(MaxRetryError, match="conntimeout") as e: retry = retry.increment(error=ConnectTimeoutError("conntimeout")) assert "Caused by redirect" not in str(e.value) - assert str(e.value.reason) == "conntimeout" - def test_history(self): + def test_history(self) -> None: retry = Retry(total=10, allowed_methods=frozenset(["GET", "POST"])) assert retry.history == tuple() connection_error = ConnectTimeoutError("conntimeout") retry = retry.increment("GET", "/test1", None, connection_error) - history = (RequestHistory("GET", "/test1", connection_error, None, None),) - assert retry.history == history + test_history1 = (RequestHistory("GET", "/test1", connection_error, None, None),) + assert retry.history == test_history1 - read_error = ReadTimeoutError(None, "/test2", "read timed out") + read_error = ReadTimeoutError(DUMMY_POOL, "/test2", "read timed out") retry = retry.increment("POST", "/test2", None, read_error) - history = ( + test_history2 = ( RequestHistory("GET", "/test1", connection_error, None, None), RequestHistory("POST", "/test2", read_error, None, None), ) - assert retry.history == history + assert retry.history == test_history2 response = HTTPResponse(status=500) retry = retry.increment("GET", "/test3", response, None) - history = ( + test_history3 = ( RequestHistory("GET", "/test1", connection_error, None, None), RequestHistory("POST", "/test2", read_error, None, None), RequestHistory("GET", "/test3", None, 500, None), ) - assert retry.history == history + assert retry.history == test_history3 - def test_retry_method_not_in_whitelist(self): - error = ReadTimeoutError(None, "/", "read timed out") + def test_retry_method_not_allowed(self) -> None: + error = ReadTimeoutError(DUMMY_POOL, "/", "read timed out") retry = Retry() with pytest.raises(ReadTimeoutError): retry.increment(method="POST", error=error) - def test_retry_default_remove_headers_on_redirect(self): + def test_retry_default_remove_headers_on_redirect(self) -> None: retry = Retry() - assert retry.remove_headers_on_redirect == {"authorization", "cookie"} + assert retry.remove_headers_on_redirect == { + "authorization", + "proxy-authorization", + "cookie", + } - def test_retry_set_remove_headers_on_redirect(self): + def test_retry_set_remove_headers_on_redirect(self) -> None: retry = Retry(remove_headers_on_redirect=["X-API-Secret"]) assert retry.remove_headers_on_redirect == {"x-api-secret"} - @pytest.mark.parametrize("value", ["-1", "+1", "1.0", six.u("\xb2")]) # \xb2 = ^2 - def test_parse_retry_after_invalid(self, value): + @pytest.mark.parametrize("value", ["-1", "+1", "1.0", "\xb2"]) # \xb2 = ^2 + def test_parse_retry_after_invalid(self, value: str) -> None: retry = Retry() with pytest.raises(InvalidHeader): retry.parse_retry_after(value) @@ -309,18 +354,18 @@ def test_parse_retry_after_invalid(self, value): @pytest.mark.parametrize( "value, expected", [("0", 0), ("1000", 1000), ("\t42 ", 42)] ) - def test_parse_retry_after(self, value, expected): + def test_parse_retry_after(self, value: str, expected: int) -> None: retry = Retry() assert retry.parse_retry_after(value) == expected @pytest.mark.parametrize("respect_retry_after_header", [True, False]) - def test_respect_retry_after_header_propagated(self, respect_retry_after_header): - + def test_respect_retry_after_header_propagated( + self, respect_retry_after_header: bool + ) -> None: retry = Retry(respect_retry_after_header=respect_retry_after_header) new_retry = retry.new() assert new_retry.respect_retry_after_header == respect_retry_after_header - @pytest.mark.freeze_time("2019-06-03 11:00:00", tz_offset=0) @pytest.mark.parametrize( "retry_after_header,respect_retry_after_header,sleep_duration", [ @@ -352,11 +397,22 @@ def test_respect_retry_after_header_propagated(self, respect_retry_after_header) ) @pytest.mark.usefixtures("stub_timezone") def test_respect_retry_after_header_sleep( - self, retry_after_header, respect_retry_after_header, sleep_duration - ): + self, + retry_after_header: str, + respect_retry_after_header: bool, + sleep_duration: int | None, + ) -> None: retry = Retry(respect_retry_after_header=respect_retry_after_header) - with mock.patch("time.sleep") as sleep_mock: + with ( + mock.patch( + "time.time", + return_value=datetime.datetime( + 2019, 6, 3, 11, tzinfo=datetime.timezone.utc + ).timestamp(), + ), + mock.patch("time.sleep") as sleep_mock, + ): # for the default behavior, it must be in RETRY_AFTER_STATUS_CODES response = HTTPResponse( status=503, headers={"Retry-After": retry_after_header} diff --git a/test/test_retry_deprecated.py b/test/test_retry_deprecated.py deleted file mode 100644 index 5133a51..0000000 --- a/test/test_retry_deprecated.py +++ /dev/null @@ -1,485 +0,0 @@ -# This is a copy-paste of test_retry.py with extra asserts about deprecated options. It will be removed for v2. -import warnings - -import mock -import pytest - -from urllib3.exceptions import ( - ConnectTimeoutError, - InvalidHeader, - MaxRetryError, - ReadTimeoutError, - ResponseError, - SSLError, -) -from urllib3.packages import six -from urllib3.packages.six.moves import xrange -from urllib3.response import HTTPResponse -from urllib3.util.retry import RequestHistory, Retry - - -# TODO: Remove this entire file once deprecated Retry options are removed in v2. -@pytest.fixture(scope="function") -def expect_retry_deprecation(): - with warnings.catch_warnings(record=True) as w: - yield - assert len([str(x.message) for x in w if "Retry" in str(x.message)]) > 0 - - -class TestRetry(object): - def test_string(self): - """Retry string representation looks the way we expect""" - retry = Retry() - assert ( - str(retry) - == "Retry(total=10, connect=None, read=None, redirect=None, status=None)" - ) - for _ in range(3): - retry = retry.increment(method="GET") - assert ( - str(retry) - == "Retry(total=7, connect=None, read=None, redirect=None, status=None)" - ) - - def test_retry_both_specified(self): - """Total can win if it's lower than the connect value""" - error = ConnectTimeoutError() - retry = Retry(connect=3, total=2) - retry = retry.increment(error=error) - retry = retry.increment(error=error) - with pytest.raises(MaxRetryError) as e: - retry.increment(error=error) - assert e.value.reason == error - - def test_retry_higher_total_loses(self): - """A lower connect timeout than the total is honored""" - error = ConnectTimeoutError() - retry = Retry(connect=2, total=3) - retry = retry.increment(error=error) - retry = retry.increment(error=error) - with pytest.raises(MaxRetryError): - retry.increment(error=error) - - def test_retry_higher_total_loses_vs_read(self): - """A lower read timeout than the total is honored""" - error = ReadTimeoutError(None, "/", "read timed out") - retry = Retry(read=2, total=3) - retry = retry.increment(method="GET", error=error) - retry = retry.increment(method="GET", error=error) - with pytest.raises(MaxRetryError): - retry.increment(method="GET", error=error) - - def test_retry_total_none(self): - """if Total is none, connect error should take precedence""" - error = ConnectTimeoutError() - retry = Retry(connect=2, total=None) - retry = retry.increment(error=error) - retry = retry.increment(error=error) - with pytest.raises(MaxRetryError) as e: - retry.increment(error=error) - assert e.value.reason == error - - error = ReadTimeoutError(None, "/", "read timed out") - retry = Retry(connect=2, total=None) - retry = retry.increment(method="GET", error=error) - retry = retry.increment(method="GET", error=error) - retry = retry.increment(method="GET", error=error) - assert not retry.is_exhausted() - - def test_retry_default(self): - """If no value is specified, should retry connects 3 times""" - retry = Retry() - assert retry.total == 10 - assert retry.connect is None - assert retry.read is None - assert retry.redirect is None - assert retry.other is None - - error = ConnectTimeoutError() - retry = Retry(connect=1) - retry = retry.increment(error=error) - with pytest.raises(MaxRetryError): - retry.increment(error=error) - - retry = Retry(connect=1) - retry = retry.increment(error=error) - assert not retry.is_exhausted() - - assert Retry(0).raise_on_redirect - assert not Retry(False).raise_on_redirect - - def test_retry_other(self): - """If an unexpected error is raised, should retry other times""" - other_error = SSLError() - retry = Retry(connect=1) - retry = retry.increment(error=other_error) - retry = retry.increment(error=other_error) - assert not retry.is_exhausted() - - retry = Retry(other=1) - retry = retry.increment(error=other_error) - with pytest.raises(MaxRetryError) as e: - retry.increment(error=other_error) - assert e.value.reason == other_error - - def test_retry_read_zero(self): - """No second chances on read timeouts, by default""" - error = ReadTimeoutError(None, "/", "read timed out") - retry = Retry(read=0) - with pytest.raises(MaxRetryError) as e: - retry.increment(method="GET", error=error) - assert e.value.reason == error - - def test_status_counter(self): - resp = HTTPResponse(status=400) - retry = Retry(status=2) - retry = retry.increment(response=resp) - retry = retry.increment(response=resp) - with pytest.raises(MaxRetryError) as e: - retry.increment(response=resp) - assert str(e.value.reason) == ResponseError.SPECIFIC_ERROR.format( - status_code=400 - ) - - def test_backoff(self): - """Backoff is computed correctly""" - max_backoff = Retry.DEFAULT_BACKOFF_MAX - - retry = Retry(total=100, backoff_factor=0.2) - assert retry.get_backoff_time() == 0 # First request - - retry = retry.increment(method="GET") - assert retry.get_backoff_time() == 0 # First retry - - retry = retry.increment(method="GET") - assert retry.backoff_factor == 0.2 - assert retry.total == 98 - assert retry.get_backoff_time() == 0.4 # Start backoff - - retry = retry.increment(method="GET") - assert retry.get_backoff_time() == 0.8 - - retry = retry.increment(method="GET") - assert retry.get_backoff_time() == 1.6 - - for _ in xrange(10): - retry = retry.increment(method="GET") - - assert retry.get_backoff_time() == max_backoff - - def test_zero_backoff(self): - retry = Retry() - assert retry.get_backoff_time() == 0 - retry = retry.increment(method="GET") - retry = retry.increment(method="GET") - assert retry.get_backoff_time() == 0 - - def test_backoff_reset_after_redirect(self): - retry = Retry(total=100, redirect=5, backoff_factor=0.2) - retry = retry.increment(method="GET") - retry = retry.increment(method="GET") - assert retry.get_backoff_time() == 0.4 - redirect_response = HTTPResponse(status=302, headers={"location": "test"}) - retry = retry.increment(method="GET", response=redirect_response) - assert retry.get_backoff_time() == 0 - retry = retry.increment(method="GET") - retry = retry.increment(method="GET") - assert retry.get_backoff_time() == 0.4 - - def test_sleep(self): - # sleep a very small amount of time so our code coverage is happy - retry = Retry(backoff_factor=0.0001) - retry = retry.increment(method="GET") - retry = retry.increment(method="GET") - retry.sleep() - - def test_status_forcelist(self): - retry = Retry(status_forcelist=xrange(500, 600)) - assert not retry.is_retry("GET", status_code=200) - assert not retry.is_retry("GET", status_code=400) - assert retry.is_retry("GET", status_code=500) - - retry = Retry(total=1, status_forcelist=[418]) - assert not retry.is_retry("GET", status_code=400) - assert retry.is_retry("GET", status_code=418) - - # String status codes are not matched. - retry = Retry(total=1, status_forcelist=["418"]) - assert not retry.is_retry("GET", status_code=418) - - def test_method_whitelist_with_status_forcelist(self, expect_retry_deprecation): - # Falsey method_whitelist means to retry on any method. - retry = Retry(status_forcelist=[500], method_whitelist=None) - assert retry.is_retry("GET", status_code=500) - assert retry.is_retry("POST", status_code=500) - - # Criteria of method_whitelist and status_forcelist are ANDed. - retry = Retry(status_forcelist=[500], method_whitelist=["POST"]) - assert not retry.is_retry("GET", status_code=500) - assert retry.is_retry("POST", status_code=500) - - def test_exhausted(self): - assert not Retry(0).is_exhausted() - assert Retry(-1).is_exhausted() - assert Retry(1).increment(method="GET").total == 0 - - @pytest.mark.parametrize("total", [-1, 0]) - def test_disabled(self, total): - with pytest.raises(MaxRetryError): - Retry(total).increment(method="GET") - - def test_error_message(self): - retry = Retry(total=0) - with pytest.raises(MaxRetryError) as e: - retry = retry.increment( - method="GET", error=ReadTimeoutError(None, "/", "read timed out") - ) - assert "Caused by redirect" not in str(e.value) - assert str(e.value.reason) == "None: read timed out" - - retry = Retry(total=1) - with pytest.raises(MaxRetryError) as e: - retry = retry.increment("POST", "/") - retry = retry.increment("POST", "/") - assert "Caused by redirect" not in str(e.value) - assert isinstance(e.value.reason, ResponseError) - assert str(e.value.reason) == ResponseError.GENERIC_ERROR - - retry = Retry(total=1) - response = HTTPResponse(status=500) - with pytest.raises(MaxRetryError) as e: - retry = retry.increment("POST", "/", response=response) - retry = retry.increment("POST", "/", response=response) - assert "Caused by redirect" not in str(e.value) - msg = ResponseError.SPECIFIC_ERROR.format(status_code=500) - assert str(e.value.reason) == msg - - retry = Retry(connect=1) - with pytest.raises(MaxRetryError) as e: - retry = retry.increment(error=ConnectTimeoutError("conntimeout")) - retry = retry.increment(error=ConnectTimeoutError("conntimeout")) - assert "Caused by redirect" not in str(e.value) - assert str(e.value.reason) == "conntimeout" - - def test_history(self, expect_retry_deprecation): - retry = Retry(total=10, method_whitelist=frozenset(["GET", "POST"])) - assert retry.history == tuple() - connection_error = ConnectTimeoutError("conntimeout") - retry = retry.increment("GET", "/test1", None, connection_error) - history = (RequestHistory("GET", "/test1", connection_error, None, None),) - assert retry.history == history - - read_error = ReadTimeoutError(None, "/test2", "read timed out") - retry = retry.increment("POST", "/test2", None, read_error) - history = ( - RequestHistory("GET", "/test1", connection_error, None, None), - RequestHistory("POST", "/test2", read_error, None, None), - ) - assert retry.history == history - - response = HTTPResponse(status=500) - retry = retry.increment("GET", "/test3", response, None) - history = ( - RequestHistory("GET", "/test1", connection_error, None, None), - RequestHistory("POST", "/test2", read_error, None, None), - RequestHistory("GET", "/test3", None, 500, None), - ) - assert retry.history == history - - def test_retry_method_not_in_whitelist(self): - error = ReadTimeoutError(None, "/", "read timed out") - retry = Retry() - with pytest.raises(ReadTimeoutError): - retry.increment(method="POST", error=error) - - def test_retry_default_remove_headers_on_redirect(self): - retry = Retry() - - assert retry.remove_headers_on_redirect == {"authorization", "cookie"} - - def test_retry_set_remove_headers_on_redirect(self): - retry = Retry(remove_headers_on_redirect=["X-API-Secret"]) - - assert list(retry.remove_headers_on_redirect) == ["x-api-secret"] - - @pytest.mark.parametrize("value", ["-1", "+1", "1.0", six.u("\xb2")]) # \xb2 = ^2 - def test_parse_retry_after_invalid(self, value): - retry = Retry() - with pytest.raises(InvalidHeader): - retry.parse_retry_after(value) - - @pytest.mark.parametrize( - "value, expected", [("0", 0), ("1000", 1000), ("\t42 ", 42)] - ) - def test_parse_retry_after(self, value, expected): - retry = Retry() - assert retry.parse_retry_after(value) == expected - - @pytest.mark.parametrize("respect_retry_after_header", [True, False]) - def test_respect_retry_after_header_propagated(self, respect_retry_after_header): - - retry = Retry(respect_retry_after_header=respect_retry_after_header) - new_retry = retry.new() - assert new_retry.respect_retry_after_header == respect_retry_after_header - - @pytest.mark.freeze_time("2019-06-03 11:00:00", tz_offset=0) - @pytest.mark.parametrize( - "retry_after_header,respect_retry_after_header,sleep_duration", - [ - ("3600", True, 3600), - ("3600", False, None), - # Will sleep due to header is 1 hour in future - ("Mon, 3 Jun 2019 12:00:00 UTC", True, 3600), - # Won't sleep due to not respecting header - ("Mon, 3 Jun 2019 12:00:00 UTC", False, None), - # Won't sleep due to current time reached - ("Mon, 3 Jun 2019 11:00:00 UTC", True, None), - # Won't sleep due to current time reached + not respecting header - ("Mon, 3 Jun 2019 11:00:00 UTC", False, None), - # Handle all the formats in RFC 7231 Section 7.1.1.1 - ("Mon, 03 Jun 2019 11:30:12 GMT", True, 1812), - ("Monday, 03-Jun-19 11:30:12 GMT", True, 1812), - # Assume that datetimes without a timezone are in UTC per RFC 7231 - ("Mon Jun 3 11:30:12 2019", True, 1812), - ], - ) - @pytest.mark.parametrize( - "stub_timezone", - [ - "UTC", - "Asia/Jerusalem", - None, - ], - indirect=True, - ) - @pytest.mark.usefixtures("stub_timezone") - def test_respect_retry_after_header_sleep( - self, retry_after_header, respect_retry_after_header, sleep_duration - ): - retry = Retry(respect_retry_after_header=respect_retry_after_header) - - with mock.patch("time.sleep") as sleep_mock: - # for the default behavior, it must be in RETRY_AFTER_STATUS_CODES - response = HTTPResponse( - status=503, headers={"Retry-After": retry_after_header} - ) - - retry.sleep(response) - - # The expected behavior is that we'll only sleep if respecting - # this header (since we won't have any backoff sleep attempts) - if respect_retry_after_header and sleep_duration is not None: - sleep_mock.assert_called_with(sleep_duration) - else: - sleep_mock.assert_not_called() - - -class TestRetryDeprecations(object): - def test_cls_get_default_method_whitelist(self, expect_retry_deprecation): - assert Retry.DEFAULT_ALLOWED_METHODS == Retry.DEFAULT_METHOD_WHITELIST - - def test_cls_get_default_redirect_headers_blacklist(self, expect_retry_deprecation): - assert ( - Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT - == Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST - ) - - def test_cls_get_default_backoff_max(self, expect_retry_deprecation): - assert Retry.DEFAULT_BACKOFF_MAX == Retry.BACKOFF_MAX - - def test_cls_set_default_method_whitelist(self, expect_retry_deprecation): - old_setting = Retry.DEFAULT_METHOD_WHITELIST - try: - Retry.DEFAULT_METHOD_WHITELIST = {"GET"} - retry = Retry() - assert retry.DEFAULT_ALLOWED_METHODS == {"GET"} - assert retry.DEFAULT_METHOD_WHITELIST == {"GET"} - assert retry.allowed_methods == {"GET"} - assert retry.method_whitelist == {"GET"} - - # Test that the default can be overridden both ways - retry = Retry(allowed_methods={"GET", "POST"}) - assert retry.DEFAULT_ALLOWED_METHODS == {"GET"} - assert retry.DEFAULT_METHOD_WHITELIST == {"GET"} - assert retry.allowed_methods == {"GET", "POST"} - assert retry.method_whitelist == {"GET", "POST"} - - retry = Retry(method_whitelist={"POST"}) - assert retry.DEFAULT_ALLOWED_METHODS == {"GET"} - assert retry.DEFAULT_METHOD_WHITELIST == {"GET"} - assert retry.allowed_methods == {"POST"} - assert retry.method_whitelist == {"POST"} - finally: - Retry.DEFAULT_METHOD_WHITELIST = old_setting - assert Retry.DEFAULT_ALLOWED_METHODS == old_setting - - def test_cls_set_default_redirect_headers_blacklist(self, expect_retry_deprecation): - old_setting = Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST - try: - Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST = {"test"} - retry = Retry() - assert retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT == {"test"} - assert retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST == {"test"} - assert retry.remove_headers_on_redirect == {"test"} - assert retry.remove_headers_on_redirect == {"test"} - - retry = Retry(remove_headers_on_redirect={"test2"}) - assert retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT == {"test"} - assert retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST == {"test"} - assert retry.remove_headers_on_redirect == {"test2"} - assert retry.remove_headers_on_redirect == {"test2"} - finally: - Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST = old_setting - assert Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST == old_setting - - def test_cls_set_default_backoff_max(self, expect_retry_deprecation): - old_setting = Retry.BACKOFF_MAX - try: - Retry.BACKOFF_MAX = 99 - retry = Retry() - assert retry.DEFAULT_BACKOFF_MAX == 99 - assert retry.BACKOFF_MAX == 99 - finally: - Retry.BACKOFF_MAX = old_setting - assert Retry.BACKOFF_MAX == old_setting - - @pytest.mark.parametrize( - "options", [(None, None), ({"GET"}, None), (None, {"GET"}), ({"GET"}, {"GET"})] - ) - def test_retry_allowed_methods_and_method_whitelist_error(self, options): - with pytest.raises(ValueError) as e: - Retry(allowed_methods=options[0], method_whitelist=options[1]) - assert str(e.value) == ( - "Using both 'allowed_methods' and 'method_whitelist' together " - "is not allowed. Instead only use 'allowed_methods'" - ) - - def test_retry_subclass_that_sets_method_whitelist(self, expect_retry_deprecation): - class SubclassRetry(Retry): - def __init__(self, **kwargs): - if "allowed_methods" in kwargs: - raise AssertionError( - "This subclass likely doesn't use 'allowed_methods'" - ) - - super(SubclassRetry, self).__init__(**kwargs) - - # Since we're setting 'method_whiteist' we get fallbacks - # within Retry.new() and Retry._is_method_retryable() - # to use 'method_whitelist' instead of 'allowed_methods' - self.method_whitelist = self.method_whitelist | {"POST"} - - retry = SubclassRetry() - assert retry.method_whitelist == Retry.DEFAULT_ALLOWED_METHODS | {"POST"} - assert retry.new(read=0).method_whitelist == retry.method_whitelist - assert retry._is_method_retryable("POST") - assert not retry._is_method_retryable("CONNECT") - - assert retry.new(method_whitelist={"GET"}).method_whitelist == {"GET", "POST"} - - # urllib3 doesn't do this during normal operation - # so we don't want users passing in 'allowed_methods' - # when their subclass doesn't support the option yet. - with pytest.raises(AssertionError) as e: - retry.new(allowed_methods={"GET"}) - assert str(e.value) == "This subclass likely doesn't use 'allowed_methods'" diff --git a/test/test_ssl.py b/test/test_ssl.py index 4a00d35..43073cb 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -1,170 +1,232 @@ -from test import notPyPy2 +from __future__ import annotations + +import ssl +import typing +from unittest import mock -import mock import pytest -from urllib3.exceptions import SNIMissingWarning +from urllib3.exceptions import ProxySchemeUnsupported, SSLError from urllib3.util import ssl_ -@pytest.mark.parametrize( - "addr", - [ - # IPv6 - "::1", - "::", - "FE80::8939:7684:D84b:a5A4%251", - # IPv4 - "127.0.0.1", - "8.8.8.8", - b"127.0.0.1", - # IPv6 w/ Zone IDs - "FE80::8939:7684:D84b:a5A4%251", - b"FE80::8939:7684:D84b:a5A4%251", - "FE80::8939:7684:D84b:a5A4%19", - b"FE80::8939:7684:D84b:a5A4%19", - ], -) -def test_is_ipaddress_true(addr): - assert ssl_.is_ipaddress(addr) - - -@pytest.mark.parametrize( - "addr", - [ - "www.python.org", - b"www.python.org", - "v2.sg.media-imdb.com", - b"v2.sg.media-imdb.com", - ], -) -def test_is_ipaddress_false(addr): - assert not ssl_.is_ipaddress(addr) - - -@pytest.mark.parametrize( - ["has_sni", "server_hostname", "uses_sni"], - [ - (True, "127.0.0.1", False), - (False, "www.python.org", False), - (False, "0.0.0.0", False), - (True, "www.google.com", True), - (True, None, False), - (False, None, False), - ], -) -def test_context_sni_with_ip_address(monkeypatch, has_sni, server_hostname, uses_sni): - monkeypatch.setattr(ssl_, "HAS_SNI", has_sni) - - sock = mock.Mock() - context = mock.create_autospec(ssl_.SSLContext) - - ssl_.ssl_wrap_socket(sock, server_hostname=server_hostname, ssl_context=context) - - if uses_sni: - context.wrap_socket.assert_called_with(sock, server_hostname=server_hostname) - else: - context.wrap_socket.assert_called_with(sock) - - -@pytest.mark.parametrize( - ["has_sni", "server_hostname", "should_warn"], - [ - (True, "www.google.com", False), - (True, "127.0.0.1", False), - (False, "127.0.0.1", False), - (False, "www.google.com", True), - (True, None, False), - (False, None, False), - ], -) -def test_sni_missing_warning_with_ip_addresses( - monkeypatch, has_sni, server_hostname, should_warn -): - monkeypatch.setattr(ssl_, "HAS_SNI", has_sni) - - sock = mock.Mock() - context = mock.create_autospec(ssl_.SSLContext) - - with mock.patch("warnings.warn") as warn: - ssl_.ssl_wrap_socket(sock, server_hostname=server_hostname, ssl_context=context) - - if should_warn: - assert warn.call_count >= 1 - warnings = [call[0][1] for call in warn.call_args_list] - assert SNIMissingWarning in warnings - else: - assert warn.call_count == 0 - - -@pytest.mark.parametrize( - ["ciphers", "expected_ciphers"], - [ - (None, ssl_.DEFAULT_CIPHERS), - ("ECDH+AESGCM:ECDH+CHACHA20", "ECDH+AESGCM:ECDH+CHACHA20"), - ], -) -def test_create_urllib3_context_set_ciphers(monkeypatch, ciphers, expected_ciphers): - - context = mock.create_autospec(ssl_.SSLContext) - context.set_ciphers = mock.Mock() - context.options = 0 - monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) - - assert ssl_.create_urllib3_context(ciphers=ciphers) is context - - assert context.set_ciphers.call_count == 1 - assert context.set_ciphers.call_args == mock.call(expected_ciphers) - - -def test_wrap_socket_given_context_no_load_default_certs(): - context = mock.create_autospec(ssl_.SSLContext) - context.load_default_certs = mock.Mock() - - sock = mock.Mock() - ssl_.ssl_wrap_socket(sock, ssl_context=context) - - context.load_default_certs.assert_not_called() - - -@notPyPy2 -def test_wrap_socket_given_ca_certs_no_load_default_certs(monkeypatch): - context = mock.create_autospec(ssl_.SSLContext) - context.load_default_certs = mock.Mock() - context.options = 0 - - monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) - - sock = mock.Mock() - ssl_.ssl_wrap_socket(sock, ca_certs="/tmp/fake-file") - - context.load_default_certs.assert_not_called() - context.load_verify_locations.assert_called_with("/tmp/fake-file", None, None) - - -def test_wrap_socket_default_loads_default_certs(monkeypatch): - context = mock.create_autospec(ssl_.SSLContext) - context.load_default_certs = mock.Mock() - context.options = 0 - - monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) - - sock = mock.Mock() - ssl_.ssl_wrap_socket(sock) - - context.load_default_certs.assert_called_with() - - -@pytest.mark.parametrize( - ["pha", "expected_pha"], [(None, None), (False, True), (True, True)] -) -def test_create_urllib3_context_pha(monkeypatch, pha, expected_pha): - context = mock.create_autospec(ssl_.SSLContext) - context.set_ciphers = mock.Mock() - context.options = 0 - context.post_handshake_auth = pha - monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) - - assert ssl_.create_urllib3_context() is context - - assert context.post_handshake_auth == expected_pha +class TestSSL: + @pytest.mark.parametrize( + "addr", + [ + # IPv6 + "::1", + "::", + "FE80::8939:7684:D84b:a5A4%251", + # IPv4 + "127.0.0.1", + "8.8.8.8", + b"127.0.0.1", + # IPv6 w/ Zone IDs + "FE80::8939:7684:D84b:a5A4%251", + b"FE80::8939:7684:D84b:a5A4%251", + "FE80::8939:7684:D84b:a5A4%19", + b"FE80::8939:7684:D84b:a5A4%19", + ], + ) + def test_is_ipaddress_true(self, addr: bytes | str) -> None: + assert ssl_.is_ipaddress(addr) + + @pytest.mark.parametrize( + "addr", + [ + "www.python.org", + b"www.python.org", + "v2.sg.media-imdb.com", + b"v2.sg.media-imdb.com", + ], + ) + def test_is_ipaddress_false(self, addr: bytes | str) -> None: + assert not ssl_.is_ipaddress(addr) + + def test_create_urllib3_context_set_ciphers( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + ciphers = "ECDH+AESGCM:ECDH+CHACHA20" + context = mock.create_autospec(ssl_.SSLContext) + context.set_ciphers = mock.Mock() + context.options = 0 + monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) + + assert ssl_.create_urllib3_context(ciphers=ciphers) is context + + assert context.set_ciphers.call_count == 1 + assert context.set_ciphers.call_args == mock.call(ciphers) + + def test_create_urllib3_no_context(self) -> None: + with mock.patch("urllib3.util.ssl_.SSLContext", None): + with pytest.raises(TypeError): + ssl_.create_urllib3_context() + + def test_wrap_socket_given_context_no_load_default_certs(self) -> None: + context = mock.create_autospec(ssl_.SSLContext) + context.load_default_certs = mock.Mock() + + sock = mock.Mock() + ssl_.ssl_wrap_socket(sock, ssl_context=context) + + context.load_default_certs.assert_not_called() + + def test_wrap_socket_given_ca_certs_no_load_default_certs( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + context = mock.create_autospec(ssl_.SSLContext) + context.load_default_certs = mock.Mock() + context.options = 0 + + monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) + + sock = mock.Mock() + ssl_.ssl_wrap_socket(sock, ca_certs="/tmp/fake-file") + + context.load_default_certs.assert_not_called() + context.load_verify_locations.assert_called_with("/tmp/fake-file", None, None) + + def test_wrap_socket_default_loads_default_certs( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + context = mock.create_autospec(ssl_.SSLContext) + context.load_default_certs = mock.Mock() + context.options = 0 + + monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) + + sock = mock.Mock() + ssl_.ssl_wrap_socket(sock) + + context.load_default_certs.assert_called_with() + + def test_wrap_socket_no_ssltransport(self) -> None: + with mock.patch("urllib3.util.ssl_.SSLTransport", None): + with pytest.raises(ProxySchemeUnsupported): + sock = mock.Mock() + ssl_.ssl_wrap_socket(sock, tls_in_tls=True) + + @pytest.mark.parametrize( + ["pha", "expected_pha", "cert_reqs"], + [ + (None, None, None), + (None, None, ssl.CERT_NONE), + (None, None, ssl.CERT_OPTIONAL), + (None, None, ssl.CERT_REQUIRED), + (False, True, None), + (False, True, ssl.CERT_NONE), + (False, True, ssl.CERT_OPTIONAL), + (False, True, ssl.CERT_REQUIRED), + (True, True, None), + (True, True, ssl.CERT_NONE), + (True, True, ssl.CERT_OPTIONAL), + (True, True, ssl.CERT_REQUIRED), + ], + ) + def test_create_urllib3_context_pha( + self, + monkeypatch: pytest.MonkeyPatch, + pha: bool | None, + expected_pha: bool | None, + cert_reqs: int | None, + ) -> None: + context = mock.create_autospec(ssl_.SSLContext) + context.set_ciphers = mock.Mock() + context.options = 0 + context.post_handshake_auth = pha + monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) + + assert ssl_.create_urllib3_context(cert_reqs=cert_reqs) is context + + assert context.post_handshake_auth == expected_pha + + def test_create_urllib3_context_default_ciphers( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + context = mock.create_autospec(ssl_.SSLContext) + context.set_ciphers = mock.Mock() + context.options = 0 + monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) + + ssl_.create_urllib3_context() + + context.set_ciphers.assert_not_called() + + @pytest.mark.parametrize( + "kwargs", + [ + { + "ssl_version": ssl.PROTOCOL_TLSv1, + "ssl_minimum_version": ssl.TLSVersion.MINIMUM_SUPPORTED, + }, + { + "ssl_version": ssl.PROTOCOL_TLSv1, + "ssl_maximum_version": ssl.TLSVersion.TLSv1, + }, + { + "ssl_version": ssl.PROTOCOL_TLSv1, + "ssl_minimum_version": ssl.TLSVersion.MINIMUM_SUPPORTED, + "ssl_maximum_version": ssl.TLSVersion.MAXIMUM_SUPPORTED, + }, + ], + ) + def test_create_urllib3_context_ssl_version_and_ssl_min_max_version_errors( + self, kwargs: dict[str, typing.Any] + ) -> None: + with pytest.raises(ValueError) as e: + ssl_.create_urllib3_context(**kwargs) + + assert str(e.value) == ( + "Can't specify both 'ssl_version' and either 'ssl_minimum_version' or 'ssl_maximum_version'" + ) + + @pytest.mark.parametrize( + "kwargs", + [ + { + "ssl_version": ssl.PROTOCOL_TLS, + "ssl_minimum_version": ssl.TLSVersion.MINIMUM_SUPPORTED, + }, + { + "ssl_version": ssl.PROTOCOL_TLS_CLIENT, + "ssl_minimum_version": ssl.TLSVersion.MINIMUM_SUPPORTED, + }, + { + "ssl_version": None, + "ssl_minimum_version": ssl.TLSVersion.MINIMUM_SUPPORTED, + }, + ], + ) + def test_create_urllib3_context_ssl_version_and_ssl_min_max_version_no_warning( + self, kwargs: dict[str, typing.Any] + ) -> None: + ssl_.create_urllib3_context(**kwargs) + + @pytest.mark.parametrize( + "kwargs", + [ + {"ssl_version": ssl.PROTOCOL_TLSv1, "ssl_minimum_version": None}, + {"ssl_version": ssl.PROTOCOL_TLSv1, "ssl_maximum_version": None}, + { + "ssl_version": ssl.PROTOCOL_TLSv1, + "ssl_minimum_version": None, + "ssl_maximum_version": None, + }, + ], + ) + def test_create_urllib3_context_ssl_version_and_ssl_min_max_version_no_error( + self, kwargs: dict[str, typing.Any] + ) -> None: + with pytest.warns( + DeprecationWarning, + match=r"'ssl_version' option is deprecated and will be removed in " + r"urllib3 v2\.1\.0\. Instead use 'ssl_minimum_version'", + ): + ssl_.create_urllib3_context(**kwargs) + + def test_assert_fingerprint_raises_exception_on_none_cert(self) -> None: + with pytest.raises(SSLError): + ssl_.assert_fingerprint( + cert=None, fingerprint="55:39:BF:70:05:12:43:FA:1F:D1:BF:4E:E8:1B:07:1D" + ) diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py index c78121f..4f0880d 100644 --- a/test/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import platform import select import socket import ssl -import sys +import threading +import typing +from unittest import mock -import mock import pytest -from dummyserver.server import DEFAULT_CA, DEFAULT_CERTS +from dummyserver.socketserver import DEFAULT_CA, DEFAULT_CERTS from dummyserver.testcase import SocketDummyServerTestCase, consume_socket from urllib3.util import ssl_ from urllib3.util.ssltransport import SSLTransport @@ -16,29 +19,27 @@ PER_TEST_TIMEOUT = 60 -def server_client_ssl_contexts(): +def server_client_ssl_contexts() -> tuple[ssl.SSLContext, ssl.SSLContext]: if hasattr(ssl, "PROTOCOL_TLS_SERVER"): server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - else: - # python 2.7 workaround. - # PROTOCOL_TLS_SERVER was added in 3.6 - server_context = ssl.SSLContext(ssl.PROTOCOL_TLS) server_context.load_cert_chain(DEFAULT_CERTS["certfile"], DEFAULT_CERTS["keyfile"]) if hasattr(ssl, "PROTOCOL_TLS_CLIENT"): client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - else: - # python 2.7 workaround. - # PROTOCOL_TLS_SERVER was added in 3.6 - client_context = ssl.SSLContext(ssl.PROTOCOL_TLS) - client_context.verify_mode = ssl.CERT_REQUIRED - client_context.check_hostname = True client_context.load_verify_locations(DEFAULT_CA) return server_context, client_context -def sample_request(binary=True): +@typing.overload +def sample_request(binary: typing.Literal[True] = ...) -> bytes: ... + + +@typing.overload +def sample_request(binary: typing.Literal[False]) -> str: ... + + +def sample_request(binary: bool = True) -> bytes | str: request = ( b"GET http://www.testing.com/ HTTP/1.1\r\n" b"Host: www.testing.com\r\n" @@ -48,36 +49,50 @@ def sample_request(binary=True): return request if binary else request.decode("utf-8") -def validate_request(provided_request, binary=True): +def validate_request( + provided_request: bytearray, binary: typing.Literal[False, True] = True +) -> None: assert provided_request is not None expected_request = sample_request(binary) assert provided_request == expected_request -def sample_response(binary=True): +@typing.overload +def sample_response(binary: typing.Literal[True] = ...) -> bytes: ... + + +@typing.overload +def sample_response(binary: typing.Literal[False]) -> str: ... + + +@typing.overload +def sample_response(binary: bool = ...) -> bytes | str: ... + + +def sample_response(binary: bool = True) -> bytes | str: response = b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n" return response if binary else response.decode("utf-8") -def validate_response(provided_response, binary=True): +def validate_response( + provided_response: bytes | bytearray | str, binary: bool = True +) -> None: assert provided_response is not None expected_response = sample_response(binary) assert provided_response == expected_response -def validate_peercert(ssl_socket): - +def validate_peercert(ssl_socket: SSLTransport) -> None: binary_cert = ssl_socket.getpeercert(binary_form=True) - assert type(binary_cert) == bytes + assert type(binary_cert) is bytes assert len(binary_cert) > 0 cert = ssl_socket.getpeercert() - assert type(cert) == dict + assert type(cert) is dict assert "serialNumber" in cert assert cert["serialNumber"] != "" -@pytest.mark.skipif(sys.version_info < (3, 5), reason="requires python3.5 or higher") class SingleTLSLayerTestCase(SocketDummyServerTestCase): """ Uses the SocketDummyServer to validate a single TLS layer can be @@ -85,22 +100,36 @@ class SingleTLSLayerTestCase(SocketDummyServerTestCase): """ @classmethod - def setup_class(cls): + def setup_class(cls) -> None: cls.server_context, cls.client_context = server_client_ssl_contexts() - def start_dummy_server(self, handler=None): - def socket_handler(listener): + def start_dummy_server( + self, + handler: typing.Callable[[socket.socket], None] | None = None, + validate: bool = True, + ) -> None: + quit_event = threading.Event() + + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] - with self.server_context.wrap_socket(sock, server_side=True) as ssock: - request = consume_socket(ssock) - validate_request(request) - ssock.send(sample_response()) + try: + with self.server_context.wrap_socket(sock, server_side=True) as ssock: + request = consume_socket( + ssock, + quit_event=quit_event, + ) + if not validate: + return + validate_request(request) + ssock.send(sample_response()) + except (ConnectionAbortedError, ConnectionResetError): + return chosen_handler = handler if handler else socket_handler - self._start_server(chosen_handler) + self._start_server(chosen_handler, quit_event=quit_event) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_start_closed_socket(self): + def test_start_closed_socket(self) -> None: """Errors generated from an unconnected socket should bubble up.""" sock = socket.socket(socket.AF_INET) context = ssl.create_default_context() @@ -109,9 +138,9 @@ def test_start_closed_socket(self): SSLTransport(sock, context) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_close_after_handshake(self): + def test_close_after_handshake(self) -> None: """Socket errors should be bubbled up""" - self.start_dummy_server() + self.start_dummy_server(validate=False) sock = socket.create_connection((self.host, self.port)) with SSLTransport( @@ -122,7 +151,7 @@ def test_close_after_handshake(self): ssock.send(b"blaaargh") @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_wrap_existing_socket(self): + def test_wrap_existing_socket(self) -> None: """Validates a single TLS layer can be established.""" self.start_dummy_server() @@ -136,7 +165,7 @@ def test_wrap_existing_socket(self): validate_response(response) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_unbuffered_text_makefile(self): + def test_unbuffered_text_makefile(self) -> None: self.start_dummy_server() sock = socket.create_connection((self.host, self.port)) @@ -150,43 +179,43 @@ def test_unbuffered_text_makefile(self): validate_response(response) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_unwrap_existing_socket(self): + def test_unwrap_existing_socket(self) -> None: """ Validates we can break up the TLS layer A full request/response is sent over TLS, and later over plain text. """ - def shutdown_handler(listener): - sock = listener.accept()[0] - ssl_sock = self.server_context.wrap_socket(sock, server_side=True) - - request = consume_socket(ssl_sock) - validate_request(request) - ssl_sock.sendall(sample_response()) - - unwrapped_sock = ssl_sock.unwrap() + def shutdown_handler(listener: socket.socket) -> None: + with ( + listener.accept()[0] as sock, + self.server_context.wrap_socket(sock, server_side=True) as ssl_sock, + ): + request = consume_socket(ssl_sock) + validate_request(request) + ssl_sock.sendall(sample_response()) - request = consume_socket(unwrapped_sock) - validate_request(request) - unwrapped_sock.sendall(sample_response()) + with ssl_sock.unwrap() as unwrapped_sock: + request = consume_socket(unwrapped_sock) + validate_request(request) + unwrapped_sock.sendall(sample_response()) self.start_dummy_server(shutdown_handler) - sock = socket.create_connection((self.host, self.port)) - ssock = SSLTransport(sock, self.client_context, server_hostname="localhost") + with socket.create_connection((self.host, self.port)) as sock: + ssock = SSLTransport(sock, self.client_context, server_hostname="localhost") - # request/response over TLS. - ssock.sendall(sample_request()) - response = consume_socket(ssock) - validate_response(response) + # request/response over TLS. + ssock.sendall(sample_request()) + response = consume_socket(ssock) + validate_response(response) - # request/response over plaintext after unwrap. - ssock.unwrap() - sock.sendall(sample_request()) - response = consume_socket(sock) - validate_response(response) + # request/response over plaintext after unwrap. + ssock.unwrap() + sock.sendall(sample_request()) + response = consume_socket(sock) + validate_response(response) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_ssl_object_attributes(self): + def test_ssl_object_attributes(self) -> None: """Ensures common ssl attributes are exposed""" self.start_dummy_server() @@ -195,11 +224,10 @@ def test_ssl_object_attributes(self): sock, self.client_context, server_hostname="localhost" ) as ssock: cipher = ssock.cipher() - assert type(cipher) == tuple + assert type(cipher) is tuple - # No chosen protocol through ALPN or NPN. + # No chosen protocol. assert ssock.selected_alpn_protocol() is None - assert ssock.selected_npn_protocol() is None shared_ciphers = ssock.shared_ciphers() # SSLContext.shared_ciphers() changed behavior completely in a patch version. @@ -217,7 +245,7 @@ def test_ssl_object_attributes(self): validate_response(response) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_socket_object_attributes(self): + def test_socket_object_attributes(self) -> None: """Ensures common socket attributes are exposed""" self.start_dummy_server() @@ -242,20 +270,22 @@ class SocketProxyDummyServer(SocketDummyServerTestCase): socket. """ - def __init__(self, destination_server_host, destination_server_port): + def __init__( + self, destination_server_host: str, destination_server_port: int + ) -> None: self.destination_server_host = destination_server_host self.destination_server_port = destination_server_port - self.server_context, self.client_context = server_client_ssl_contexts() + self.server_ctx, _ = server_client_ssl_contexts() - def start_proxy_handler(self): + def start_proxy_handler(self) -> None: """ Socket handler for the proxy. Terminates the first TLS layer and tunnels any bytes needed for client <-> server communicatin. """ - def proxy_handler(listener): + def proxy_handler(listener: socket.socket) -> None: sock = listener.accept()[0] - with self.server_context.wrap_socket(sock, server_side=True) as client_sock: + with self.server_ctx.wrap_socket(sock, server_side=True) as client_sock: upstream_sock = socket.create_connection( (self.destination_server_host, self.destination_server_port) ) @@ -265,7 +295,12 @@ def proxy_handler(listener): self._start_server(proxy_handler) - def _read_write_loop(self, client_sock, server_sock, chunks=65536): + def _read_write_loop( + self, + client_sock: socket.socket, + server_sock: socket.socket, + chunks: int = 65536, + ) -> None: inputs = [client_sock, server_sock] output = [client_sock, server_sock] @@ -273,7 +308,7 @@ def _read_write_loop(self, client_sock, server_sock, chunks=65536): readable, writable, exception = select.select(inputs, output, inputs) if exception: - # Error ocurred with either of the sockets, time to + # Error occurred with either of the sockets, time to # wrap up, parent func will close sockets. break @@ -286,7 +321,7 @@ def _read_write_loop(self, client_sock, server_sock, chunks=65536): read_socket = server_sock write_socket = client_sock - # Ensure buffer is not full before writting + # Ensure buffer is not full before writing if write_socket in writable: try: b = read_socket.recv(chunks) @@ -302,7 +337,6 @@ def _read_write_loop(self, client_sock, server_sock, chunks=65536): return -@pytest.mark.skipif(sys.version_info < (3, 5), reason="requires python3.5 or higher") class TlsInTlsTestCase(SocketDummyServerTestCase): """ Creates a TLS in TLS tunnel by chaining a 'SocketProxyDummyServer' and a @@ -314,41 +348,44 @@ class TlsInTlsTestCase(SocketDummyServerTestCase): """ @classmethod - def setup_class(cls): + def setup_class(cls) -> None: cls.server_context, cls.client_context = server_client_ssl_contexts() @classmethod - def start_proxy_server(cls): + def start_proxy_server(cls) -> None: # Proxy server will handle the first TLS connection and create a # connection to the destination server. cls.proxy_server = SocketProxyDummyServer(cls.host, cls.port) cls.proxy_server.start_proxy_handler() @classmethod - def teardown_class(cls): + def teardown_class(cls) -> None: if hasattr(cls, "proxy_server"): cls.proxy_server.teardown_class() - super(TlsInTlsTestCase, cls).teardown_class() + super().teardown_class() @classmethod - def start_destination_server(cls): + def start_destination_server(cls) -> None: """ Socket handler for the destination_server. Terminates the second TLS layer and send a basic HTTP response. """ - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] - with cls.server_context.wrap_socket(sock, server_side=True) as ssock: - request = consume_socket(ssock) - validate_request(request) - ssock.send(sample_response()) + try: + with cls.server_context.wrap_socket(sock, server_side=True) as ssock: + request = consume_socket(ssock) + validate_request(request) + ssock.send(sample_response()) + except (ssl.SSLEOFError, ssl.SSLZeroReturnError, OSError): + return sock.close() cls._start_server(socket_handler) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_tls_in_tls_tunnel(self): + def test_tls_in_tls_tunnel(self) -> None: """ Basic communication over the TLS in TLS tunnel. """ @@ -370,7 +407,7 @@ def test_tls_in_tls_tunnel(self): validate_response(response) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_wrong_sni_hint(self): + def test_wrong_sni_hint(self) -> None: """ Provides a wrong sni hint to validate an exception is thrown. """ @@ -383,17 +420,14 @@ def test_wrong_sni_hint(self): with self.client_context.wrap_socket( sock, server_hostname="localhost" ) as proxy_sock: - with pytest.raises(Exception) as e: + with pytest.raises(ssl.SSLCertVerificationError): SSLTransport( proxy_sock, self.client_context, server_hostname="veryverywrong" ) - # ssl.CertificateError is a child of ValueError in python3.6 or - # before. After python3.7 it's a child of SSLError - assert e.type in [ssl.SSLError, ssl.CertificateError] @pytest.mark.timeout(PER_TEST_TIMEOUT) @pytest.mark.parametrize("buffering", [None, 0]) - def test_tls_in_tls_makefile_raw_rw_binary(self, buffering): + def test_tls_in_tls_makefile_raw_rw_binary(self, buffering: int | None) -> None: """ Uses makefile with read, write and binary modes without buffering. """ @@ -409,13 +443,12 @@ def test_tls_in_tls_makefile_raw_rw_binary(self, buffering): with SSLTransport( proxy_sock, self.client_context, server_hostname="localhost" ) as destination_sock: - file = destination_sock.makefile("rwb", buffering) - file.write(sample_request()) + file.write(sample_request()) # type: ignore[arg-type] file.flush() response = bytearray(65536) - wrote = file.readinto(response) + wrote = file.readinto(response) # type: ignore[union-attr] assert wrote is not None # Allocated response is bigger than the actual response, we # rtrim remaining x00 bytes. @@ -428,7 +461,7 @@ def test_tls_in_tls_makefile_raw_rw_binary(self, buffering): reason="Skipping windows due to text makefile support", ) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_tls_in_tls_makefile_rw_text(self): + def test_tls_in_tls_makefile_rw_text(self) -> None: """ Creates a separate buffer for reading and writing using text mode and utf-8 encoding. @@ -445,22 +478,23 @@ def test_tls_in_tls_makefile_rw_text(self): with SSLTransport( proxy_sock, self.client_context, server_hostname="localhost" ) as destination_sock: - read = destination_sock.makefile("r", encoding="utf-8") write = destination_sock.makefile("w", encoding="utf-8") - write.write(sample_request(binary=False)) + write.write(sample_request(binary=False)) # type: ignore[arg-type, call-overload] write.flush() response = read.read() + assert type(response) is str if "\r" not in response: # Carriage return will be removed when reading as a file on # some platforms. We add it before the comparison. + assert type(response) is str response = response.replace("\n", "\r\n") validate_response(response, binary=False) @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_tls_in_tls_recv_into_sendall(self): + def test_tls_in_tls_recv_into_sendall(self) -> None: """ Valides recv_into and sendall also work as expected. Other tests are using recv/send. @@ -477,46 +511,63 @@ def test_tls_in_tls_recv_into_sendall(self): with SSLTransport( proxy_sock, self.client_context, server_hostname="localhost" ) as destination_sock: - destination_sock.sendall(sample_request()) response = bytearray(65536) destination_sock.recv_into(response) str_response = response.decode("utf-8").rstrip("\x00") validate_response(str_response, binary=False) - @pytest.mark.timeout(PER_TEST_TIMEOUT) - def test_tls_in_tls_recv_into_unbuffered(self): - """ - Valides recv_into without a preallocated buffer. - """ - self.start_destination_server() - self.start_proxy_server() - sock = socket.create_connection( - (self.proxy_server.host, self.proxy_server.port) +class TestSSLTransportWithMock: + def test_constructor_params(self) -> None: + server_hostname = "example-domain.com" + sock = mock.Mock() + context = mock.create_autospec(ssl_.SSLContext) + ssl_transport = SSLTransport( + sock, context, server_hostname=server_hostname, suppress_ragged_eofs=False ) - with self.client_context.wrap_socket( - sock, server_hostname="localhost" - ) as proxy_sock: - with SSLTransport( - proxy_sock, self.client_context, server_hostname="localhost" - ) as destination_sock: + context.wrap_bio.assert_called_with( + mock.ANY, mock.ANY, server_hostname=server_hostname + ) + assert not ssl_transport.suppress_ragged_eofs - destination_sock.send(sample_request()) - response = destination_sock.recv_into(None) - validate_response(response) + def test_various_flags_errors(self) -> None: + server_hostname = "example-domain.com" + sock = mock.Mock() + context = mock.create_autospec(ssl_.SSLContext) + ssl_transport = SSLTransport( + sock, context, server_hostname=server_hostname, suppress_ragged_eofs=False + ) + with pytest.raises(ValueError): + ssl_transport.recv(flags=1) + + with pytest.raises(ValueError): + ssl_transport.recv_into(bytearray(), flags=1) + with pytest.raises(ValueError): + ssl_transport.sendall(bytearray(), flags=1) -@pytest.mark.skipif(sys.version_info < (3, 5), reason="requires python3.5 or higher") -class TestSSLTransportWithMock(object): - def test_constructor_params(self): + with pytest.raises(ValueError): + ssl_transport.send(None, flags=1) # type: ignore[arg-type] + + def test_makefile_wrong_mode_error(self) -> None: server_hostname = "example-domain.com" sock = mock.Mock() context = mock.create_autospec(ssl_.SSLContext) ssl_transport = SSLTransport( sock, context, server_hostname=server_hostname, suppress_ragged_eofs=False ) - context.wrap_bio.assert_called_with( - mock.ANY, mock.ANY, server_hostname=server_hostname + with pytest.raises(ValueError): + ssl_transport.makefile(mode="x") + + def test_wrap_ssl_read_error(self) -> None: + server_hostname = "example-domain.com" + sock = mock.Mock() + context = mock.create_autospec(ssl_.SSLContext) + ssl_transport = SSLTransport( + sock, context, server_hostname=server_hostname, suppress_ragged_eofs=False ) - assert not ssl_transport.suppress_ragged_eofs + with mock.patch.object(ssl_transport, "_ssl_io_loop") as _ssl_io_loop: + _ssl_io_loop.side_effect = ssl.SSLError() + with pytest.raises(ssl.SSLError): + ssl_transport._wrap_ssl_read(1) diff --git a/test/test_util.py b/test/test_util.py index 2f16dbf..49c1b27 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,39 +1,43 @@ -# coding: utf-8 -import hashlib +from __future__ import annotations + import io import logging import socket import ssl +import sys +import typing import warnings from itertools import chain -from test import notBrotlipy, onlyBrotlipy, onlyPy2, onlyPy3 +from test import ImportBlocker, ModuleStash, notBrotli, notZstd, onlyBrotli, onlyZstd +from unittest import mock +from unittest.mock import MagicMock, Mock, patch +from urllib.parse import urlparse import pytest -from mock import Mock, patch -from urllib3 import add_stderr_logger, disable_warnings, util +from urllib3 import add_stderr_logger, disable_warnings +from urllib3.connection import ProxyConfig from urllib3.exceptions import ( InsecureRequestWarning, LocationParseError, - SNIMissingWarning, TimeoutStateError, UnrewindableBodyError, ) -from urllib3.packages import six -from urllib3.poolmanager import ProxyConfig from urllib3.util import is_fp_closed from urllib3.util.connection import _has_ipv6, allowed_gai_family, create_connection -from urllib3.util.proxy import connection_requires_http_tunnel, create_proxy_ssl_context +from urllib3.util.proxy import connection_requires_http_tunnel from urllib3.util.request import _FAILEDTELL, make_headers, rewind_body from urllib3.util.response import assert_header_parsing from urllib3.util.ssl_ import ( - _const_compare_digest_backport, + _TYPE_VERSION_INFO, + _is_has_never_check_common_name_reliable, resolve_cert_reqs, resolve_ssl_version, ssl_wrap_socket, ) -from urllib3.util.timeout import Timeout -from urllib3.util.url import Url, get_host, parse_url, split_first +from urllib3.util.timeout import _DEFAULT_TIMEOUT, Timeout +from urllib3.util.url import Url, _encode_invalid_chars, parse_url +from urllib3.util.util import to_bytes, to_str from . import clear_warnings @@ -43,8 +47,7 @@ TIMEOUT_EPOCH = 1000 -class TestUtil(object): - +class TestUtil: url_host_map = [ # Hosts ("http://google.com/mail", ("http", "google.com", None)), @@ -135,38 +138,38 @@ class TestUtil(object): ), ] - @pytest.mark.parametrize("url, expected_host", url_host_map) - def test_get_host(self, url, expected_host): - returned_host = get_host(url) - assert returned_host == expected_host + @pytest.mark.parametrize(["url", "scheme_host_port"], url_host_map) + def test_scheme_host_port( + self, url: str, scheme_host_port: tuple[str, str, int | None] + ) -> None: + parsed_url = parse_url(url) + scheme, host, port = scheme_host_port + + assert (parsed_url.scheme or "http") == scheme + assert parsed_url.hostname == parsed_url.host == host + assert parsed_url.port == port + + def test_encode_invalid_chars_none(self) -> None: + assert _encode_invalid_chars(None, set()) is None - # TODO: Add more tests @pytest.mark.parametrize( - "location", + "url", [ "http://google.com:foo", "http://::1/", "http://::1:80/", "http://google.com:-80", - six.u("http://google.com:\xb2\xb2"), # \xb2 = ^2 - ], - ) - def test_invalid_host(self, location): - with pytest.raises(LocationParseError): - get_host(location) - - @pytest.mark.parametrize( - "url", - [ + "http://google.com:65536", + "http://google.com:\xb2\xb2", # \xb2 = ^2 # Invalid IDNA labels - u"http://\uD7FF.com", - u"http://❤️", + "http://\uD7FF.com", + "http://❤️", # Unicode surrogates - u"http://\uD800.com", - u"http://\uDC00.com", + "http://\uD800.com", + "http://\uDC00.com", ], ) - def test_invalid_url(self, url): + def test_invalid_url(self, url: str) -> None: with pytest.raises(LocationParseError): parse_url(url) @@ -207,16 +210,18 @@ def test_invalid_url(self, url): ), ], ) - def test_parse_url_normalization(self, url, expected_normalized_url): + def test_parse_url_normalization( + self, url: str, expected_normalized_url: str + ) -> None: """Assert parse_url normalizes the scheme/host, and only the scheme/host""" actual_normalized_url = parse_url(url).url assert actual_normalized_url == expected_normalized_url @pytest.mark.parametrize("char", [chr(i) for i in range(0x00, 0x21)] + ["\x7F"]) - def test_control_characters_are_percent_encoded(self, char): + def test_control_characters_are_percent_encoded(self, char: str) -> None: percent_char = "%" + (hex(ord(char))[2:].zfill(2).upper()) url = parse_url( - "http://user{0}@example.com/path{0}?query{0}#fragment{0}".format(char) + f"http://user{char}@example.com/path{char}?query{char}#fragment{char}" ) assert url == Url( @@ -264,15 +269,6 @@ def test_control_characters_are_percent_encoded(self, char): "http://foo:bar@localhost/", Url("http", auth="foo:bar", host="localhost", path="/"), ), - # Unicode type (Python 2.x) - ( - u"http://foo:bar@localhost/", - Url(u"http", auth=u"foo:bar", host=u"localhost", path=u"/"), - ), - ( - "http://foo:bar@localhost/", - Url("http", auth="foo:bar", host="localhost", path="/"), - ), ] non_round_tripping_parse_url_host_map = [ @@ -286,26 +282,26 @@ def test_control_characters_are_percent_encoded(self, char): ("http://google.com:/", Url("http", host="google.com", path="/")), # Uppercase IRI ( - u"http://Königsgäßchen.de/straße", + "http://Königsgäßchen.de/straße", Url("http", host="xn--knigsgchen-b4a3dun.de", path="/stra%C3%9Fe"), ), # Percent-encode in userinfo ( - u"http://user@email.com:password@example.com/", + "http://user@email.com:password@example.com/", Url("http", auth="user%40email.com:password", host="example.com", path="/"), ), ( - u'http://user":quoted@example.com/', + 'http://user":quoted@example.com/', Url("http", auth="user%22:quoted", host="example.com", path="/"), ), # Unicode Surrogates - (u"http://google.com/\uD800", Url("http", host="google.com", path="%ED%A0%80")), + ("http://google.com/\uD800", Url("http", host="google.com", path="%ED%A0%80")), ( - u"http://google.com?q=\uDC00", + "http://google.com?q=\uDC00", Url("http", host="google.com", path="", query="q=%ED%B0%80"), ), ( - u"http://google.com#\uDC00", + "http://google.com#\uDC00", Url("http", host="google.com", path="", fragment="%ED%B0%80"), ), ] @@ -314,12 +310,13 @@ def test_control_characters_are_percent_encoded(self, char): "url, expected_url", chain(parse_url_host_map, non_round_tripping_parse_url_host_map), ) - def test_parse_url(self, url, expected_url): + def test_parse_url(self, url: str, expected_url: Url) -> None: returned_url = parse_url(url) assert returned_url == expected_url + assert returned_url.hostname == returned_url.host == expected_url.host @pytest.mark.parametrize("url, expected_url", parse_url_host_map) - def test_unparse_url(self, url, expected_url): + def test_unparse_url(self, url: str, expected_url: Url) -> None: assert url == expected_url.url @pytest.mark.parametrize( @@ -334,31 +331,31 @@ def test_unparse_url(self, url, expected_url): ("/abc/./.././d/././e/.././f/./../../ghi", Url(path="/ghi")), ], ) - def test_parse_and_normalize_url_paths(self, url, expected_url): + def test_parse_and_normalize_url_paths(self, url: str, expected_url: Url) -> None: actual_url = parse_url(url) assert actual_url == expected_url assert actual_url.url == expected_url.url - def test_parse_url_invalid_IPv6(self): + def test_parse_url_invalid_IPv6(self) -> None: with pytest.raises(LocationParseError): parse_url("[::1") - def test_parse_url_negative_port(self): + def test_parse_url_negative_port(self) -> None: with pytest.raises(LocationParseError): parse_url("https://www.google.com:-80/") - def test_parse_url_remove_leading_zeros(self): + def test_parse_url_remove_leading_zeros(self) -> None: url = parse_url("https://example.com:0000000000080") assert url.port == 80 - def test_parse_url_only_zeros(self): + def test_parse_url_only_zeros(self) -> None: url = parse_url("https://example.com:0") assert url.port == 0 url = parse_url("https://example.com:000000000000") assert url.port == 0 - def test_Url_str(self): + def test_Url_str(self) -> None: U = Url("http", host="google.com") assert str(U) == U.url @@ -375,19 +372,60 @@ def test_Url_str(self): ] @pytest.mark.parametrize("url, expected_request_uri", request_uri_map) - def test_request_uri(self, url, expected_request_uri): + def test_request_uri(self, url: str, expected_request_uri: str) -> None: returned_url = parse_url(url) assert returned_url.request_uri == expected_request_uri + url_authority_map: list[tuple[str, str | None]] = [ + ("http://user:pass@google.com/mail", "user:pass@google.com"), + ("http://user:pass@google.com:80/mail", "user:pass@google.com:80"), + ("http://user@google.com:80/mail", "user@google.com:80"), + ("http://user:pass@192.168.1.1/path", "user:pass@192.168.1.1"), + ("http://user:pass@192.168.1.1:80/path", "user:pass@192.168.1.1:80"), + ("http://user@192.168.1.1:80/path", "user@192.168.1.1:80"), + ("http://user:pass@[::1]/path", "user:pass@[::1]"), + ("http://user:pass@[::1]:80/path", "user:pass@[::1]:80"), + ("http://user@[::1]:80/path", "user@[::1]:80"), + ("http://user:pass@localhost/path", "user:pass@localhost"), + ("http://user:pass@localhost:80/path", "user:pass@localhost:80"), + ("http://user@localhost:80/path", "user@localhost:80"), + ] + url_netloc_map = [ ("http://google.com/mail", "google.com"), ("http://google.com:80/mail", "google.com:80"), + ("http://192.168.0.1/path", "192.168.0.1"), + ("http://192.168.0.1:80/path", "192.168.0.1:80"), + ("http://[::1]/path", "[::1]"), + ("http://[::1]:80/path", "[::1]:80"), + ("http://localhost", "localhost"), + ("http://localhost:80", "localhost:80"), ("google.com/foobar", "google.com"), ("google.com:12345", "google.com:12345"), + ("/", None), + ] + + combined_netloc_authority_map = url_authority_map + url_netloc_map + + # We compose this list due to variances between parse_url + # and urlparse when URIs don't provide a scheme. + url_authority_with_schemes_map = [ + u for u in combined_netloc_authority_map if u[0].startswith("http") ] + @pytest.mark.parametrize("url, expected_authority", combined_netloc_authority_map) + def test_authority(self, url: str, expected_authority: str | None) -> None: + assert parse_url(url).authority == expected_authority + + @pytest.mark.parametrize("url, expected_authority", url_authority_with_schemes_map) + def test_authority_matches_urllib_netloc( + self, url: str, expected_authority: str | None + ) -> None: + """Validate this matches the behavior of urlparse().netloc""" + assert urlparse(url).netloc == expected_authority + @pytest.mark.parametrize("url, expected_netloc", url_netloc_map) - def test_netloc(self, url, expected_netloc): + def test_netloc(self, url: str, expected_netloc: str | None) -> None: assert parse_url(url).netloc == expected_netloc url_vulnerabilities = [ @@ -403,7 +441,7 @@ def test_netloc(self, url, expected_netloc): ), # NodeJS unicode -> double dot ( - u"http://google.com/\uff2e\uff2e/abc", + "http://google.com/\uff2e\uff2e/abc", Url("http", host="google.com", path="/%EF%BC%AE%EF%BC%AE/abc"), ), # Scheme without :// @@ -414,14 +452,14 @@ def test_netloc(self, url, expected_netloc): ("//google.com/a/b/c", Url(host="google.com", path="/a/b/c")), # International URLs ( - u"http://ヒ:キ@ヒ.abc.ニ/ヒ?キ#ワ", + "http://ヒ:キ@ヒ.abc.ニ/ヒ?キ#ワ", Url( - u"http", - host=u"xn--pdk.abc.xn--idk", - auth=u"%E3%83%92:%E3%82%AD", - path=u"/%E3%83%92", - query=u"%E3%82%AD", - fragment=u"%E3%83%AF", + "http", + host="xn--pdk.abc.xn--idk", + auth="%E3%83%92:%E3%82%AD", + path="/%E3%83%92", + query="%E3%82%AD", + fragment="%E3%83%AF", ), ), # Injected headers (CVE-2016-5699, CVE-2019-9740, CVE-2019-9947) @@ -457,72 +495,80 @@ def test_netloc(self, url, expected_netloc): ), ), # Tons of '@' causing backtracking - ("https://" + ("@" * 10000) + "[", False), - ( + pytest.param( + "https://" + ("@" * 10000) + "[", + False, + id="Tons of '@' causing backtracking 1", + ), + pytest.param( "https://user:" + ("@" * 10000) + "example.com", Url( scheme="https", auth="user:" + ("%40" * 9999), host="example.com", ), + id="Tons of '@' causing backtracking 2", ), ] @pytest.mark.parametrize("url, expected_url", url_vulnerabilities) - def test_url_vulnerabilities(self, url, expected_url): + def test_url_vulnerabilities( + self, url: str, expected_url: typing.Literal[False] | Url + ) -> None: if expected_url is False: with pytest.raises(LocationParseError): parse_url(url) else: assert parse_url(url) == expected_url - @onlyPy2 - def test_parse_url_bytes_to_str_python_2(self): - url = parse_url(b"https://www.google.com/") - assert url == Url("https", host="www.google.com", path="/") - - assert isinstance(url.scheme, str) - assert isinstance(url.host, str) - assert isinstance(url.path, str) - - @onlyPy2 - def test_parse_url_unicode_python_2(self): - url = parse_url(u"https://www.google.com/") - assert url == Url(u"https", host=u"www.google.com", path=u"/") - - assert isinstance(url.scheme, six.text_type) - assert isinstance(url.host, six.text_type) - assert isinstance(url.path, six.text_type) - - @onlyPy3 - def test_parse_url_bytes_type_error_python_3(self): + def test_parse_url_bytes_type_error(self) -> None: with pytest.raises(TypeError): - parse_url(b"https://www.google.com/") + parse_url(b"https://www.google.com/") # type: ignore[arg-type] @pytest.mark.parametrize( "kwargs, expected", [ + pytest.param( + {"accept_encoding": True}, + {"accept-encoding": "gzip,deflate,br,zstd"}, + marks=[onlyBrotli(), onlyZstd()], # type: ignore[list-item] + ), pytest.param( {"accept_encoding": True}, {"accept-encoding": "gzip,deflate,br"}, - marks=onlyBrotlipy(), + marks=[onlyBrotli(), notZstd()], # type: ignore[list-item] + ), + pytest.param( + {"accept_encoding": True}, + {"accept-encoding": "gzip,deflate,zstd"}, + marks=[notBrotli(), onlyZstd()], # type: ignore[list-item] ), pytest.param( {"accept_encoding": True}, {"accept-encoding": "gzip,deflate"}, - marks=notBrotlipy(), + marks=[notBrotli(), notZstd()], # type: ignore[list-item] ), ({"accept_encoding": "foo,bar"}, {"accept-encoding": "foo,bar"}), ({"accept_encoding": ["foo", "bar"]}, {"accept-encoding": "foo,bar"}), + pytest.param( + {"accept_encoding": True, "user_agent": "banana"}, + {"accept-encoding": "gzip,deflate,br,zstd", "user-agent": "banana"}, + marks=[onlyBrotli(), onlyZstd()], # type: ignore[list-item] + ), pytest.param( {"accept_encoding": True, "user_agent": "banana"}, {"accept-encoding": "gzip,deflate,br", "user-agent": "banana"}, - marks=onlyBrotlipy(), + marks=[onlyBrotli(), notZstd()], # type: ignore[list-item] + ), + pytest.param( + {"accept_encoding": True, "user_agent": "banana"}, + {"accept-encoding": "gzip,deflate,zstd", "user-agent": "banana"}, + marks=[notBrotli(), onlyZstd()], # type: ignore[list-item] ), pytest.param( {"accept_encoding": True, "user_agent": "banana"}, {"accept-encoding": "gzip,deflate", "user-agent": "banana"}, - marks=notBrotlipy(), + marks=[notBrotli(), notZstd()], # type: ignore[list-item] ), ({"user_agent": "banana"}, {"user-agent": "banana"}), ({"keep_alive": True}, {"connection": "keep-alive"}), @@ -534,10 +580,12 @@ def test_parse_url_bytes_type_error_python_3(self): ({"disable_cache": True}, {"cache-control": "no-cache"}), ], ) - def test_make_headers(self, kwargs, expected): - assert make_headers(**kwargs) == expected + def test_make_headers( + self, kwargs: dict[str, bool | str], expected: dict[str, str] + ) -> None: + assert make_headers(**kwargs) == expected # type: ignore[arg-type] - def test_rewind_body(self): + def test_rewind_body(self) -> None: body = io.BytesIO(b"test data") assert body.read() == b"test data" @@ -548,7 +596,7 @@ def test_rewind_body(self): rewind_body(body, 5) assert body.read() == b"data" - def test_rewind_body_failed_tell(self): + def test_rewind_body_failed_tell(self) -> None: body = io.BytesIO(b"test data") body.read() # Consume body @@ -557,40 +605,25 @@ def test_rewind_body_failed_tell(self): with pytest.raises(UnrewindableBodyError): rewind_body(body, body_pos) - def test_rewind_body_bad_position(self): + def test_rewind_body_bad_position(self) -> None: body = io.BytesIO(b"test data") body.read() # Consume body # Pass non-integer position with pytest.raises(ValueError): - rewind_body(body, body_pos=None) + rewind_body(body, body_pos=None) # type: ignore[arg-type] with pytest.raises(ValueError): - rewind_body(body, body_pos=object()) + rewind_body(body, body_pos=object()) # type: ignore[arg-type] - def test_rewind_body_failed_seek(self): - class BadSeek: - def seek(self, pos, offset=0): - raise IOError + def test_rewind_body_failed_seek(self) -> None: + class BadSeek(io.StringIO): + def seek(self, offset: int, whence: int = 0) -> typing.NoReturn: + raise OSError with pytest.raises(UnrewindableBodyError): rewind_body(BadSeek(), body_pos=2) - @pytest.mark.parametrize( - "input, expected", - [ - (("abcd", "b"), ("a", "cd", "b")), - (("abcd", "cb"), ("a", "cd", "b")), - (("abcd", ""), ("abcd", "", None)), - (("abcd", "a"), ("", "bcd", "a")), - (("abcd", "ab"), ("", "bcd", "a")), - (("abcd", "eb"), ("a", "cd", "b")), - ], - ) - def test_split_first(self, input, expected): - output = split_first(*input) - assert output == expected - - def test_add_stderr_logger(self): + def test_add_stderr_logger(self) -> None: handler = add_stderr_logger(level=logging.INFO) # Don't actually print debug logger = logging.getLogger("urllib3") assert handler in logger.handlers @@ -598,16 +631,19 @@ def test_add_stderr_logger(self): logger.debug("Testing add_stderr_logger") logger.removeHandler(handler) - def test_disable_warnings(self): + def test_disable_warnings(self) -> None: with warnings.catch_warnings(record=True) as w: clear_warnings() + warnings.simplefilter("default", InsecureRequestWarning) warnings.warn("This is a test.", InsecureRequestWarning) assert len(w) == 1 disable_warnings() warnings.warn("This is a test.", InsecureRequestWarning) assert len(w) == 1 - def _make_time_pass(self, seconds, timeout, time_mock): + def _make_time_pass( + self, seconds: int, timeout: Timeout, time_mock: Mock + ) -> Timeout: """Make some time pass for the timeout object""" time_mock.return_value = TIMEOUT_EPOCH timeout.start_connect() @@ -624,20 +660,22 @@ def _make_time_pass(self, seconds, timeout, time_mock): ({"read": True}, "cannot be a boolean"), ({"connect": 0}, "less than or equal"), ({"read": "foo"}, "int, float or None"), + ({"read": "1.0"}, "int, float or None"), ], ) - def test_invalid_timeouts(self, kwargs, message): - with pytest.raises(ValueError) as e: + def test_invalid_timeouts( + self, kwargs: dict[str, int | bool], message: str + ) -> None: + with pytest.raises(ValueError, match=message): Timeout(**kwargs) - assert message in str(e.value) - @patch("urllib3.util.timeout.current_time") - def test_timeout(self, current_time): + @patch("time.monotonic") + def test_timeout(self, time_monotonic: MagicMock) -> None: timeout = Timeout(total=3) # make 'no time' elapse timeout = self._make_time_pass( - seconds=0, timeout=timeout, time_mock=current_time + seconds=0, timeout=timeout, time_mock=time_monotonic ) assert timeout.read_timeout == 3 assert timeout.connect_timeout == 3 @@ -646,19 +684,19 @@ def test_timeout(self, current_time): assert timeout.connect_timeout == 2 timeout = Timeout() - assert timeout.connect_timeout == Timeout.DEFAULT_TIMEOUT + assert timeout.connect_timeout == _DEFAULT_TIMEOUT # Connect takes 5 seconds, leaving 5 seconds for read timeout = Timeout(total=10, read=7) timeout = self._make_time_pass( - seconds=5, timeout=timeout, time_mock=current_time + seconds=5, timeout=timeout, time_mock=time_monotonic ) assert timeout.read_timeout == 5 # Connect takes 2 seconds, read timeout still 7 seconds timeout = Timeout(total=10, read=7) timeout = self._make_time_pass( - seconds=2, timeout=timeout, time_mock=current_time + seconds=2, timeout=timeout, time_mock=time_monotonic ) assert timeout.read_timeout == 7 @@ -673,15 +711,24 @@ def test_timeout(self, current_time): timeout = Timeout(5) assert timeout.total == 5 - def test_timeout_str(self): + def test_timeout_default_resolve(self) -> None: + """The timeout default is resolved when read_timeout is accessed.""" + timeout = Timeout() + with patch("urllib3.util.timeout.getdefaulttimeout", return_value=2): + assert timeout.read_timeout == 2 + + with patch("urllib3.util.timeout.getdefaulttimeout", return_value=3): + assert timeout.read_timeout == 3 + + def test_timeout_str(self) -> None: timeout = Timeout(connect=1, read=2, total=3) assert str(timeout) == "Timeout(connect=1, read=2, total=3)" timeout = Timeout(connect=1, read=None, total=3) assert str(timeout) == "Timeout(connect=1, read=None, total=3)" - @patch("urllib3.util.timeout.current_time") - def test_timeout_elapsed(self, current_time): - current_time.return_value = TIMEOUT_EPOCH + @patch("time.monotonic") + def test_timeout_elapsed(self, time_monotonic: MagicMock) -> None: + time_monotonic.return_value = TIMEOUT_EPOCH timeout = Timeout(total=3) with pytest.raises(TimeoutStateError): timeout.get_connect_duration() @@ -690,101 +737,88 @@ def test_timeout_elapsed(self, current_time): with pytest.raises(TimeoutStateError): timeout.start_connect() - current_time.return_value = TIMEOUT_EPOCH + 2 + time_monotonic.return_value = TIMEOUT_EPOCH + 2 assert timeout.get_connect_duration() == 2 - current_time.return_value = TIMEOUT_EPOCH + 37 + time_monotonic.return_value = TIMEOUT_EPOCH + 37 assert timeout.get_connect_duration() == 37 - def test_is_fp_closed_object_supports_closed(self): - class ClosedFile(object): + def test_is_fp_closed_object_supports_closed(self) -> None: + class ClosedFile: @property - def closed(self): + def closed(self) -> typing.Literal[True]: return True assert is_fp_closed(ClosedFile()) - def test_is_fp_closed_object_has_none_fp(self): - class NoneFpFile(object): + def test_is_fp_closed_object_has_none_fp(self) -> None: + class NoneFpFile: @property - def fp(self): + def fp(self) -> None: return None assert is_fp_closed(NoneFpFile()) - def test_is_fp_closed_object_has_fp(self): - class FpFile(object): + def test_is_fp_closed_object_has_fp(self) -> None: + class FpFile: @property - def fp(self): + def fp(self) -> typing.Literal[True]: return True assert not is_fp_closed(FpFile()) - def test_is_fp_closed_object_has_neither_fp_nor_closed(self): - class NotReallyAFile(object): + def test_is_fp_closed_object_has_neither_fp_nor_closed(self) -> None: + class NotReallyAFile: pass with pytest.raises(ValueError): is_fp_closed(NotReallyAFile()) - def test_const_compare_digest_fallback(self): - target = hashlib.sha256(b"abcdef").digest() - assert _const_compare_digest_backport(target, target) - - prefix = target[:-1] - assert not _const_compare_digest_backport(target, prefix) - - suffix = target + b"0" - assert not _const_compare_digest_backport(target, suffix) - - incorrect = hashlib.sha256(b"xyz").digest() - assert not _const_compare_digest_backport(target, incorrect) - - def test_has_ipv6_disabled_on_compile(self): + def test_has_ipv6_disabled_on_compile(self) -> None: with patch("socket.has_ipv6", False): assert not _has_ipv6("::1") - def test_has_ipv6_enabled_but_fails(self): + def test_has_ipv6_enabled_but_fails(self) -> None: with patch("socket.has_ipv6", True): with patch("socket.socket") as mock: instance = mock.return_value instance.bind = Mock(side_effect=Exception("No IPv6 here!")) assert not _has_ipv6("::1") - def test_has_ipv6_enabled_and_working(self): + def test_has_ipv6_enabled_and_working(self) -> None: with patch("socket.has_ipv6", True): with patch("socket.socket") as mock: instance = mock.return_value instance.bind.return_value = True assert _has_ipv6("::1") - def test_has_ipv6_disabled_on_appengine(self): - gae_patch = patch( - "urllib3.contrib._appengine_environ.is_appengine_sandbox", return_value=True - ) - with gae_patch: - assert not _has_ipv6("::1") - - def test_ip_family_ipv6_enabled(self): + def test_ip_family_ipv6_enabled(self) -> None: with patch("urllib3.util.connection.HAS_IPV6", True): assert allowed_gai_family() == socket.AF_UNSPEC - def test_ip_family_ipv6_disabled(self): + def test_ip_family_ipv6_disabled(self) -> None: with patch("urllib3.util.connection.HAS_IPV6", False): assert allowed_gai_family() == socket.AF_INET @pytest.mark.parametrize("headers", [b"foo", None, object]) - def test_assert_header_parsing_throws_typeerror_with_non_headers(self, headers): + def test_assert_header_parsing_throws_typeerror_with_non_headers( + self, headers: bytes | object | None + ) -> None: with pytest.raises(TypeError): - assert_header_parsing(headers) + assert_header_parsing(headers) # type: ignore[arg-type] - def test_connection_requires_http_tunnel_no_proxy(self): + def test_connection_requires_http_tunnel_no_proxy(self) -> None: assert not connection_requires_http_tunnel( proxy_url=None, proxy_config=None, destination_scheme=None ) - def test_connection_requires_http_tunnel_http_proxy(self): + def test_connection_requires_http_tunnel_http_proxy(self) -> None: proxy = parse_url("http://proxy:8080") - proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False) + proxy_config = ProxyConfig( + ssl_context=None, + use_forwarding_for_https=False, + assert_hostname=None, + assert_fingerprint=None, + ) destination_scheme = "http" assert not connection_requires_http_tunnel( proxy, proxy_config, destination_scheme @@ -793,20 +827,20 @@ def test_connection_requires_http_tunnel_http_proxy(self): destination_scheme = "https" assert connection_requires_http_tunnel(proxy, proxy_config, destination_scheme) - def test_connection_requires_http_tunnel_https_proxy(self): + def test_connection_requires_http_tunnel_https_proxy(self) -> None: proxy = parse_url("https://proxy:8443") - proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False) + proxy_config = ProxyConfig( + ssl_context=None, + use_forwarding_for_https=False, + assert_hostname=None, + assert_fingerprint=None, + ) destination_scheme = "http" assert not connection_requires_http_tunnel( proxy, proxy_config, destination_scheme ) - def test_create_proxy_ssl_context(self): - ssl_context = create_proxy_ssl_context(ssl_version=None, cert_reqs=None) - ssl_context.verify_mode = ssl.CERT_REQUIRED - - @onlyPy3 - def test_assert_header_parsing_no_error_on_multipart(self): + def test_assert_header_parsing_no_error_on_multipart(self) -> None: from http import client header_msg = io.BytesIO() @@ -820,10 +854,12 @@ def test_assert_header_parsing_no_error_on_multipart(self): assert_header_parsing(client.parse_headers(header_msg)) @pytest.mark.parametrize("host", [".localhost", "...", "t" * 64]) - def test_create_connection_with_invalid_idna_labels(self, host): - with pytest.raises(LocationParseError) as ctx: + def test_create_connection_with_invalid_idna_labels(self, host: str) -> None: + with pytest.raises( + LocationParseError, + match=f"Failed to parse: '{host}', label empty or too long", + ): create_connection((host, 80)) - assert str(ctx.value) == "Failed to parse: '%s', label empty or too long" % host @pytest.mark.parametrize( "host", @@ -837,14 +873,38 @@ def test_create_connection_with_invalid_idna_labels(self, host): ) @patch("socket.getaddrinfo") @patch("socket.socket") - def test_create_connection_with_valid_idna_labels(self, socket, getaddrinfo, host): + def test_create_connection_with_valid_idna_labels( + self, socket: MagicMock, getaddrinfo: MagicMock, host: str + ) -> None: getaddrinfo.return_value = [(None, None, None, None, None)] socket.return_value = Mock() create_connection((host, 80)) + @patch("socket.getaddrinfo") + def test_create_connection_error(self, getaddrinfo: MagicMock) -> None: + getaddrinfo.return_value = [] + with pytest.raises(OSError, match="getaddrinfo returns an empty list"): + create_connection(("example.com", 80)) + + @patch("socket.getaddrinfo") + def test_dnsresolver_forced_error(self, getaddrinfo: MagicMock) -> None: + getaddrinfo.side_effect = socket.gaierror() + with pytest.raises(socket.gaierror): + # dns is valid but we force the error just for the sake of the test + create_connection(("example.com", 80)) + + def test_dnsresolver_expected_error(self) -> None: + with pytest.raises(socket.gaierror): + # windows: [Errno 11001] getaddrinfo failed in windows + # linux: [Errno -2] Name or service not known + # macos: [Errno 8] nodename nor servname provided, or not known + create_connection(("badhost.invalid", 80)) + @patch("socket.getaddrinfo") @patch("socket.socket") - def test_create_connection_with_scoped_ipv6(self, socket, getaddrinfo): + def test_create_connection_with_scoped_ipv6( + self, socket: MagicMock, getaddrinfo: MagicMock + ) -> None: # Check that providing create_connection with a scoped IPv6 address # properly propagates the scope to getaddrinfo, and that the returned # scoped ID makes it to the socket creation call. @@ -858,14 +918,51 @@ def test_create_connection_with_scoped_ipv6(self, socket, getaddrinfo): fake_scoped_sa6, ) ] - socket.return_value = fake_sock = Mock() + socket.return_value = fake_sock = MagicMock() create_connection(("a::b%iface", 80)) assert getaddrinfo.call_args[0][0] == "a::b%iface" fake_sock.connect.assert_called_once_with(fake_scoped_sa6) + @pytest.mark.parametrize( + "input,params,expected", + ( + ("test", {}, "test"), # str input + (b"test", {}, "test"), # bytes input + (b"test", {"encoding": "utf-8"}, "test"), # bytes input with utf-8 + (b"test", {"encoding": "ascii"}, "test"), # bytes input with ascii + ), + ) + def test_to_str( + self, input: bytes | str, params: dict[str, str], expected: str + ) -> None: + assert to_str(input, **params) == expected + + def test_to_str_error(self) -> None: + with pytest.raises(TypeError, match="not expecting type int"): + to_str(1) # type: ignore[arg-type] + + @pytest.mark.parametrize( + "input,params,expected", + ( + (b"test", {}, b"test"), # str input + ("test", {}, b"test"), # bytes input + ("é", {}, b"\xc3\xa9"), # bytes input + ("test", {"encoding": "utf-8"}, b"test"), # bytes input with utf-8 + ("test", {"encoding": "ascii"}, b"test"), # bytes input with ascii + ), + ) + def test_to_bytes( + self, input: bytes | str, params: dict[str, str], expected: bytes + ) -> None: + assert to_bytes(input, **params) == expected + + def test_to_bytes_error(self) -> None: + with pytest.raises(TypeError, match="not expecting type int"): + to_bytes(1) # type: ignore[arg-type] -class TestUtilSSL(object): + +class TestUtilSSL: """Test utils that use an SSL backend.""" @pytest.mark.parametrize( @@ -878,7 +975,9 @@ class TestUtilSSL(object): ("CERT_REQUIRED", ssl.CERT_REQUIRED), ], ) - def test_resolve_cert_reqs(self, candidate, requirements): + def test_resolve_cert_reqs( + self, candidate: int | str | None, requirements: int + ) -> None: assert resolve_cert_reqs(candidate) == requirements @pytest.mark.parametrize( @@ -890,11 +989,11 @@ def test_resolve_cert_reqs(self, candidate, requirements): (ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23), ], ) - def test_resolve_ssl_version(self, candidate, version): + def test_resolve_ssl_version(self, candidate: int | str, version: int) -> None: assert resolve_ssl_version(candidate) == version - def test_ssl_wrap_socket_loads_the_cert_chain(self): - socket = object() + def test_ssl_wrap_socket_loads_the_cert_chain(self) -> None: + socket = Mock() mock_context = Mock() ssl_wrap_socket( ssl_context=mock_context, sock=socket, certfile="/path/to/certfile" @@ -903,24 +1002,24 @@ def test_ssl_wrap_socket_loads_the_cert_chain(self): mock_context.load_cert_chain.assert_called_once_with("/path/to/certfile", None) @patch("urllib3.util.ssl_.create_urllib3_context") - def test_ssl_wrap_socket_creates_new_context(self, create_urllib3_context): - socket = object() - ssl_wrap_socket(sock=socket, cert_reqs="CERT_REQUIRED") + def test_ssl_wrap_socket_creates_new_context( + self, create_urllib3_context: mock.MagicMock + ) -> None: + socket = Mock() + ssl_wrap_socket(socket, cert_reqs=ssl.CERT_REQUIRED) - create_urllib3_context.assert_called_once_with( - None, "CERT_REQUIRED", ciphers=None - ) + create_urllib3_context.assert_called_once_with(None, 2, ciphers=None) - def test_ssl_wrap_socket_loads_verify_locations(self): - socket = object() + def test_ssl_wrap_socket_loads_verify_locations(self) -> None: + socket = Mock() mock_context = Mock() ssl_wrap_socket(ssl_context=mock_context, ca_certs="/path/to/pem", sock=socket) mock_context.load_verify_locations.assert_called_once_with( "/path/to/pem", None, None ) - def test_ssl_wrap_socket_loads_certificate_directories(self): - socket = object() + def test_ssl_wrap_socket_loads_certificate_directories(self) -> None: + socket = Mock() mock_context = Mock() ssl_wrap_socket( ssl_context=mock_context, ca_cert_dir="/path/to/pems", sock=socket @@ -929,8 +1028,8 @@ def test_ssl_wrap_socket_loads_certificate_directories(self): None, "/path/to/pems", None ) - def test_ssl_wrap_socket_loads_certificate_data(self): - socket = object() + def test_ssl_wrap_socket_loads_certificate_data(self) -> None: + socket = Mock() mock_context = Mock() ssl_wrap_socket( ssl_context=mock_context, ca_cert_data="TOTALLY PEM DATA", sock=socket @@ -939,7 +1038,9 @@ def test_ssl_wrap_socket_loads_certificate_data(self): None, None, "TOTALLY PEM DATA" ) - def _wrap_socket_and_mock_warn(self, sock, server_hostname): + def _wrap_socket_and_mock_warn( + self, sock: socket.socket, server_hostname: str | None + ) -> tuple[Mock, MagicMock]: mock_context = Mock() with patch("warnings.warn") as warn: ssl_wrap_socket( @@ -949,34 +1050,77 @@ def _wrap_socket_and_mock_warn(self, sock, server_hostname): ) return mock_context, warn - def test_ssl_wrap_socket_sni_hostname_use_or_warn(self): - """Test that either an SNI hostname is used or a warning is made.""" - sock = object() - context, warn = self._wrap_socket_and_mock_warn(sock, "www.google.com") - if util.HAS_SNI: - warn.assert_not_called() - context.wrap_socket.assert_called_once_with( - sock, server_hostname="www.google.com" - ) - else: - assert warn.call_count >= 1 - warnings = [call[0][1] for call in warn.call_args_list] - assert SNIMissingWarning in warnings - context.wrap_socket.assert_called_once_with(sock) - - def test_ssl_wrap_socket_sni_ip_address_no_warn(self): + def test_ssl_wrap_socket_sni_ip_address_no_warn(self) -> None: """Test that a warning is not made if server_hostname is an IP address.""" - sock = object() + sock = Mock() context, warn = self._wrap_socket_and_mock_warn(sock, "8.8.8.8") - if util.IS_SECURETRANSPORT: - context.wrap_socket.assert_called_once_with(sock, server_hostname="8.8.8.8") - else: - context.wrap_socket.assert_called_once_with(sock) + context.wrap_socket.assert_called_once_with(sock, server_hostname="8.8.8.8") warn.assert_not_called() - def test_ssl_wrap_socket_sni_none_no_warn(self): + def test_ssl_wrap_socket_sni_none_no_warn(self) -> None: """Test that a warning is not made if server_hostname is not given.""" - sock = object() + sock = Mock() context, warn = self._wrap_socket_and_mock_warn(sock, None) - context.wrap_socket.assert_called_once_with(sock) + context.wrap_socket.assert_called_once_with(sock, server_hostname=None) warn.assert_not_called() + + @pytest.mark.parametrize( + "openssl_version, openssl_version_number, implementation_name, version_info, pypy_version_info, reliable", + [ + # OpenSSL and Python OK -> reliable + ("OpenSSL 1.1.1", 0x101010CF, "cpython", (3, 9, 3), None, True), + # Python OK -> reliable + ("OpenSSL 1.1.1", 0x10101000, "cpython", (3, 9, 3), None, True), + # PyPy: depends on the version + ("OpenSSL 1.1.1", 0x10101000, "pypy", (3, 9, 9), (7, 3, 7), False), + ("OpenSSL 1.1.1", 0x101010CF, "pypy", (3, 9, 19), (7, 3, 16), True), + # OpenSSL OK -> reliable + ("OpenSSL 1.1.1", 0x101010CF, "cpython", (3, 9, 2), None, True), + # not OpenSSSL -> unreliable + ("LibreSSL 2.8.3", 0x101010CF, "cpython", (3, 10, 0), None, False), + # old OpenSSL and old Python, unreliable + ("OpenSSL 1.1.0", 0x10101000, "cpython", (3, 9, 2), None, False), + ], + ) + def test_is_has_never_check_common_name_reliable( + self, + openssl_version: str, + openssl_version_number: int, + implementation_name: str, + version_info: _TYPE_VERSION_INFO, + pypy_version_info: _TYPE_VERSION_INFO | None, + reliable: bool, + ) -> None: + assert ( + _is_has_never_check_common_name_reliable( + openssl_version, + openssl_version_number, + implementation_name, + version_info, + pypy_version_info, + ) + == reliable + ) + + +idna_blocker = ImportBlocker("idna") +module_stash = ModuleStash("urllib3") + + +class TestUtilWithoutIdna: + @classmethod + def setup_class(cls) -> None: + sys.modules.pop("idna", None) + + module_stash.stash() + sys.meta_path.insert(0, idna_blocker) + + @classmethod + def teardown_class(cls) -> None: + sys.meta_path.remove(idna_blocker) + module_stash.pop() + + def test_parse_url_without_idna(self) -> None: + url = "http://\uD7FF.com" + with pytest.raises(LocationParseError, match=f"Failed to parse: {url}"): + parse_url(url) diff --git a/test/test_wait.py b/test/test_wait.py index 38dad79..3721274 100644 --- a/test/test_wait.py +++ b/test/test_wait.py @@ -1,13 +1,11 @@ +from __future__ import annotations + import signal -import socket import threading - -try: - from time import monotonic -except ImportError: - from time import time as monotonic - import time +import typing +from socket import socket, socketpair +from types import FrameType import pytest @@ -20,24 +18,25 @@ wait_for_write, ) -from .socketpair_helper import socketpair +TYPE_SOCKET_PAIR = tuple[socket, socket] +TYPE_WAIT_FOR = typing.Callable[..., bool] @pytest.fixture -def spair(): +def spair() -> typing.Generator[TYPE_SOCKET_PAIR]: a, b = socketpair() yield a, b a.close() b.close() -variants = [wait_for_socket, select_wait_for_socket] +variants: list[TYPE_WAIT_FOR] = [wait_for_socket, select_wait_for_socket] if _have_working_poll(): variants.append(poll_wait_for_socket) @pytest.mark.parametrize("wfs", variants) -def test_wait_for_socket(wfs, spair): +def test_wait_for_socket(wfs: TYPE_WAIT_FOR, spair: TYPE_SOCKET_PAIR) -> None: a, b = spair with pytest.raises(RuntimeError): @@ -56,7 +55,7 @@ def test_wait_for_socket(wfs, spair): try: while True: a.send(b"x" * 999999) - except (OSError, socket.error): + except OSError: pass # Now it's not writable anymore @@ -80,7 +79,7 @@ def test_wait_for_socket(wfs, spair): wfs(b, read=True) -def test_wait_for_read_write(spair): +def test_wait_for_read_write(spair: TYPE_SOCKET_PAIR) -> None: a, b = spair assert not wait_for_read(a, 0) @@ -96,7 +95,7 @@ def test_wait_for_read_write(spair): try: while True: a.send(b"x" * 999999) - except (OSError, socket.error): + except OSError: pass # Now it's not writable anymore @@ -105,18 +104,18 @@ def test_wait_for_read_write(spair): @pytest.mark.skipif(not hasattr(signal, "setitimer"), reason="need setitimer() support") @pytest.mark.parametrize("wfs", variants) -def test_eintr(wfs, spair): +def test_eintr(wfs: TYPE_WAIT_FOR, spair: TYPE_SOCKET_PAIR) -> None: a, b = spair interrupt_count = [0] - def handler(sig, frame): + def handler(sig: int, frame: FrameType | None) -> typing.Any: assert sig == signal.SIGALRM interrupt_count[0] += 1 old_handler = signal.signal(signal.SIGALRM, handler) try: assert not wfs(a, read=True, timeout=0) - start = monotonic() + start = time.monotonic() try: # Start delivering SIGALRM 10 times per second signal.setitimer(signal.ITIMER_REAL, 0.1, 0.1) @@ -125,7 +124,7 @@ def handler(sig, frame): finally: # Stop delivering SIGALRM signal.setitimer(signal.ITIMER_REAL, 0) - end = monotonic() + end = time.monotonic() dur = end - start assert 0.9 < dur < 3 finally: @@ -136,11 +135,11 @@ def handler(sig, frame): @pytest.mark.skipif(not hasattr(signal, "setitimer"), reason="need setitimer() support") @pytest.mark.parametrize("wfs", variants) -def test_eintr_zero_timeout(wfs, spair): +def test_eintr_zero_timeout(wfs: TYPE_WAIT_FOR, spair: TYPE_SOCKET_PAIR) -> None: a, b = spair interrupt_count = [0] - def handler(sig, frame): + def handler(sig: int, frame: FrameType | None) -> typing.Any: assert sig == signal.SIGALRM interrupt_count[0] += 1 @@ -154,8 +153,11 @@ def handler(sig, frame): signal.setitimer(signal.ITIMER_REAL, 0.001, 0.001) # Hammer the system call for a while to trigger the # race. + end = time.monotonic() + 5 for i in range(100000): wfs(a, read=True, timeout=0) + if time.monotonic() >= end: + break finally: # Stop delivering SIGALRM signal.setitimer(signal.ITIMER_REAL, 0) @@ -167,22 +169,22 @@ def handler(sig, frame): @pytest.mark.skipif(not hasattr(signal, "setitimer"), reason="need setitimer() support") @pytest.mark.parametrize("wfs", variants) -def test_eintr_infinite_timeout(wfs, spair): +def test_eintr_infinite_timeout(wfs: TYPE_WAIT_FOR, spair: TYPE_SOCKET_PAIR) -> None: a, b = spair interrupt_count = [0] - def handler(sig, frame): + def handler(sig: int, frame: FrameType | None) -> typing.Any: assert sig == signal.SIGALRM interrupt_count[0] += 1 - def make_a_readable_after_one_second(): + def make_a_readable_after_one_second() -> None: time.sleep(1) b.send(b"x") old_handler = signal.signal(signal.SIGALRM, handler) try: assert not wfs(a, read=True, timeout=0) - start = monotonic() + start = time.monotonic() try: # Start delivering SIGALRM 10 times per second signal.setitimer(signal.ITIMER_REAL, 0.1, 0.1) @@ -194,7 +196,7 @@ def make_a_readable_after_one_second(): # Stop delivering SIGALRM signal.setitimer(signal.ITIMER_REAL, 0) thread.join() - end = monotonic() + end = time.monotonic() dur = end - start assert 0.9 < dur < 3 finally: diff --git a/test/tz_stub.py b/test/tz_stub.py index c48f5df..fc8e559 100644 --- a/test/tz_stub.py +++ b/test/tz_stub.py @@ -1,14 +1,17 @@ +from __future__ import annotations + import datetime import os import time +import typing +import zoneinfo from contextlib import contextmanager import pytest -from dateutil import tz @contextmanager -def stub_timezone_ctx(tzname): +def stub_timezone_ctx(tzname: str | None) -> typing.Generator[None]: """ Switch to a locally-known timezone specified by `tzname`. On exit, restore the previous timezone. @@ -22,16 +25,16 @@ def stub_timezone_ctx(tzname): if not hasattr(time, "tzset"): pytest.skip("Timezone patching is not supported") - # Make sure the new timezone exists, at least in dateutil - new_tz = tz.gettz(tzname) - if new_tz is None: - raise ValueError("Invalid timezone specified: %r" % (tzname,)) + # Make sure the new timezone exists + try: + zoneinfo.ZoneInfo(tzname) + except zoneinfo.ZoneInfoNotFoundError: + raise ValueError(f"Invalid timezone specified: {tzname!r}") # Get the current timezone - local_tz = tz.tzlocal() - if local_tz is None: - raise EnvironmentError("Cannot determine current timezone") - old_tzname = datetime.datetime.now(local_tz).tzname() + old_tzname = datetime.datetime.now().astimezone().tzname() + if old_tzname is None: + raise OSError("Cannot determine current timezone") os.environ["TZ"] = tzname time.tzset() diff --git a/test/with_dummyserver/test_chunked_transfer.py b/test/with_dummyserver/test_chunked_transfer.py index 011bb77..13dd6fd 100644 --- a/test/with_dummyserver/test_chunked_transfer.py +++ b/test/with_dummyserver/test_chunked_transfer.py @@ -1,4 +1,8 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations + +import io +import socket +import typing import pytest @@ -11,15 +15,12 @@ from urllib3.util import SKIP_HEADER from urllib3.util.retry import Retry -# Retry failed tests -pytestmark = pytest.mark.flaky - class TestChunkedTransfer(SocketDummyServerTestCase): - def start_chunked_handler(self): + def start_chunked_handler(self) -> None: self.buffer = b"" - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] while not self.buffer.endswith(b"\r\n0\r\n\r\n"): @@ -35,30 +36,54 @@ def socket_handler(listener): self._start_server(socket_handler) - def test_chunks(self): + @pytest.mark.parametrize( + "chunks", + [ + ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"], + [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"], + ], + ) + def test_chunks(self, chunks: list[bytes | str]) -> None: self.start_chunked_handler() - chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: - pool.urlopen("GET", "/", body=chunks, headers=dict(DNT="1"), chunked=True) + pool.urlopen("GET", "/", body=chunks, headers=dict(DNT="1"), chunked=True) # type: ignore[arg-type] assert b"Transfer-Encoding" in self.buffer body = self.buffer.split(b"\r\n\r\n", 1)[1] lines = body.split(b"\r\n") # Empty chunks should have been skipped, as this could not be distinguished # from terminating the transmission - for i, chunk in enumerate([c for c in chunks if c]): + for i, chunk in enumerate( + [c.decode() if isinstance(c, bytes) else c for c in chunks if c] + ): assert lines[i * 2] == hex(len(chunk))[2:].encode("utf-8") assert lines[i * 2 + 1] == chunk.encode("utf-8") - def _test_body(self, data): + def _test_body( + self, + data: ( + bytes + | str + | io.BytesIO + | io.StringIO + | typing.Iterable[bytes] + | typing.Iterable[str] + | None + ), + expected_data: bytes | None = None, + ) -> None: self.start_chunked_handler() with HTTPConnectionPool(self.host, self.port, retries=False) as pool: - pool.urlopen("GET", "/", data, chunked=True) + pool.urlopen("GET", "/", body=data, chunked=True) # type: ignore[arg-type] header, body = self.buffer.split(b"\r\n\r\n", 1) assert b"Transfer-Encoding: chunked" in header.split(b"\r\n") if data: - bdata = data if isinstance(data, bytes) else data.encode("utf-8") + if expected_data is not None: + bdata = expected_data + else: + assert isinstance(data, (bytes, str)) + bdata = data if isinstance(data, bytes) else data.encode("utf-8") assert b"\r\n" + bdata + b"\r\n" in body assert body.endswith(b"\r\n0\r\n\r\n") @@ -68,29 +93,70 @@ def _test_body(self, data): else: assert body == b"0\r\n\r\n" - def test_bytestring_body(self): + def test_bytestring_body(self) -> None: self._test_body(b"thisshouldbeonechunk\r\nasdf") - def test_unicode_body(self): - self._test_body(u"thisshouldbeonechunk\r\näöüß") - - def test_empty_body(self): + def test_unicode_body(self) -> None: + self._test_body( + "thisshouldbeonechunk\r\näöüß\xFF", + expected_data=b"thisshouldbeonechunk\r\n\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f\xc3\xbf", + ) + + @pytest.mark.parametrize( + "bytes_data", + [ + b"thisshouldbeonechunk\r\n\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f\xc3\xbf", # utf-8 + b"thisshouldbeonechunk\r\n\xe4\xf6\xfc\xdf\xff", # latin-1 + ], + ) + def test_bytes_body_fileio(self, bytes_data: bytes) -> None: + self._test_body(io.BytesIO(bytes_data), expected_data=bytes_data) + + def test_unicode_body_fileio(self) -> None: + self._test_body( + io.StringIO("thisshouldbeonechunk\r\näöüß\xFF"), + expected_data=b"thisshouldbeonechunk\r\n\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f\xc3\xbf", + ) + + @pytest.mark.parametrize( + "bytes_data", + [ + b"thisshouldbeonechunk\r\n\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f\xc3\xbf", # utf-8 + b"thisshouldbeonechunk\r\n\xe4\xf6\xfc\xdf\xff", # latin-1 + ], + ) + def test_bytes_body_iterable(self, bytes_data: bytes) -> None: + def send_body() -> typing.Iterable[bytes]: + yield bytes_data + + self._test_body(send_body(), expected_data=bytes_data) + + def test_unicode_body_iterable(self) -> None: + def send_body() -> typing.Iterable[str]: + yield "thisshouldbeonechunk\r\näöüß\xFF" + + self._test_body( + send_body(), + expected_data=b"thisshouldbeonechunk\r\n\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f\xc3\xbf", + ) + + def test_empty_body(self) -> None: self._test_body(None) - def test_empty_string_body(self): + def test_empty_string_body(self) -> None: self._test_body("") - def test_empty_iterable_body(self): - self._test_body([]) + def test_empty_iterable_body(self) -> None: + self._test_body(None) - def _get_header_lines(self, prefix): + def _get_header_lines(self, prefix: bytes) -> list[bytes]: header_block = self.buffer.split(b"\r\n\r\n", 1)[0].lower() header_lines = header_block.split(b"\r\n")[1:] return [x for x in header_lines if x.startswith(prefix)] - def test_removes_duplicate_host_header(self): + def test_removes_duplicate_host_header(self) -> None: self.start_chunked_handler() - chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"] + chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen( "GET", "/", body=chunks, headers={"Host": "test.org"}, chunked=True @@ -99,27 +165,27 @@ def test_removes_duplicate_host_header(self): host_headers = self._get_header_lines(b"host") assert len(host_headers) == 1 - def test_provides_default_host_header(self): + def test_provides_default_host_header(self) -> None: self.start_chunked_handler() - chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"] + chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen("GET", "/", body=chunks, chunked=True) host_headers = self._get_header_lines(b"host") assert len(host_headers) == 1 - def test_provides_default_user_agent_header(self): + def test_provides_default_user_agent_header(self) -> None: self.start_chunked_handler() - chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"] + chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen("GET", "/", body=chunks, chunked=True) ua_headers = self._get_header_lines(b"user-agent") assert len(ua_headers) == 1 - def test_preserve_user_agent_header(self): + def test_preserve_user_agent_header(self) -> None: self.start_chunked_handler() - chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"] + chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen( "GET", @@ -136,9 +202,9 @@ def test_preserve_user_agent_header(self): # provided. assert ua_headers[0] == b"user-agent: test-agent" - def test_remove_user_agent_header(self): + def test_remove_user_agent_header(self) -> None: self.start_chunked_handler() - chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"] + chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen( "GET", @@ -151,18 +217,18 @@ def test_remove_user_agent_header(self): ua_headers = self._get_header_lines(b"user-agent") assert len(ua_headers) == 0 - def test_provides_default_transfer_encoding_header(self): + def test_provides_default_transfer_encoding_header(self) -> None: self.start_chunked_handler() - chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"] + chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen("GET", "/", body=chunks, chunked=True) te_headers = self._get_header_lines(b"transfer-encoding") assert len(te_headers) == 1 - def test_preserve_transfer_encoding_header(self): + def test_preserve_transfer_encoding_header(self) -> None: self.start_chunked_handler() - chunks = ["foo", "bar", "", "bazzzzzzzzzzzzzzzzzzzzzz"] + chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"] with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.urlopen( "GET", @@ -179,11 +245,11 @@ def test_preserve_transfer_encoding_header(self): # was provided. assert te_headers[0] == b"transfer-encoding: test-transfer-encoding" - def test_preserve_chunked_on_retry_after(self): + def test_preserve_chunked_on_retry_after(self) -> None: self.chunked_requests = 0 - self.socks = [] + self.socks: list[socket.socket] = [] - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: for _ in range(2): sock = listener.accept()[0] self.socks.append(sock) @@ -208,10 +274,12 @@ def socket_handler(listener): sock.close() assert self.chunked_requests == 2 - def test_preserve_chunked_on_redirect(self, monkeypatch): + def test_preserve_chunked_on_redirect( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: self.chunked_requests = 0 - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: for i in range(2): sock = listener.accept()[0] request = ConnectionMarker.consume_request(sock) @@ -236,10 +304,12 @@ def socket_handler(listener): ) assert self.chunked_requests == 2 - def test_preserve_chunked_on_broken_connection(self, monkeypatch): + def test_preserve_chunked_on_broken_connection( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: self.chunked_requests = 0 - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: for i in range(2): sock = listener.accept()[0] request = ConnectionMarker.consume_request(sock) diff --git a/test/with_dummyserver/test_connection.py b/test/with_dummyserver/test_connection.py new file mode 100644 index 0000000..b9c547c --- /dev/null +++ b/test/with_dummyserver/test_connection.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import contextlib +import typing +from http.client import ResponseNotReady +from unittest import mock + +import pytest + +from dummyserver.testcase import HypercornDummyServerTestCase as server +from urllib3 import HTTPConnectionPool +from urllib3.response import HTTPResponse + + +@pytest.fixture() +def pool() -> typing.Generator[HTTPConnectionPool]: + server.setup_class() + + with HTTPConnectionPool(server.host, server.port) as pool: + yield pool + + server.teardown_class() + + +def test_returns_urllib3_HTTPResponse(pool: HTTPConnectionPool) -> None: + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + response = conn.getresponse() + assert isinstance(response, HTTPResponse) + + +@mock.patch("urllib3.connection.sys.audit") +def test_audit_event(audit_mock: mock.Mock, pool: HTTPConnectionPool) -> None: + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + audit_mock.assert_any_call("http.client.connect", conn, conn.host, conn.port) + # Ensure the event is raised only once. + connect_events = [ + call + for call in audit_mock.mock_calls + if call.args[0] == "http.client.connect" + ] + assert len(connect_events) == 1 + + +def test_does_not_release_conn(pool: HTTPConnectionPool) -> None: + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + response = conn.getresponse() + + response.release_conn() + assert pool.pool.qsize() == 0 # type: ignore[union-attr] + + +def test_releases_conn(pool: HTTPConnectionPool) -> None: + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + response = conn.getresponse() + + # If these variables are set by the pool + # then the response can release the connection + # back into the pool. + response._pool = pool # type: ignore[attr-defined] + response._connection = conn # type: ignore[attr-defined] + + response.release_conn() + assert pool.pool.qsize() == 1 # type: ignore[union-attr] + + +def test_double_getresponse(pool: HTTPConnectionPool) -> None: + with contextlib.closing(pool._get_conn()) as conn: + conn.request("GET", "/") + _ = conn.getresponse() + + # Calling getrepsonse() twice should cause an error + with pytest.raises(ResponseNotReady): + conn.getresponse() + + +def test_connection_state_properties(pool: HTTPConnectionPool) -> None: + conn = pool._get_conn() + + assert conn.is_closed is True + assert conn.is_connected is False + assert conn.has_connected_to_proxy is False + assert conn.is_verified is False + assert conn.proxy_is_verified is None + + conn.connect() + + assert conn.is_closed is False + assert conn.is_connected is True + assert conn.has_connected_to_proxy is False + assert conn.is_verified is False + assert conn.proxy_is_verified is None + + conn.request("GET", "/") + resp = conn.getresponse() + assert resp.status == 200 + + conn.close() + + assert conn.is_closed is True + assert conn.is_connected is False + assert conn.has_connected_to_proxy is False + assert conn.is_verified is False + assert conn.proxy_is_verified is None + + +def test_set_tunnel_is_reset(pool: HTTPConnectionPool) -> None: + conn = pool._get_conn() + + assert conn.is_closed is True + assert conn.is_connected is False + assert conn.has_connected_to_proxy is False + assert conn.is_verified is False + assert conn.proxy_is_verified is None + + conn.set_tunnel(host="host", port=8080, scheme="http") + + assert conn._tunnel_host == "host" # type: ignore[attr-defined] + assert conn._tunnel_port == 8080 # type: ignore[attr-defined] + assert conn._tunnel_scheme == "http" # type: ignore[attr-defined] + + conn.close() + + assert conn._tunnel_host is None # type: ignore[attr-defined] + assert conn._tunnel_port is None # type: ignore[attr-defined] + assert conn._tunnel_scheme is None # type: ignore[attr-defined] + + +def test_invalid_tunnel_scheme(pool: HTTPConnectionPool) -> None: + conn = pool._get_conn() + + with pytest.raises(ValueError) as e: + conn.set_tunnel(host="host", port=8080, scheme="socks") + assert ( + str(e.value) + == "Invalid proxy scheme for tunneling: 'socks', must be either 'http' or 'https'" + ) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index cde027b..767b463 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -1,23 +1,21 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations import io -import json import logging -import os import platform import socket -import sys import time +import typing import warnings -from test import LONG_TIMEOUT, SHORT_TIMEOUT, onlyPy2 +from test import LONG_TIMEOUT, SHORT_TIMEOUT from threading import Event +from unittest import mock +from urllib.parse import urlencode -import mock import pytest -import six -from dummyserver.server import HAS_IPV6_AND_DNS, NoIPv6Warning -from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase +from dummyserver.socketserver import NoIPv6Warning +from dummyserver.testcase import HypercornDummyServerTestCase, SocketDummyServerTestCase from urllib3 import HTTPConnectionPool, encode_multipart_formdata from urllib3._collections import HTTPHeaderDict from urllib3.connection import _get_default_user_agent @@ -26,33 +24,27 @@ DecodeError, EmptyPoolError, MaxRetryError, + NameResolutionError, NewConnectionError, ReadTimeoutError, UnrewindableBodyError, ) -from urllib3.packages.six import b, u -from urllib3.packages.six.moves.urllib.parse import urlencode +from urllib3.fields import _TYPE_FIELD_VALUE_TUPLE from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS from urllib3.util.retry import RequestHistory, Retry -from urllib3.util.timeout import Timeout +from urllib3.util.timeout import _TYPE_TIMEOUT, Timeout from .. import INVALID_SOURCE_ADDRESSES, TARPIT_HOST, VALID_SOURCE_ADDRESSES from ..port_helpers import find_unused_port -pytestmark = pytest.mark.flaky -log = logging.getLogger("urllib3.connectionpool") -log.setLevel(logging.NOTSET) -log.addHandler(logging.StreamHandler(sys.stdout)) - - -def wait_for_socket(ready_event): +def wait_for_socket(ready_event: Event) -> None: ready_event.wait() ready_event.clear() class TestConnectionPoolTimeouts(SocketDummyServerTestCase): - def test_timeout_float(self): + def test_timeout_float(self) -> None: block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=2) @@ -67,7 +59,7 @@ def test_timeout_float(self): block_event.set() # Pre-release block pool.request("GET", "/", timeout=LONG_TIMEOUT) - def test_conn_closed(self): + def test_conn_closed(self) -> None: block_event = Event() self.start_basic_handler(block_send=block_event, num=1) @@ -79,15 +71,15 @@ def test_conn_closed(self): try: with pytest.raises(ReadTimeoutError): pool.urlopen("GET", "/") - if conn.sock: + if not conn.is_closed: with pytest.raises(socket.error): - conn.sock.recv(1024) + conn.sock.recv(1024) # type: ignore[attr-defined] finally: pool._put_conn(conn) block_event.set() - def test_timeout(self): + def test_timeout(self) -> None: # Requests should time out when expected block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=3) @@ -108,13 +100,18 @@ def test_timeout(self): self.host, self.port, timeout=short_timeout, retries=False ) as pool: wait_for_socket(ready_event) - now = time.time() + now = time.perf_counter() with pytest.raises(ReadTimeoutError): pool.request("GET", "/", timeout=LONG_TIMEOUT) - delta = time.time() - now + delta = time.perf_counter() - now message = "timeout was pool-level SHORT_TIMEOUT rather than request-level LONG_TIMEOUT" - assert delta >= LONG_TIMEOUT, message + if platform.system() == "Windows": + # Adjust tolerance for floating-point comparison on Windows to + # avoid flakiness in CI #3413 + assert delta >= (LONG_TIMEOUT - 1e-3), message + else: + assert delta >= (LONG_TIMEOUT - 1e-5), message block_event.set() # Release request # Timeout passed directly to request should raise a request timeout @@ -123,7 +120,7 @@ def test_timeout(self): pool.request("GET", "/", timeout=SHORT_TIMEOUT) block_event.set() # Release request - def test_connect_timeout(self): + def test_connect_timeout(self) -> None: url = "/" host, port = TARPIT_HOST, 80 timeout = Timeout(connect=SHORT_TIMEOUT) @@ -150,7 +147,7 @@ def test_connect_timeout(self): with pytest.raises(ConnectTimeoutError): pool.request("GET", url, timeout=timeout) - def test_total_applies_connect(self): + def test_total_applies_connect(self) -> None: host, port = TARPIT_HOST, 80 timeout = Timeout(total=None, connect=SHORT_TIMEOUT) @@ -171,7 +168,7 @@ def test_total_applies_connect(self): finally: conn.close() - def test_total_timeout(self): + def test_total_timeout(self) -> None: block_event = Event() ready_event = self.start_basic_handler(block_send=block_event, num=2) @@ -196,7 +193,7 @@ def test_total_timeout(self): with pytest.raises(ReadTimeoutError): pool.request("GET", "/") - def test_create_connection_timeout(self): + def test_create_connection_timeout(self) -> None: self.start_basic_handler(block_send=Event(), num=0) # needed for self.port timeout = Timeout(connect=SHORT_TIMEOUT, total=LONG_TIMEOUT) @@ -208,23 +205,45 @@ def test_create_connection_timeout(self): conn.connect() -class TestConnectionPool(HTTPDummyServerTestCase): - def test_get(self): +class TestConnectionPool(HypercornDummyServerTestCase): + def test_http2_test_error(self, http_version: str) -> None: + with HTTPConnectionPool(self.host, self.port) as pool: + if http_version == "h2": + with pytest.raises( + ValueError, match="HTTP/2 support currently only applies to HTTPS.*" + ): + r = pool.request("GET", "/") + else: + r = pool.request("GET", "/") + assert r.status == 200 + + def test_get(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/specific_method", fields={"method": "GET"}) assert r.status == 200, r.data - def test_post_url(self): + def test_debug_log(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.DEBUG, logger="urllib3.connectionpool") + with HTTPConnectionPool(self.host, self.port) as pool: + r = pool.urlopen("GET", "/") + assert r.status == 200 + logs = [record.getMessage() for record in caplog.records] + assert logs == [ + f"Starting new HTTP connection (1): {self.host}:{self.port}", + f'http://{self.host}:{self.port} "GET / HTTP/1.1" 200 0', + ] + + def test_post_url(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("POST", "/specific_method", fields={"method": "POST"}) assert r.status == 200, r.data - def test_urlopen_put(self): + def test_urlopen_put(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.urlopen("PUT", "/specific_method?method=PUT") assert r.status == 200, r.data - def test_wrong_specific_method(self): + def test_wrong_specific_method(self) -> None: # To make sure the dummy server is actually returning failed responses with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/specific_method", fields={"method": "POST"}) @@ -234,20 +253,20 @@ def test_wrong_specific_method(self): r = pool.request("POST", "/specific_method", fields={"method": "GET"}) assert r.status == 400, r.data - def test_upload(self): + def test_upload(self) -> None: data = "I'm in ur multipart form-data, hazing a cheezburgr" - fields = { + fields: dict[str, _TYPE_FIELD_VALUE_TUPLE] = { "upload_param": "filefield", "upload_filename": "lolcat.txt", - "upload_size": len(data), "filefield": ("lolcat.txt", data), } + fields["upload_size"] = len(data) # type: ignore[assignment] with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("POST", "/upload", fields=fields) assert r.status == 200, r.data - def test_one_name_multiple_values(self): + def test_one_name_multiple_values(self) -> None: fields = [("foo", "a"), ("foo", "b")] with HTTPConnectionPool(self.host, self.port) as pool: @@ -259,7 +278,7 @@ def test_one_name_multiple_values(self): r = pool.request("POST", "/echo", fields=fields) assert r.data.count(b'name="foo"') == 2 - def test_request_method_body(self): + def test_request_method_body(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: body = b"hi" r = pool.request("POST", "/echo", body=body) @@ -269,23 +288,23 @@ def test_request_method_body(self): with pytest.raises(TypeError): pool.request("POST", "/echo", body=body, fields=fields) - def test_unicode_upload(self): - fieldname = u("myfile") - filename = u("\xe2\x99\xa5.txt") - data = u("\xe2\x99\xa5").encode("utf8") + def test_unicode_upload(self) -> None: + fieldname = "myfile" + filename = "\xe2\x99\xa5.txt" + data = "\xe2\x99\xa5".encode() size = len(data) - fields = { - u("upload_param"): fieldname, - u("upload_filename"): filename, - u("upload_size"): size, + fields: dict[str, _TYPE_FIELD_VALUE_TUPLE] = { + "upload_param": fieldname, + "upload_filename": filename, fieldname: (filename, data), } + fields["upload_size"] = size # type: ignore[assignment] with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("POST", "/upload", fields=fields) assert r.status == 200, r.data - def test_nagle(self): + def test_nagle(self) -> None: """Test that connections have TCP_NODELAY turned on""" # This test needs to be here in order to be run. socket.create_connection actually tries # to connect to the host provided so we need a dummyserver to be running. @@ -293,23 +312,31 @@ def test_nagle(self): conn = pool._get_conn() try: pool._make_request(conn, "GET", "/") - tcp_nodelay_setting = conn.sock.getsockopt( + tcp_nodelay_setting = conn.sock.getsockopt( # type: ignore[attr-defined] socket.IPPROTO_TCP, socket.TCP_NODELAY ) assert tcp_nodelay_setting finally: conn.close() - def test_socket_options(self): + @pytest.mark.parametrize( + "socket_options", + [ + [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)], + ((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),), + ], + ) + def test_socket_options(self, socket_options: tuple[int, int, int]) -> None: """Test that connections accept socket options.""" # This test needs to be here in order to be run. socket.create_connection actually tries to # connect to the host provided so we need a dummyserver to be running. with HTTPConnectionPool( self.host, self.port, - socket_options=[(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)], + socket_options=socket_options, ) as pool: - s = pool._new_conn()._new_conn() # Get the socket + # Get the socket of a new connection. + s = pool._new_conn()._new_conn() # type: ignore[attr-defined] try: using_keepalive = ( s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0 @@ -318,19 +345,24 @@ def test_socket_options(self): finally: s.close() - def test_disable_default_socket_options(self): - """Test that passing None disables all socket options.""" + @pytest.mark.parametrize("socket_options", [None, []]) + def test_disable_default_socket_options( + self, socket_options: list[int] | None + ) -> None: + """Test that passing None or empty list disables all socket options.""" # This test needs to be here in order to be run. socket.create_connection actually tries # to connect to the host provided so we need a dummyserver to be running. - with HTTPConnectionPool(self.host, self.port, socket_options=None) as pool: - s = pool._new_conn()._new_conn() + with HTTPConnectionPool( + self.host, self.port, socket_options=socket_options + ) as pool: + s = pool._new_conn()._new_conn() # type: ignore[attr-defined] try: using_nagle = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) == 0 assert using_nagle finally: s.close() - def test_defaults_are_applied(self): + def test_defaults_are_applied(self) -> None: """Test that modifying the default socket options works.""" # This test needs to be here in order to be run. socket.create_connection actually tries # to connect to the host provided so we need a dummyserver to be running. @@ -339,10 +371,9 @@ def test_defaults_are_applied(self): conn = pool._new_conn() try: # Update the default socket options - conn.default_socket_options += [ - (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - ] - s = conn._new_conn() + assert conn.socket_options is not None + conn.socket_options += [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)] + s = conn._new_conn() # type: ignore[attr-defined] nagle_disabled = ( s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) > 0 ) @@ -355,15 +386,15 @@ def test_defaults_are_applied(self): conn.close() s.close() - def test_connection_error_retries(self): + def test_connection_error_retries(self) -> None: """ECONNREFUSED error should raise a connection error, with retries""" port = find_unused_port() with HTTPConnectionPool(self.host, port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=Retry(connect=3)) - assert type(e.value.reason) == NewConnectionError + assert type(e.value.reason) is NewConnectionError - def test_timeout_success(self): + def test_timeout_success(self) -> None: timeout = Timeout(connect=3, read=5, total=None) with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: pool.request("GET", "/") @@ -391,8 +422,10 @@ def test_timeout_success(self): @socket_timeout_reuse_testdata def test_socket_timeout_updated_on_reuse_constructor( - self, timeout, expect_settimeout_calls - ): + self, + timeout: _TYPE_TIMEOUT, + expect_settimeout_calls: typing.Sequence[float | None], + ) -> None: with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: # Make a request to create a new connection. pool.urlopen("GET", "/") @@ -412,8 +445,10 @@ def test_socket_timeout_updated_on_reuse_constructor( @socket_timeout_reuse_testdata def test_socket_timeout_updated_on_reuse_parameter( - self, timeout, expect_settimeout_calls - ): + self, + timeout: _TYPE_TIMEOUT, + expect_settimeout_calls: typing.Sequence[float | None], + ) -> None: with HTTPConnectionPool(self.host, self.port) as pool: # Make a request to create a new connection. pool.urlopen("GET", "/", timeout=LONG_TIMEOUT) @@ -431,16 +466,17 @@ def test_socket_timeout_updated_on_reuse_parameter( [mock.call(x) for x in expect_settimeout_calls] ) - def test_tunnel(self): - # note the actual httplib.py has no tests for this functionality + def test_tunnel(self) -> None: timeout = Timeout(total=None) with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: conn = pool._get_conn() try: conn.set_tunnel(self.host, self.port) - conn._tunnel = mock.Mock(return_value=None) - pool._make_request(conn, "GET", "/") - conn._tunnel.assert_called_once_with() + with mock.patch.object( + conn, "_tunnel", create=True, return_value=None + ) as conn_tunnel: + pool._make_request(conn, "GET", "/") + conn_tunnel.assert_called_once_with() finally: conn.close() @@ -449,13 +485,21 @@ def test_tunnel(self): with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool: conn = pool._get_conn() try: - conn._tunnel = mock.Mock(return_value=None) - pool._make_request(conn, "GET", "/") - assert not conn._tunnel.called + with mock.patch.object( + conn, "_tunnel", create=True, return_value=None + ) as conn_tunnel: + pool._make_request(conn, "GET", "/") + assert not conn_tunnel.called finally: conn.close() - def test_redirect(self): + def test_redirect_relative_url_no_deprecation(self) -> None: + with HTTPConnectionPool(self.host, self.port) as pool: + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + pool.request("GET", "/redirect", fields={"target": "/"}) + + def test_redirect(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/redirect", fields={"target": "/"}, redirect=False) assert r.status == 303 @@ -464,24 +508,24 @@ def test_redirect(self): assert r.status == 200 assert r.data == b"Dummy server!" - def test_303_redirect_makes_request_lose_body(self): + def test_303_redirect_makes_request_lose_body(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: response = pool.request( "POST", "/redirect", fields={"target": "/headers_and_params", "status": "303 See Other"}, ) - data = json.loads(response.data) + data = response.json() assert data["params"] == {} assert "Content-Type" not in HTTPHeaderDict(data["headers"]) - def test_bad_connect(self): + def test_bad_connect(self) -> None: with HTTPConnectionPool("badhost.invalid", self.port) as pool: with pytest.raises(MaxRetryError) as e: pool.request("GET", "/", retries=5) - assert type(e.value.reason) == NewConnectionError + assert type(e.value.reason) is NameResolutionError - def test_keepalive(self): + def test_keepalive(self) -> None: with HTTPConnectionPool(self.host, self.port, block=True, maxsize=1) as pool: r = pool.request("GET", "/keepalive?close=0") r = pool.request("GET", "/keepalive?close=0") @@ -490,7 +534,7 @@ def test_keepalive(self): assert pool.num_connections == 1 assert pool.num_requests == 2 - def test_keepalive_close(self): + def test_keepalive_close(self) -> None: with HTTPConnectionPool( self.host, self.port, block=True, maxsize=1, timeout=2 ) as pool: @@ -505,7 +549,7 @@ def test_keepalive_close(self): # We grab the HTTPConnection object straight from the Queue, # because _get_conn() is where the check & reset occurs - # pylint: disable-msg=W0212 + assert pool.pool is not None conn = pool.pool.get() assert conn.sock is None pool._put_conn(conn) @@ -539,13 +583,13 @@ def test_keepalive_close(self): # Next request r = pool.request("GET", "/keepalive?close=0") - def test_post_with_urlencode(self): + def test_post_with_urlencode(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: data = {"banana": "hammock", "lol": "cat"} r = pool.request("POST", "/echo", fields=data, encode_multipart=False) assert r.data.decode("utf-8") == urlencode(data) - def test_post_with_multipart(self): + def test_post_with_multipart(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: data = {"banana": "hammock", "lol": "cat"} r = pool.request("POST", "/echo", fields=data, encode_multipart=True) @@ -569,7 +613,7 @@ def test_post_with_multipart(self): assert body[i] == expected_body[i] - def test_post_with_multipart__iter__(self): + def test_post_with_multipart__iter__(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: data = {"hello": "world"} r = pool.request( @@ -590,7 +634,7 @@ def test_post_with_multipart__iter__(self): b"--boundary--\r\n", ] - def test_check_gzip(self): + def test_check_gzip(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request( "GET", "/encodingrequest", headers={"accept-encoding": "gzip"} @@ -598,7 +642,7 @@ def test_check_gzip(self): assert r.headers.get("content-encoding") == "gzip" assert r.data == b"hello, world!" - def test_check_deflate(self): + def test_check_deflate(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request( "GET", "/encodingrequest", headers={"accept-encoding": "deflate"} @@ -606,7 +650,7 @@ def test_check_deflate(self): assert r.headers.get("content-encoding") == "deflate" assert r.data == b"hello, world!" - def test_bad_decode(self): + def test_bad_decode(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: with pytest.raises(DecodeError): pool.request( @@ -622,7 +666,7 @@ def test_bad_decode(self): headers={"accept-encoding": "garbage-gzip"}, ) - def test_connection_count(self): + def test_connection_count(self) -> None: with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool: pool.request("GET", "/") pool.request("GET", "/") @@ -631,7 +675,7 @@ def test_connection_count(self): assert pool.num_connections == 1 assert pool.num_requests == 3 - def test_connection_count_bigpool(self): + def test_connection_count_bigpool(self) -> None: with HTTPConnectionPool(self.host, self.port, maxsize=16) as http_pool: http_pool.request("GET", "/") http_pool.request("GET", "/") @@ -640,7 +684,7 @@ def test_connection_count_bigpool(self): assert http_pool.num_connections == 1 assert http_pool.num_requests == 3 - def test_partial_response(self): + def test_partial_response(self) -> None: with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool: req_data = {"lol": "cat"} resp_data = urlencode(req_data).encode("utf-8") @@ -650,7 +694,7 @@ def test_partial_response(self): assert r.read(5) == resp_data[:5] assert r.read() == resp_data[5:] - def test_lazy_load_twice(self): + def test_lazy_load_twice(self) -> None: # This test is sad and confusing. Need to figure out what's # going on with partial reads and socket reuse. @@ -703,12 +747,13 @@ def test_lazy_load_twice(self): assert pool.num_connections == 1 - def test_for_double_release(self): + def test_for_double_release(self) -> None: MAXSIZE = 5 # Check default state with HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) as pool: assert pool.num_connections == 0 + assert pool.pool is not None assert pool.pool.qsize() == MAXSIZE # Make an empty slot for testing @@ -733,16 +778,17 @@ def test_for_double_release(self): pool.urlopen("GET", "/") assert pool.pool.qsize() == MAXSIZE - 2 - def test_release_conn_parameter(self): + def test_release_conn_parameter(self) -> None: MAXSIZE = 5 with HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) as pool: + assert pool.pool is not None assert pool.pool.qsize() == MAXSIZE # Make request without releasing connection pool.request("GET", "/", release_conn=False, preload_content=False) assert pool.pool.qsize() == MAXSIZE - 1 - def test_dns_error(self): + def test_dns_error(self) -> None: with HTTPConnectionPool( "thishostdoesnotexist.invalid", self.port, timeout=0.001 ) as pool: @@ -750,48 +796,47 @@ def test_dns_error(self): pool.request("GET", "/test", retries=2) @pytest.mark.parametrize("char", [" ", "\r", "\n", "\x00"]) - def test_invalid_method_not_allowed(self, char): + def test_invalid_method_not_allowed(self, char: str) -> None: with pytest.raises(ValueError): with HTTPConnectionPool(self.host, self.port) as pool: pool.request("GET" + char, "/") - def test_percent_encode_invalid_target_chars(self): + def test_percent_encode_invalid_target_chars(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/echo_params?q=\r&k=\n \n") assert r.data == b"[('k', '\\n \\n'), ('q', '\\r')]" - @pytest.mark.skipif( - six.PY2 - and platform.system() == "Darwin" - and os.environ.get("GITHUB_ACTIONS") == "true", - reason="fails on macOS 2.7 in GitHub Actions for an unknown reason", - ) - def test_source_address(self): + def test_source_address(self) -> None: for addr, is_ipv6 in VALID_SOURCE_ADDRESSES: - if is_ipv6 and not HAS_IPV6_AND_DNS: + if is_ipv6: + # TODO enable if HAS_IPV6_AND_DNS when this is fixed: + # https://github.com/pgjones/hypercorn/issues/160 warnings.warn("No IPv6 support: skipping.", NoIPv6Warning) continue with HTTPConnectionPool( self.host, self.port, source_address=addr, retries=False ) as pool: r = pool.request("GET", "/source_address") - assert r.data == b(addr[0]) + assert r.data == addr[0].encode() - @pytest.mark.skipif( - six.PY2 - and platform.system() == "Darwin" - and os.environ.get("GITHUB_ACTIONS") == "true", - reason="fails on macOS 2.7 in GitHub Actions for an unknown reason", + @pytest.mark.parametrize( + "invalid_source_address, is_ipv6", INVALID_SOURCE_ADDRESSES ) - def test_source_address_error(self): - for addr in INVALID_SOURCE_ADDRESSES: - with HTTPConnectionPool( - self.host, self.port, source_address=addr, retries=False - ) as pool: + def test_source_address_error( + self, invalid_source_address: tuple[str, int], is_ipv6: bool + ) -> None: + with HTTPConnectionPool( + self.host, self.port, source_address=invalid_source_address, retries=False + ) as pool: + if is_ipv6: + # with pytest.raises(NameResolutionError): + with pytest.raises(NewConnectionError): + pool.request("GET", f"/source_address?{invalid_source_address}") + else: with pytest.raises(NewConnectionError): - pool.request("GET", "/source_address?{0}".format(addr)) + pool.request("GET", f"/source_address?{invalid_source_address}") - def test_stream_keepalive(self): + def test_stream_keepalive(self) -> None: x = 2 with HTTPConnectionPool(self.host, self.port) as pool: @@ -809,21 +854,21 @@ def test_stream_keepalive(self): assert pool.num_connections == 1 assert pool.num_requests == x - def test_read_chunked_short_circuit(self): + def test_read_chunked_short_circuit(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: response = pool.request("GET", "/chunked", preload_content=False) response.read() with pytest.raises(StopIteration): next(response.read_chunked()) - def test_read_chunked_on_closed_response(self): + def test_read_chunked_on_closed_response(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: response = pool.request("GET", "/chunked", preload_content=False) response.close() with pytest.raises(StopIteration): next(response.read_chunked()) - def test_chunked_gzip(self): + def test_chunked_gzip(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: response = pool.request( "GET", "/chunked_gzip", preload_content=False, decode_content=True @@ -831,7 +876,7 @@ def test_chunked_gzip(self): assert b"123" * 4 == response.read() - def test_cleanup_on_connection_error(self): + def test_cleanup_on_connection_error(self) -> None: """ Test that connections are recycled to the pool on connection errors where no http response is received. @@ -840,6 +885,7 @@ def test_cleanup_on_connection_error(self): with HTTPConnectionPool( self.host, self.port, maxsize=poolsize, block=True ) as http: + assert http.pool is not None assert http.pool.qsize() == poolsize # force a connection error by supplying a non-existent @@ -866,18 +912,18 @@ def test_cleanup_on_connection_error(self): # the pool should still contain poolsize elements assert http.pool.qsize() == http.pool.maxsize - def test_mixed_case_hostname(self): + def test_mixed_case_hostname(self) -> None: with HTTPConnectionPool("LoCaLhOsT", self.port) as pool: - response = pool.request("GET", "http://LoCaLhOsT:%d/" % self.port) + response = pool.request("GET", f"http://LoCaLhOsT:{self.port}/") assert response.status == 200 - def test_preserves_path_dot_segments(self): + def test_preserves_path_dot_segments(self) -> None: """ConnectionPool preserves dot segments in the URI""" with HTTPConnectionPool(self.host, self.port) as pool: response = pool.request("GET", "/echo_uri/seg0/../seg2") - assert response.data == b"/echo_uri/seg0/../seg2" + assert response.data == b"/echo_uri/seg0/../seg2?" - def test_default_user_agent_header(self): + def test_default_user_agent_header(self) -> None: """ConnectionPool has a default user agent""" default_ua = _get_default_user_agent() custom_ua = "I'm not a web scraper, what are you talking about?" @@ -885,26 +931,26 @@ def test_default_user_agent_header(self): with HTTPConnectionPool(self.host, self.port) as pool: # Use default user agent if no user agent was specified. r = pool.request("GET", "/headers") - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() assert request_headers.get("User-Agent") == _get_default_user_agent() # Prefer the request user agent over the default. headers = {"UsEr-AGENt": custom_ua} r = pool.request("GET", "/headers", headers=headers) - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() assert request_headers.get("User-Agent") == custom_ua # Do not modify pool headers when using the default user agent. pool_headers = {"foo": "bar"} pool.headers = pool_headers r = pool.request("GET", "/headers") - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() assert request_headers.get("User-Agent") == default_ua assert "User-Agent" not in pool_headers pool.headers.update({"User-Agent": custom_ua2}) r = pool.request("GET", "/headers") - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() assert request_headers.get("User-Agent") == custom_ua2 @pytest.mark.parametrize( @@ -919,10 +965,12 @@ def test_default_user_agent_header(self): ], ) @pytest.mark.parametrize("chunked", [True, False]) - def test_user_agent_header_not_sent_twice(self, headers, chunked): + def test_user_agent_header_not_sent_twice( + self, headers: dict[str, str] | None, chunked: bool + ) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/headers", headers=headers, chunked=chunked) - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() if not headers: assert request_headers["User-Agent"].startswith("python-urllib3/") @@ -930,21 +978,21 @@ def test_user_agent_header_not_sent_twice(self, headers, chunked): else: assert request_headers["User-Agent"] == "key" - def test_no_user_agent_header(self): + def test_no_user_agent_header(self) -> None: """ConnectionPool can suppress sending a user agent header""" custom_ua = "I'm not a web scraper, what are you talking about?" with HTTPConnectionPool(self.host, self.port) as pool: # Suppress user agent in the request headers. no_ua_headers = {"User-Agent": SKIP_HEADER} r = pool.request("GET", "/headers", headers=no_ua_headers) - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() assert "User-Agent" not in request_headers assert no_ua_headers["User-Agent"] == SKIP_HEADER # Suppress user agent in the pool headers. pool.headers = no_ua_headers r = pool.request("GET", "/headers") - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() assert "User-Agent" not in request_headers assert no_ua_headers["User-Agent"] == SKIP_HEADER @@ -952,66 +1000,22 @@ def test_no_user_agent_header(self): pool_headers = {"User-Agent": custom_ua} pool.headers = pool_headers r = pool.request("GET", "/headers", headers=no_ua_headers) - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() assert "User-Agent" not in request_headers assert no_ua_headers["User-Agent"] == SKIP_HEADER assert pool_headers.get("User-Agent") == custom_ua - @pytest.mark.parametrize( - "accept_encoding", - [ - "Accept-Encoding", - "accept-encoding", - b"Accept-Encoding", - b"accept-encoding", - None, - ], - ) - @pytest.mark.parametrize("host", ["Host", "host", b"Host", b"host", None]) - @pytest.mark.parametrize( - "user_agent", ["User-Agent", "user-agent", b"User-Agent", b"user-agent", None] - ) - @pytest.mark.parametrize("chunked", [True, False]) - def test_skip_header(self, accept_encoding, host, user_agent, chunked): - headers = {} - - if accept_encoding is not None: - headers[accept_encoding] = SKIP_HEADER - if host is not None: - headers[host] = SKIP_HEADER - if user_agent is not None: - headers[user_agent] = SKIP_HEADER - - with HTTPConnectionPool(self.host, self.port) as pool: - r = pool.request("GET", "/headers", headers=headers, chunked=chunked) - request_headers = json.loads(r.data.decode("utf8")) - - if accept_encoding is None: - assert "Accept-Encoding" in request_headers - else: - assert accept_encoding not in request_headers - if host is None: - assert "Host" in request_headers - else: - assert host not in request_headers - if user_agent is None: - assert "User-Agent" in request_headers - else: - assert user_agent not in request_headers - @pytest.mark.parametrize("header", ["Content-Length", "content-length"]) @pytest.mark.parametrize("chunked", [True, False]) - def test_skip_header_non_supported(self, header, chunked): + def test_skip_header_non_supported(self, header: str, chunked: bool) -> None: with HTTPConnectionPool(self.host, self.port) as pool: - with pytest.raises(ValueError) as e: + with pytest.raises( + ValueError, + match="urllib3.util.SKIP_HEADER only supports 'Accept-Encoding', 'Host', 'User-Agent'", + ) as e: pool.request( "GET", "/headers", headers={header: SKIP_HEADER}, chunked=chunked ) - assert ( - str(e.value) - == "urllib3.util.SKIP_HEADER only supports 'Accept-Encoding', 'Host', 'User-Agent'" - ) - # Ensure that the error message stays up to date with 'SKIP_HEADER_SUPPORTED_HEADERS' assert all( ("'" + header.title() + "'") in str(e.value) @@ -1021,7 +1025,12 @@ def test_skip_header_non_supported(self, header, chunked): @pytest.mark.parametrize("chunked", [True, False]) @pytest.mark.parametrize("pool_request", [True, False]) @pytest.mark.parametrize("header_type", [dict, HTTPHeaderDict]) - def test_headers_not_modified_by_request(self, chunked, pool_request, header_type): + def test_headers_not_modified_by_request( + self, + chunked: bool, + pool_request: bool, + header_type: type[dict[str, str] | HTTPHeaderDict], + ) -> None: # Test that the .request*() methods of ConnectionPool and HTTPConnection # don't modify the given 'headers' structure, instead they should # make their own internal copies at request time. @@ -1034,75 +1043,72 @@ def test_headers_not_modified_by_request(self, chunked, pool_request, header_typ pool.request("GET", "/headers", chunked=chunked) else: conn = pool._get_conn() - if chunked: - conn.request_chunked("GET", "/headers") - else: - conn.request("GET", "/headers") + conn.request("GET", "/headers", chunked=chunked) + conn.getresponse().close() + conn.close() assert pool.headers == {"key": "val"} - assert isinstance(pool.headers, header_type) + assert type(pool.headers) is header_type with HTTPConnectionPool(self.host, self.port) as pool: if pool_request: pool.request("GET", "/headers", headers=headers, chunked=chunked) else: conn = pool._get_conn() - if chunked: - conn.request_chunked("GET", "/headers", headers=headers) - else: - conn.request("GET", "/headers", headers=headers) + conn.request("GET", "/headers", headers=headers, chunked=chunked) + conn.getresponse().close() + conn.close() assert headers == {"key": "val"} - def test_bytes_header(self): + def test_request_chunked_is_deprecated( + self, + ) -> None: + with HTTPConnectionPool(self.host, self.port) as pool: + conn = pool._get_conn() + + with pytest.warns(DeprecationWarning) as w: + conn.request_chunked("GET", "/headers") # type: ignore[attr-defined] + assert len(w) == 1 and str(w[0].message) == ( + "HTTPConnection.request_chunked() is deprecated and will be removed in urllib3 v2.1.0. " + "Instead use HTTPConnection.request(..., chunked=True)." + ) + + resp = conn.getresponse() + assert resp.status == 200 + assert resp.json()["Transfer-Encoding"] == "chunked" + conn.close() + + def test_bytes_header(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: - headers = {"User-Agent": b"test header"} + headers = {"User-Agent": "test header"} r = pool.request("GET", "/headers", headers=headers) - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() assert "User-Agent" in request_headers assert request_headers["User-Agent"] == "test header" @pytest.mark.parametrize( - "user_agent", [u"Schönefeld/1.18.0", u"Schönefeld/1.18.0".encode("iso-8859-1")] + "user_agent", ["Schönefeld/1.18.0", "Schönefeld/1.18.0".encode("iso-8859-1")] ) - def test_user_agent_non_ascii_user_agent(self, user_agent): - if six.PY2 and not isinstance(user_agent, str): - pytest.skip( - "Python 2 raises UnicodeEncodeError when passed a unicode header" - ) - + def test_user_agent_non_ascii_user_agent(self, user_agent: str) -> None: with HTTPConnectionPool(self.host, self.port, retries=False) as pool: r = pool.urlopen( "GET", "/headers", headers={"User-Agent": user_agent}, ) - request_headers = json.loads(r.data.decode("utf8")) + request_headers = r.json() assert "User-Agent" in request_headers - assert request_headers["User-Agent"] == u"Schönefeld/1.18.0" - - @onlyPy2 - def test_user_agent_non_ascii_fails_on_python_2(self): - with HTTPConnectionPool(self.host, self.port, retries=False) as pool: - with pytest.raises(UnicodeEncodeError) as e: - pool.urlopen( - "GET", - "/headers", - headers={"User-Agent": u"Schönefeld/1.18.0"}, - ) - assert str(e.value) == ( - "'ascii' codec can't encode character u'\\xf6' in " - "position 3: ordinal not in range(128)" - ) + assert request_headers["User-Agent"] == "Schönefeld/1.18.0" -class TestRetry(HTTPDummyServerTestCase): - def test_max_retry(self): +class TestRetry(HypercornDummyServerTestCase): + def test_max_retry(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: with pytest.raises(MaxRetryError): pool.request("GET", "/redirect", fields={"target": "/"}, retries=0) - def test_disabled_retry(self): + def test_disabled_retry(self) -> None: """Disabled retries should disable redirect handling.""" with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/redirect", fields={"target": "/"}, retries=False) @@ -1119,11 +1125,11 @@ def test_disabled_retry(self): with HTTPConnectionPool( "thishostdoesnotexist.invalid", self.port, timeout=0.001 ) as pool: - with pytest.raises(NewConnectionError): + with pytest.raises(NameResolutionError): pool.request("GET", "/test", retries=False) - def test_read_retries(self): - """Should retry for status codes in the whitelist""" + def test_read_retries(self) -> None: + """Should retry for status codes in the forcelist""" with HTTPConnectionPool(self.host, self.port) as pool: retry = Retry(read=1, status_forcelist=[418]) resp = pool.request( @@ -1134,8 +1140,8 @@ def test_read_retries(self): ) assert resp.status == 200 - def test_read_total_retries(self): - """HTTP response w/ status code in the whitelist should be retried""" + def test_read_total_retries(self) -> None: + """HTTP response w/ status code in the forcelist should be retried""" with HTTPConnectionPool(self.host, self.port) as pool: headers = {"test-name": "test_read_total_retries"} retry = Retry(total=1, status_forcelist=[418]) @@ -1144,47 +1150,49 @@ def test_read_total_retries(self): ) assert resp.status == 200 - def test_retries_wrong_whitelist(self): - """HTTP response w/ status code not in whitelist shouldn't be retried""" + def test_retries_wrong_forcelist(self) -> None: + """HTTP response w/ status code not in forcelist shouldn't be retried""" with HTTPConnectionPool(self.host, self.port) as pool: retry = Retry(total=1, status_forcelist=[202]) resp = pool.request( "GET", "/successful_retry", - headers={"test-name": "test_wrong_whitelist"}, + headers={"test-name": "test_wrong_forcelist"}, retries=retry, ) assert resp.status == 418 - def test_default_method_whitelist_retried(self): - """urllib3 should retry methods in the default method whitelist""" + def test_default_method_forcelist_retried(self) -> None: + """urllib3 should retry methods in the default method forcelist""" with HTTPConnectionPool(self.host, self.port) as pool: retry = Retry(total=1, status_forcelist=[418]) resp = pool.request( "OPTIONS", "/successful_retry", - headers={"test-name": "test_default_whitelist"}, + headers={"test-name": "test_default_forcelist"}, retries=retry, ) assert resp.status == 200 - def test_retries_wrong_method_list(self): - """Method not in our whitelist should not be retried, even if code matches""" + def test_retries_wrong_method_list(self) -> None: + """Method not in our allowed list should not be retried, even if code matches""" with HTTPConnectionPool(self.host, self.port) as pool: - headers = {"test-name": "test_wrong_method_whitelist"} - retry = Retry(total=1, status_forcelist=[418], method_whitelist=["POST"]) + headers = {"test-name": "test_wrong_allowed_method"} + retry = Retry(total=1, status_forcelist=[418], allowed_methods=["POST"]) resp = pool.request( "GET", "/successful_retry", headers=headers, retries=retry ) assert resp.status == 418 - def test_read_retries_unsuccessful(self): + def test_read_retries_unsuccessful( + self, + ) -> None: with HTTPConnectionPool(self.host, self.port) as pool: headers = {"test-name": "test_read_retries_unsuccessful"} resp = pool.request("GET", "/successful_retry", headers=headers, retries=1) assert resp.status == 418 - def test_retry_reuse_safe(self): + def test_retry_reuse_safe(self) -> None: """It should be possible to reuse a Retry object across requests""" with HTTPConnectionPool(self.host, self.port) as pool: headers = {"test-name": "test_retry_safe"} @@ -1200,7 +1208,7 @@ def test_retry_reuse_safe(self): ) assert resp.status == 200 - def test_retry_return_in_response(self): + def test_retry_return_in_response(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: headers = {"test-name": "test_retry_return_in_response"} retry = Retry(total=2, status_forcelist=[418]) @@ -1208,20 +1216,22 @@ def test_retry_return_in_response(self): "GET", "/successful_retry", headers=headers, retries=retry ) assert resp.status == 200 + assert resp.retries is not None assert resp.retries.total == 1 assert resp.retries.history == ( RequestHistory("GET", "/successful_retry", None, 418, None), ) - def test_retry_redirect_history(self): + def test_retry_redirect_history(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: resp = pool.request("GET", "/redirect", fields={"target": "/"}) assert resp.status == 200 + assert resp.retries is not None assert resp.retries.history == ( RequestHistory("GET", "/redirect?target=%2F", None, 303, "/"), ) - def test_multi_redirect_history(self): + def test_multi_redirect_history(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request( "GET", @@ -1230,6 +1240,7 @@ def test_multi_redirect_history(self): redirect=False, ) assert r.status == 303 + assert r.retries is not None assert r.retries.history == tuple() with HTTPConnectionPool(self.host, self.port) as pool: @@ -1249,6 +1260,7 @@ def test_multi_redirect_history(self): (307, "/multi_redirect?redirect_codes=302,200"), (302, "/multi_redirect?redirect_codes=200"), ] + assert r.retries is not None actual = [ (history.status, history.redirect_location) for history in r.retries.history @@ -1256,8 +1268,8 @@ def test_multi_redirect_history(self): assert actual == expected -class TestRetryAfter(HTTPDummyServerTestCase): - def test_retry_after(self): +class TestRetryAfter(HypercornDummyServerTestCase): + def test_retry_after(self) -> None: # Request twice in a second to get a 429 response. with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request( @@ -1315,11 +1327,12 @@ def test_retry_after(self): ) assert r.status == 418 - def test_redirect_after(self): + def test_redirect_after(self) -> None: with HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/redirect_after", retries=False) assert r.status == 303 + # Real timestamps are needed for this test t = time.time() r = pool.request("GET", "/redirect_after") assert r.status == 200 @@ -1342,10 +1355,10 @@ def test_redirect_after(self): assert delta < 1 -class TestFileBodiesOnRetryOrRedirect(HTTPDummyServerTestCase): - def test_retries_put_filehandle(self): +class TestFileBodiesOnRetryOrRedirect(HypercornDummyServerTestCase): + def test_retries_put_filehandle(self) -> None: """HTTP PUT retry with a file-like object should not timeout""" - with HTTPConnectionPool(self.host, self.port, timeout=0.1) as pool: + with HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT) as pool: retry = Retry(total=3, status_forcelist=[418]) # httplib reads in 8k chunks; use a larger content length content_length = 65535 @@ -1366,9 +1379,9 @@ def test_retries_put_filehandle(self): ) assert resp.status == 200 - def test_redirect_put_file(self): + def test_redirect_put_file(self) -> None: """PUT with file object should work with a redirection response""" - with HTTPConnectionPool(self.host, self.port, timeout=0.1) as pool: + with HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT) as pool: retry = Retry(total=3, status_forcelist=[418]) # httplib reads in 8k chunks; use a larger content length content_length = 65535 @@ -1391,26 +1404,27 @@ def test_redirect_put_file(self): assert resp.status == 200 assert resp.data == data - def test_redirect_with_failed_tell(self): + def test_redirect_with_failed_tell(self) -> None: """Abort request if failed to get a position from tell()""" class BadTellObject(io.BytesIO): - def tell(self): - raise IOError + def tell(self) -> typing.NoReturn: + raise OSError body = BadTellObject(b"the data") url = "/redirect?target=/successful_retry" # httplib uses fileno if Content-Length isn't supplied, # which is unsupported by BytesIO. headers = {"Content-Length": "8"} - with HTTPConnectionPool(self.host, self.port, timeout=0.1) as pool: - with pytest.raises(UnrewindableBodyError) as e: + with HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT) as pool: + with pytest.raises( + UnrewindableBodyError, match="Unable to record file position for" + ): pool.urlopen("PUT", url, headers=headers, body=body) - assert "Unable to record file position for" in str(e.value) -class TestRetryPoolSize(HTTPDummyServerTestCase): - def test_pool_size_retry(self): +class TestRetryPoolSize(HypercornDummyServerTestCase): + def test_pool_size_retry(self) -> None: retries = Retry(total=1, raise_on_status=False, status_forcelist=[404]) with HTTPConnectionPool( self.host, self.port, maxsize=10, retries=retries, block=True @@ -1419,8 +1433,8 @@ def test_pool_size_retry(self): assert pool.num_connections == 1 -class TestRedirectPoolSize(HTTPDummyServerTestCase): - def test_pool_size_redirect(self): +class TestRedirectPoolSize(HypercornDummyServerTestCase): + def test_pool_size_redirect(self) -> None: retries = Retry( total=1, raise_on_status=False, status_forcelist=[404], redirect=True ) diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index f37f8e6..162d089 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -1,64 +1,53 @@ +from __future__ import annotations + +import concurrent.futures +import contextlib import datetime -import json -import logging import os.path import shutil import ssl -import sys import tempfile +import time +import typing import warnings +from pathlib import Path from test import ( LONG_TIMEOUT, SHORT_TIMEOUT, TARPIT_HOST, - notSecureTransport, - onlyPy279OrNewer, requires_network, - requires_ssl_context_keyfile_password, resolvesLocalhostFQDN, ) +from test.conftest import ServerConfig +from unittest import mock -import mock import pytest import trustme +import urllib3.http2 +import urllib3.http2.probe as http2_probe import urllib3.util as util -from dummyserver.server import ( +import urllib3.util.ssl_ +from dummyserver.socketserver import ( DEFAULT_CA, DEFAULT_CA_KEY, DEFAULT_CERTS, encrypt_key_pem, ) -from dummyserver.testcase import HTTPSDummyServerTestCase +from dummyserver.testcase import HTTPSHypercornDummyServerTestCase from urllib3 import HTTPSConnectionPool -from urllib3.connection import RECENT_DATE, VerifiedHTTPSConnection +from urllib3.connection import RECENT_DATE, HTTPSConnection, VerifiedHTTPSConnection from urllib3.exceptions import ( ConnectTimeoutError, - InsecurePlatformWarning, InsecureRequestWarning, MaxRetryError, ProtocolError, SSLError, SystemTimeWarning, ) -from urllib3.packages import six +from urllib3.util.ssl_match_hostname import CertificateError from urllib3.util.timeout import Timeout -from .. import has_alpn - -# Retry failed tests -pytestmark = pytest.mark.flaky - -ResourceWarning = getattr( - six.moves.builtins, "ResourceWarning", type("ResourceWarning", (), {}) -) - - -log = logging.getLogger("urllib3.connectionpool") -log.setLevel(logging.NOTSET) -log.addHandler(logging.StreamHandler(sys.stdout)) - - TLSv1_CERTS = DEFAULT_CERTS.copy() TLSv1_CERTS["ssl_version"] = getattr(ssl, "PROTOCOL_TLSv1", None) @@ -79,15 +68,39 @@ CLIENT_CERT = CLIENT_INTERMEDIATE_PEM -class TestHTTPS(HTTPSDummyServerTestCase): - tls_protocol_name = None +class BaseTestHTTPS(HTTPSHypercornDummyServerTestCase): + tls_protocol_name: str | None = None - def tls_protocol_deprecated(self): + def tls_protocol_not_default(self) -> bool: return self.tls_protocol_name in {"TLSv1", "TLSv1.1"} + def tls_version(self) -> ssl.TLSVersion: + if self.tls_protocol_name is None: + return pytest.skip("Skipping base test class") + try: + from ssl import TLSVersion + except ImportError: + return pytest.skip("ssl.TLSVersion isn't available") + return TLSVersion[self.tls_protocol_name.replace(".", "_")] + + def ssl_version(self) -> int: + if self.tls_protocol_name is None: + return pytest.skip("Skipping base test class") + + if self.tls_protocol_name == "TLSv1.3" and ssl.HAS_TLSv1_3: + return ssl.PROTOCOL_TLS_CLIENT + if self.tls_protocol_name == "TLSv1.2" and ssl.HAS_TLSv1_2: + return ssl.PROTOCOL_TLSv1_2 + if self.tls_protocol_name == "TLSv1.1" and ssl.HAS_TLSv1_1: + return ssl.PROTOCOL_TLSv1_1 + if self.tls_protocol_name == "TLSv1" and ssl.HAS_TLSv1: + return ssl.PROTOCOL_TLSv1 + else: + return pytest.skip(f"{self.tls_protocol_name} isn't available") + @classmethod - def setup_class(cls): - super(TestHTTPS, cls).setup_class() + def setup_class(cls) -> None: + super().setup_class() cls.certs_dir = tempfile.mkdtemp() # Start from existing root CA as we don't want to change the server certificate yet @@ -101,7 +114,7 @@ def setup_class(cls): # client cert chain intermediate_ca = root_ca.create_child_ca() - cert = intermediate_ca.issue_cert(u"example.com") + cert = intermediate_ca.issue_cert("example.com") encrypted_key = encrypt_key_pem(cert.private_key_pem, b"letmein") cert.private_key_pem.write_to_path( @@ -120,27 +133,39 @@ def setup_class(cls): ) @classmethod - def teardown_class(cls): - super(TestHTTPS, cls).teardown_class() + def teardown_class(cls) -> None: + super().teardown_class() shutil.rmtree(cls.certs_dir) - def test_simple(self): + def test_simple(self, http_version: str) -> None: with HTTPSConnectionPool( - self.host, self.port, ca_certs=DEFAULT_CA + self.host, + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: r = https_pool.request("GET", "/") assert r.status == 200, r.data + assert r.headers["server"] == f"hypercorn-{http_version}" + assert r.data == b"Dummy server!" - @resolvesLocalhostFQDN - def test_dotted_fqdn(self): + def test_default_port(self) -> None: + conn = HTTPSConnection(self.host, port=None) + assert conn.port == 443 + + @resolvesLocalhostFQDN() + def test_dotted_fqdn(self) -> None: with HTTPSConnectionPool( - self.host + ".", self.port, ca_certs=DEFAULT_CA + self.host + ".", + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as pool: r = pool.request("GET", "/") assert r.status == 200, r.data - def test_client_intermediate(self): + def test_client_intermediate(self) -> None: """Check that certificate chains work well with client certs We generate an intermediate CA from the root CA, and issue a client certificate @@ -154,12 +179,13 @@ def test_client_intermediate(self): key_file=os.path.join(self.certs_dir, CLIENT_INTERMEDIATE_KEY), cert_file=os.path.join(self.certs_dir, CLIENT_INTERMEDIATE_PEM), ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: r = https_pool.request("GET", "/certificate") - subject = json.loads(r.data.decode("utf-8")) + subject = r.json() assert subject["organizationalUnitName"].startswith("Testing cert") - def test_client_no_intermediate(self): + def test_client_no_intermediate(self) -> None: """Check that missing links in certificate chains indeed break The only difference with test_client_intermediate is that we don't send the @@ -171,12 +197,12 @@ def test_client_no_intermediate(self): cert_file=os.path.join(self.certs_dir, CLIENT_NO_INTERMEDIATE_PEM), key_file=os.path.join(self.certs_dir, CLIENT_INTERMEDIATE_KEY), ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: with pytest.raises((SSLError, ProtocolError)): https_pool.request("GET", "/certificate", retries=False) - @requires_ssl_context_keyfile_password - def test_client_key_password(self): + def test_client_key_password(self) -> None: with HTTPSConnectionPool( self.host, self.port, @@ -184,184 +210,157 @@ def test_client_key_password(self): key_file=os.path.join(self.certs_dir, PASSWORD_CLIENT_KEYFILE), cert_file=os.path.join(self.certs_dir, CLIENT_CERT), key_password="letmein", + ssl_minimum_version=self.tls_version(), ) as https_pool: r = https_pool.request("GET", "/certificate") - subject = json.loads(r.data.decode("utf-8")) + subject = r.json() assert subject["organizationalUnitName"].startswith("Testing cert") - @requires_ssl_context_keyfile_password - def test_client_encrypted_key_requires_password(self): + def test_client_encrypted_key_requires_password(self) -> None: with HTTPSConnectionPool( self.host, self.port, key_file=os.path.join(self.certs_dir, PASSWORD_CLIENT_KEYFILE), cert_file=os.path.join(self.certs_dir, CLIENT_CERT), key_password=None, + ssl_minimum_version=self.tls_version(), ) as https_pool: - with pytest.raises(MaxRetryError) as e: + with pytest.raises(MaxRetryError, match="password is required") as e: https_pool.request("GET", "/certificate") - assert "password is required" in str(e.value) - assert isinstance(e.value.reason, SSLError) + assert type(e.value.reason) is SSLError - def test_verified(self): + def test_verified(self) -> None: with HTTPSConnectionPool( - self.host, self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA + self.host, + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection with warnings.catch_warnings(record=True) as w: r = https_pool.request("GET", "/") assert r.status == 200 - # If we're using a deprecated TLS version we can remove 'DeprecationWarning' - if self.tls_protocol_deprecated(): - w = [x for x in w if x.category != DeprecationWarning] - - # Modern versions of Python, or systems using PyOpenSSL, don't - # emit warnings. - if ( - sys.version_info >= (2, 7, 9) - or util.IS_PYOPENSSL - or util.IS_SECURETRANSPORT - ): - assert w == [] - else: - assert len(w) > 1 - assert any(x.category == InsecureRequestWarning for x in w) + assert [str(wm) for wm in w] == [] - def test_verified_with_context(self): - ctx = util.ssl_.create_urllib3_context(cert_reqs=ssl.CERT_REQUIRED) + def test_verified_with_context(self) -> None: + ctx = util.ssl_.create_urllib3_context( + cert_reqs=ssl.CERT_REQUIRED, ssl_minimum_version=self.tls_version() + ) ctx.load_verify_locations(cafile=DEFAULT_CA) with HTTPSConnectionPool(self.host, self.port, ssl_context=ctx) as https_pool: - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection with mock.patch("warnings.warn") as warn: r = https_pool.request("GET", "/") assert r.status == 200 + assert not warn.called, warn.call_args_list - # Modern versions of Python, or systems using PyOpenSSL, don't - # emit warnings. - if ( - sys.version_info >= (2, 7, 9) - or util.IS_PYOPENSSL - or util.IS_SECURETRANSPORT - ): - assert not warn.called, warn.call_args_list - else: - assert warn.called - if util.HAS_SNI: - call = warn.call_args_list[0] - else: - call = warn.call_args_list[1] - error = call[0][1] - assert error == InsecurePlatformWarning - - def test_context_combines_with_ca_certs(self): - ctx = util.ssl_.create_urllib3_context(cert_reqs=ssl.CERT_REQUIRED) + def test_context_combines_with_ca_certs(self) -> None: + ctx = util.ssl_.create_urllib3_context( + cert_reqs=ssl.CERT_REQUIRED, ssl_minimum_version=self.tls_version() + ) with HTTPSConnectionPool( self.host, self.port, ca_certs=DEFAULT_CA, ssl_context=ctx ) as https_pool: - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection with mock.patch("warnings.warn") as warn: r = https_pool.request("GET", "/") assert r.status == 200 + assert not warn.called, warn.call_args_list - # Modern versions of Python, or systems using PyOpenSSL, don't - # emit warnings. - if ( - sys.version_info >= (2, 7, 9) - or util.IS_PYOPENSSL - or util.IS_SECURETRANSPORT - ): - assert not warn.called, warn.call_args_list - else: - assert warn.called - if util.HAS_SNI: - call = warn.call_args_list[0] - else: - call = warn.call_args_list[1] - error = call[0][1] - assert error == InsecurePlatformWarning - - @onlyPy279OrNewer - @notSecureTransport # SecureTransport does not support cert directories - def test_ca_dir_verified(self, tmpdir): + def test_ca_dir_verified(self, tmp_path: Path) -> None: # OpenSSL looks up certificates by the hash for their name, see c_rehash # TODO infer the bytes using `cryptography.x509.Name.public_bytes`. # https://github.com/pyca/cryptography/pull/3236 - shutil.copyfile(DEFAULT_CA, str(tmpdir / "81deb5f7.0")) + shutil.copyfile(DEFAULT_CA, str(tmp_path / "81deb5f7.0")) with HTTPSConnectionPool( - self.host, self.port, cert_reqs="CERT_REQUIRED", ca_cert_dir=str(tmpdir) + self.host, + self.port, + cert_reqs="CERT_REQUIRED", + ca_cert_dir=str(tmp_path), + ssl_minimum_version=self.tls_version(), ) as https_pool: - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection with warnings.catch_warnings(record=True) as w: r = https_pool.request("GET", "/") assert r.status == 200 - # If we're using a deprecated TLS version we can remove 'DeprecationWarning' - if self.tls_protocol_deprecated(): - w = [x for x in w if x.category != DeprecationWarning] + assert [str(wm) for wm in w] == [] - assert w == [] - - def test_invalid_common_name(self): + def test_invalid_common_name(self) -> None: with HTTPSConnectionPool( - "127.0.0.1", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA + "127.0.0.1", + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: with pytest.raises(MaxRetryError) as e: - https_pool.request("GET", "/") - assert isinstance(e.value.reason, SSLError) + https_pool.request("GET", "/", retries=0) + assert type(e.value.reason) is SSLError assert "doesn't match" in str( e.value.reason ) or "certificate verify failed" in str(e.value.reason) - def test_verified_with_bad_ca_certs(self): + def test_verified_with_bad_ca_certs(self) -> None: with HTTPSConnectionPool( - self.host, self.port, cert_reqs="CERT_REQUIRED", ca_certs=self.bad_ca_path + self.host, + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=self.bad_ca_path, + ssl_minimum_version=self.tls_version(), ) as https_pool: with pytest.raises(MaxRetryError) as e: https_pool.request("GET", "/") - assert isinstance(e.value.reason, SSLError) - assert "certificate verify failed" in str(e.value.reason), ( - "Expected 'certificate verify failed', instead got: %r" % e.value.reason - ) + assert type(e.value.reason) is SSLError + assert ( + "certificate verify failed" in str(e.value.reason) + # PyPy is more specific + or "self signed certificate in certificate chain" in str(e.value.reason) + ), f"Expected 'certificate verify failed', instead got: {e.value.reason!r}" - def test_wrap_socket_failure_resource_leak(self): + def test_wrap_socket_failure_resource_leak(self) -> None: with HTTPSConnectionPool( self.host, self.port, cert_reqs="CERT_REQUIRED", ca_certs=self.bad_ca_path, + ssl_minimum_version=self.tls_version(), ) as https_pool: - conn = https_pool._get_conn() - try: + with contextlib.closing(https_pool._get_conn()) as conn: with pytest.raises(ssl.SSLError): conn.connect() - assert conn.sock - finally: - conn.close() + assert conn.sock is not None # type: ignore[attr-defined] - def test_verified_without_ca_certs(self): + def test_verified_without_ca_certs(self) -> None: # default is cert_reqs=None which is ssl.CERT_NONE with HTTPSConnectionPool( - self.host, self.port, cert_reqs="CERT_REQUIRED" + self.host, + self.port, + cert_reqs="CERT_REQUIRED", + ssl_minimum_version=self.tls_version(), ) as https_pool: with pytest.raises(MaxRetryError) as e: https_pool.request("GET", "/") - assert isinstance(e.value.reason, SSLError) + assert type(e.value.reason) is SSLError # there is a different error message depending on whether or # not pyopenssl is injected assert ( "No root certificates specified" in str(e.value.reason) + # PyPy is more specific + or "self signed certificate in certificate chain" in str(e.value.reason) # PyPy sometimes uses all-caps here or "certificate verify failed" in str(e.value.reason).lower() or "invalid certificate chain" in str(e.value.reason) @@ -372,18 +371,22 @@ def test_verified_without_ca_certs(self): "instead got: %r" % e.value.reason ) - def test_no_ssl(self): + def test_no_ssl(self) -> None: with HTTPSConnectionPool(self.host, self.port) as pool: - pool.ConnectionCls = None - with pytest.raises(SSLError): + pool.ConnectionCls = None # type: ignore[assignment] + with pytest.raises(ImportError): pool._new_conn() - with pytest.raises(MaxRetryError) as cm: + with pytest.raises(ImportError): pool.request("GET", "/", retries=0) - assert isinstance(cm.value.reason, SSLError) - def test_unverified_ssl(self): + def test_unverified_ssl(self) -> None: """Test that bare HTTPSConnection can connect, make requests""" - with HTTPSConnectionPool(self.host, self.port, cert_reqs=ssl.CERT_NONE) as pool: + with HTTPSConnectionPool( + self.host, + self.port, + cert_reqs=ssl.CERT_NONE, + ssl_minimum_version=self.tls_version(), + ) as pool: with mock.patch("warnings.warn") as warn: r = pool.request("GET", "/") assert r.status == 200 @@ -395,9 +398,13 @@ def test_unverified_ssl(self): calls = warn.call_args_list assert InsecureRequestWarning in [x[0][1] for x in calls] - def test_ssl_unverified_with_ca_certs(self): + def test_ssl_unverified_with_ca_certs(self) -> None: with HTTPSConnectionPool( - self.host, self.port, cert_reqs="CERT_NONE", ca_certs=self.bad_ca_path + self.host, + self.port, + cert_reqs="CERT_NONE", + ca_certs=self.bad_ca_path, + ssl_minimum_version=self.tls_version(), ) as pool: with mock.patch("warnings.warn") as warn: r = pool.request("GET", "/") @@ -409,43 +416,39 @@ def test_ssl_unverified_with_ca_certs(self): # warnings, which we want to ignore here. calls = warn.call_args_list - # If we're using a deprecated TLS version we can remove 'DeprecationWarning' - if self.tls_protocol_deprecated(): - calls = [call for call in calls if call[0][1] != DeprecationWarning] - - if ( - sys.version_info >= (2, 7, 9) - or util.IS_PYOPENSSL - or util.IS_SECURETRANSPORT - ): - category = calls[0][0][1] - elif util.HAS_SNI: - category = calls[1][0][1] - else: - category = calls[2][0][1] + category = calls[0][0][1] assert category == InsecureRequestWarning - def test_assert_hostname_false(self): + def test_assert_hostname_false(self) -> None: with HTTPSConnectionPool( - "localhost", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA + "localhost", + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: https_pool.assert_hostname = False https_pool.request("GET", "/") - def test_assert_specific_hostname(self): + def test_assert_specific_hostname(self) -> None: with HTTPSConnectionPool( - "localhost", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA + "localhost", + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: https_pool.assert_hostname = "localhost" https_pool.request("GET", "/") - def test_server_hostname(self): + def test_server_hostname(self) -> None: with HTTPSConnectionPool( "127.0.0.1", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA, server_hostname="localhost", + ssl_minimum_version=self.tls_version(), ) as https_pool: conn = https_pool._new_conn() conn.request("GET", "/") @@ -454,54 +457,73 @@ def test_server_hostname(self): # pyopenssl doesn't let you pull the server_hostname back off the # socket, so only add this assertion if the attribute is there (i.e. # the python ssl module). - if hasattr(conn.sock, "server_hostname"): - assert conn.sock.server_hostname == "localhost" + if hasattr(conn.sock, "server_hostname"): # type: ignore[attr-defined] + assert conn.sock.server_hostname == "localhost" # type: ignore[attr-defined] + conn.getresponse().close() + conn.close() - def test_assert_fingerprint_md5(self): + def test_assert_fingerprint_md5(self) -> None: with HTTPSConnectionPool( - "localhost", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA + "localhost", + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + assert_fingerprint=("55:39:BF:70:05:12:43:FA:1F:D1:BF:4E:E8:1B:07:1D"), + ssl_minimum_version=self.tls_version(), ) as https_pool: - https_pool.assert_fingerprint = ( - "55:39:BF:70:05:12:43:FA:1F:D1:BF:4E:E8:1B:07:1D" - ) - https_pool.request("GET", "/") - def test_assert_fingerprint_sha1(self): + def test_assert_fingerprint_sha1(self) -> None: with HTTPSConnectionPool( - "localhost", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA - ) as https_pool: - https_pool.assert_fingerprint = ( + "localhost", + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + assert_fingerprint=( "72:8B:55:4C:9A:FC:1E:88:A1:1C:AD:1B:B2:E7:CC:3E:DB:C8:F9:8A" - ) + ), + ssl_minimum_version=self.tls_version(), + ) as https_pool: https_pool.request("GET", "/") - def test_assert_fingerprint_sha256(self): + def test_assert_fingerprint_sha256(self) -> None: with HTTPSConnectionPool( - "localhost", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA - ) as https_pool: - https_pool.assert_fingerprint = ( + "localhost", + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + assert_fingerprint=( "E3:59:8E:69:FF:C5:9F:C7:88:87:44:58:22:7F:90:8D:D9:BC:12:C4:90:79:D5:" "DC:A8:5D:4F:60:40:1E:A6:D2" - ) + ), + ssl_minimum_version=self.tls_version(), + ) as https_pool: https_pool.request("GET", "/") - def test_assert_invalid_fingerprint(self): - def _test_request(pool): + def test_assert_invalid_fingerprint(self) -> None: + def _test_request(pool: HTTPSConnectionPool) -> SSLError: with pytest.raises(MaxRetryError) as cm: pool.request("GET", "/", retries=0) - assert isinstance(cm.value.reason, SSLError) + assert type(cm.value.reason) is SSLError return cm.value.reason with HTTPSConnectionPool( - self.host, self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA + self.host, + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: - https_pool.assert_fingerprint = ( - "AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA" + "AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA" ) e = _test_request(https_pool) - assert "Fingerprints did not match." in str(e) + expected = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + got = "728b554c9afc1e88a11cad1bb2e7cc3edbc8f98a" + assert ( + str(e) + == f'Fingerprints did not match. Expected "{expected}", got "{got}"' + ) # Uneven length https_pool.assert_fingerprint = "AA:A" @@ -513,43 +535,47 @@ def _test_request(pool): e = _test_request(https_pool) assert "Fingerprint of invalid length:" in str(e) - def test_verify_none_and_bad_fingerprint(self): + def test_verify_none_and_bad_fingerprint(self) -> None: with HTTPSConnectionPool( - "127.0.0.1", self.port, cert_reqs="CERT_NONE", ca_certs=self.bad_ca_path + "127.0.0.1", + self.port, + cert_reqs="CERT_NONE", + assert_hostname=False, + assert_fingerprint=( + "AA:8B:55:4C:9A:FC:1E:88:A1:1C:AD:1B:B2:E7:CC:3E:DB:C8:F9:8A" + ), ) as https_pool: - https_pool.assert_fingerprint = ( - "AA:AA:AA:AA:AA:AAAA:AA:AAAA:AA:AA:AA:AA:AA:AA:AA:AA:AA:AA" - ) with pytest.raises(MaxRetryError) as cm: https_pool.request("GET", "/", retries=0) - assert isinstance(cm.value.reason, SSLError) + assert type(cm.value.reason) is SSLError - def test_verify_none_and_good_fingerprint(self): + def test_verify_none_and_good_fingerprint(self) -> None: with HTTPSConnectionPool( - "127.0.0.1", self.port, cert_reqs="CERT_NONE", ca_certs=self.bad_ca_path - ) as https_pool: - https_pool.assert_fingerprint = ( + "127.0.0.1", + self.port, + cert_reqs="CERT_NONE", + assert_hostname=False, + assert_fingerprint=( "72:8B:55:4C:9A:FC:1E:88:A1:1C:AD:1B:B2:E7:CC:3E:DB:C8:F9:8A" - ) + ), + ) as https_pool: https_pool.request("GET", "/") - @notSecureTransport - def test_good_fingerprint_and_hostname_mismatch(self): - # This test doesn't run with SecureTransport because we don't turn off - # hostname validation without turning off all validation, which this - # test doesn't do (deliberately). We should revisit this if we make - # new decisions. + def test_good_fingerprint_and_hostname_mismatch(self) -> None: with HTTPSConnectionPool( - "127.0.0.1", self.port, cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA - ) as https_pool: - https_pool.assert_fingerprint = ( + "127.0.0.1", + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + assert_fingerprint=( "72:8B:55:4C:9A:FC:1E:88:A1:1C:AD:1B:B2:E7:CC:3E:DB:C8:F9:8A" - ) + ), + ssl_minimum_version=self.tls_version(), + ) as https_pool: https_pool.request("GET", "/") - @requires_network - def test_https_timeout(self): - + @requires_network() + def test_https_timeout(self) -> None: timeout = Timeout(total=None, connect=SHORT_TIMEOUT) with HTTPSConnectionPool( TARPIT_HOST, @@ -557,6 +583,7 @@ def test_https_timeout(self): timeout=timeout, retries=False, cert_reqs="CERT_REQUIRED", + ssl_minimum_version=self.tls_version(), ) as https_pool: with pytest.raises(ConnectTimeoutError): https_pool.request("GET", "/") @@ -568,35 +595,56 @@ def test_https_timeout(self): timeout=timeout, retries=False, cert_reqs="CERT_REQUIRED", - ) as https_pool: - https_pool.ca_certs = DEFAULT_CA - https_pool.assert_fingerprint = ( + ca_certs=DEFAULT_CA, + assert_fingerprint=( "72:8B:55:4C:9A:FC:1E:88:A1:1C:AD:1B:B2:E7:CC:3E:DB:C8:F9:8A" - ) + ), + ssl_minimum_version=self.tls_version(), + ) as https_pool: + # TODO This was removed in https://github.com/urllib3/urllib3/pull/703/files + # We need to put something back or remove this block. + pass timeout = Timeout(total=None) with HTTPSConnectionPool( - self.host, self.port, timeout=timeout, cert_reqs="CERT_NONE" + self.host, + self.port, + timeout=timeout, + cert_reqs="CERT_NONE", + ssl_minimum_version=self.tls_version(), ) as https_pool: - https_pool.request("GET", "/") + with pytest.warns(InsecureRequestWarning): + https_pool.request("GET", "/") - def test_tunnel(self): + def test_tunnel(self, http_version: str) -> None: """test the _tunnel behavior""" timeout = Timeout(total=None) with HTTPSConnectionPool( - self.host, self.port, timeout=timeout, cert_reqs="CERT_NONE" + self.host, + self.port, + timeout=timeout, + cert_reqs="CERT_NONE", + ssl_minimum_version=self.tls_version(), ) as https_pool: - conn = https_pool._new_conn() - try: - conn.set_tunnel(self.host, self.port) - conn._tunnel = mock.Mock() - https_pool._make_request(conn, "GET", "/") - conn._tunnel.assert_called_once_with() - finally: - conn.close() - - @requires_network - def test_enhanced_timeout(self): + with contextlib.closing(https_pool._new_conn()) as conn: + if http_version == "h2": + with pytest.raises(NotImplementedError) as e: + conn.set_tunnel(self.host, self.port) + assert ( + str(e.value) + == "HTTP/2 does not support setting up a tunnel through a proxy" + ) + else: + conn.set_tunnel(self.host, self.port) + with mock.patch.object( + conn, "_tunnel", create=True, return_value=None + ) as conn_tunnel: + with pytest.warns(InsecureRequestWarning): + https_pool._make_request(conn, "GET", "/") + conn_tunnel.assert_called_once_with() + + @requires_network() + def test_enhanced_timeout(self) -> None: with HTTPSConnectionPool( TARPIT_HOST, self.port, @@ -604,14 +652,11 @@ def test_enhanced_timeout(self): retries=False, cert_reqs="CERT_REQUIRED", ) as https_pool: - conn = https_pool._new_conn() - try: + with contextlib.closing(https_pool._new_conn()) as conn: with pytest.raises(ConnectTimeoutError): https_pool.request("GET", "/") with pytest.raises(ConnectTimeoutError): https_pool._make_request(conn, "GET", "/") - finally: - conn.close() with HTTPSConnectionPool( TARPIT_HOST, @@ -630,16 +675,13 @@ def test_enhanced_timeout(self): retries=False, cert_reqs="CERT_REQUIRED", ) as https_pool: - conn = https_pool._new_conn() - try: + with contextlib.closing(https_pool._new_conn()) as conn: with pytest.raises(ConnectTimeoutError): https_pool.request( "GET", "/", timeout=Timeout(total=None, connect=SHORT_TIMEOUT) ) - finally: - conn.close() - def test_enhanced_ssl_connection(self): + def test_enhanced_ssl_connection(self) -> None: fingerprint = "72:8B:55:4C:9A:FC:1E:88:A1:1C:AD:1B:B2:E7:CC:3E:DB:C8:F9:8A" with HTTPSConnectionPool( @@ -648,14 +690,17 @@ def test_enhanced_ssl_connection(self): cert_reqs="CERT_REQUIRED", ca_certs=DEFAULT_CA, assert_fingerprint=fingerprint, + ssl_minimum_version=self.tls_version(), ) as https_pool: r = https_pool.request("GET", "/") assert r.status == 200 - @onlyPy279OrNewer - def test_ssl_correct_system_time(self): + def test_ssl_correct_system_time(self) -> None: with HTTPSConnectionPool( - self.host, self.port, ca_certs=DEFAULT_CA + self.host, + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: https_pool.cert_reqs = "CERT_REQUIRED" https_pool.ca_certs = DEFAULT_CA @@ -663,10 +708,12 @@ def test_ssl_correct_system_time(self): w = self._request_without_resource_warnings("GET", "/") assert [] == w - @onlyPy279OrNewer - def test_ssl_wrong_system_time(self): + def test_ssl_wrong_system_time(self) -> None: with HTTPSConnectionPool( - self.host, self.port, ca_certs=DEFAULT_CA + self.host, + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: https_pool.cert_reqs = "CERT_REQUIRED" https_pool.ca_certs = DEFAULT_CA @@ -679,215 +726,583 @@ def test_ssl_wrong_system_time(self): warning = w[0] assert SystemTimeWarning == warning.category + assert isinstance(warning.message, Warning) assert str(RECENT_DATE) in warning.message.args[0] - def _request_without_resource_warnings(self, method, url): + def _request_without_resource_warnings( + self, method: str, url: str + ) -> list[warnings.WarningMessage]: with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") with HTTPSConnectionPool( - self.host, self.port, ca_certs=DEFAULT_CA + self.host, + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: https_pool.request(method, url) w = [x for x in w if not isinstance(x.message, ResourceWarning)] - # If we're using a deprecated TLS version we can remove 'DeprecationWarning' - if self.tls_protocol_deprecated(): - w = [x for x in w if x.category != DeprecationWarning] - return w - def test_set_ssl_version_to_tls_version(self): + def test_set_ssl_version_to_tls_version(self) -> None: if self.tls_protocol_name is None: pytest.skip("Skipping base test class") with HTTPSConnectionPool( self.host, self.port, ca_certs=DEFAULT_CA ) as https_pool: - https_pool.ssl_version = self.certs["ssl_version"] - r = https_pool.request("GET", "/") + https_pool.ssl_version = ssl_version = self.certs["ssl_version"] + if ssl_version is getattr(ssl, "PROTOCOL_TLS", object()): + cmgr: contextlib.AbstractContextManager[object] = ( + contextlib.nullcontext() + ) + else: + cmgr = pytest.warns( + DeprecationWarning, + match=r"'ssl_version' option is deprecated and will be removed " + r"in urllib3 v2\.1\.0\. Instead use 'ssl_minimum_version'", + ) + with cmgr: + r = https_pool.request("GET", "/") assert r.status == 200, r.data - def test_set_cert_default_cert_required(self): + def test_set_cert_default_cert_required(self) -> None: conn = VerifiedHTTPSConnection(self.host, self.port) - conn.set_cert() + with pytest.warns(DeprecationWarning) as w: + conn.set_cert() assert conn.cert_reqs == ssl.CERT_REQUIRED + assert len(w) == 1 and str(w[0].message) == ( + "HTTPSConnection.set_cert() is deprecated and will be removed in urllib3 v2.1.0. " + "Instead provide the parameters to the HTTPSConnection constructor." + ) + + @pytest.mark.parametrize("verify_mode", [ssl.CERT_NONE, ssl.CERT_REQUIRED]) + def test_set_cert_inherits_cert_reqs_from_ssl_context( + self, verify_mode: int + ) -> None: + ssl_context = urllib3.util.ssl_.create_urllib3_context(cert_reqs=verify_mode) + assert ssl_context.verify_mode == verify_mode + + conn = HTTPSConnection(self.host, self.port, ssl_context=ssl_context) + with pytest.warns(DeprecationWarning) as w: + conn.set_cert() + + assert conn.cert_reqs == verify_mode + assert ( + conn.ssl_context is not None and conn.ssl_context.verify_mode == verify_mode + ) + assert len(w) == 1 and str(w[0].message) == ( + "HTTPSConnection.set_cert() is deprecated and will be removed in urllib3 v2.1.0. " + "Instead provide the parameters to the HTTPSConnection constructor." + ) - def test_tls_protocol_name_of_socket(self): + def test_tls_protocol_name_of_socket(self) -> None: if self.tls_protocol_name is None: pytest.skip("Skipping base test class") with HTTPSConnectionPool( - self.host, self.port, ca_certs=DEFAULT_CA + self.host, + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), + ssl_maximum_version=self.tls_version(), ) as https_pool: - conn = https_pool._get_conn() - try: + with contextlib.closing(https_pool._get_conn()) as conn: conn.connect() - if not hasattr(conn.sock, "version"): + if not hasattr(conn.sock, "version"): # type: ignore[attr-defined] pytest.skip("SSLSocket.version() not available") - assert conn.sock.version() == self.tls_protocol_name - finally: - conn.close() + assert conn.sock.version() == self.tls_protocol_name # type: ignore[attr-defined] - def test_default_tls_version_deprecations(self): + def test_ssl_version_is_deprecated(self) -> None: if self.tls_protocol_name is None: pytest.skip("Skipping base test class") + if self.ssl_version() == ssl.PROTOCOL_TLS_CLIENT: + pytest.skip( + "Skipping because ssl_version=ssl.PROTOCOL_TLS_CLIENT is not deprecated" + ) with HTTPSConnectionPool( - self.host, self.port, ca_certs=DEFAULT_CA + self.host, self.port, ca_certs=DEFAULT_CA, ssl_version=self.ssl_version() ) as https_pool: - conn = https_pool._get_conn() - try: - with warnings.catch_warnings(record=True) as w: + with contextlib.closing(https_pool._get_conn()) as conn: + with pytest.warns(DeprecationWarning) as w: conn.connect() - if not hasattr(conn.sock, "version"): - pytest.skip("SSLSocket.version() not available") - finally: - conn.close() - - if self.tls_protocol_deprecated(): - assert len(w) == 1 - assert str(w[0].message) == ( - "Negotiating TLSv1/TLSv1.1 by default is deprecated " - "and will be disabled in urllib3 v2.0.0. Connecting to " - "'%s' with '%s' can be enabled by explicitly opting-in " - "with 'ssl_version'" % (self.host, self.tls_protocol_name) + + assert len(w) >= 1 + assert any(x.category == DeprecationWarning for x in w) + assert any( + str(x.message) + == ( + "'ssl_version' option is deprecated and will be removed in " + "urllib3 v2.1.0. Instead use 'ssl_minimum_version'" ) - else: - assert w == [] + for x in w + ) - def test_no_tls_version_deprecation_with_ssl_version(self): + @pytest.mark.parametrize( + "ssl_version", [None, ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS_CLIENT] + ) + def test_ssl_version_with_protocol_tls_or_client_not_deprecated( + self, ssl_version: int | None + ) -> None: if self.tls_protocol_name is None: pytest.skip("Skipping base test class") + if self.tls_protocol_not_default(): + pytest.skip( + f"Skipping because '{self.tls_protocol_name}' isn't set by default" + ) with HTTPSConnectionPool( - self.host, self.port, ca_certs=DEFAULT_CA, ssl_version=util.PROTOCOL_TLS + self.host, self.port, ca_certs=DEFAULT_CA, ssl_version=ssl_version ) as https_pool: - conn = https_pool._get_conn() - try: + with contextlib.closing(https_pool._get_conn()) as conn: with warnings.catch_warnings(record=True) as w: conn.connect() - finally: - conn.close() - assert w == [] + assert [str(wm) for wm in w if wm.category != ResourceWarning] == [] - def test_no_tls_version_deprecation_with_ssl_context(self): + def test_no_tls_version_deprecation_with_ssl_context(self) -> None: if self.tls_protocol_name is None: pytest.skip("Skipping base test class") + ctx = util.ssl_.create_urllib3_context(ssl_minimum_version=self.tls_version()) + with HTTPSConnectionPool( self.host, self.port, ca_certs=DEFAULT_CA, - ssl_context=util.ssl_.create_urllib3_context(), + ssl_context=ctx, ) as https_pool: - conn = https_pool._get_conn() - try: + with contextlib.closing(https_pool._get_conn()) as conn: with warnings.catch_warnings(record=True) as w: conn.connect() - finally: - conn.close() - assert w == [] + assert [str(wm) for wm in w if wm.category != ResourceWarning] == [] + + def test_tls_version_maximum_and_minimum(self) -> None: + if self.tls_protocol_name is None: + pytest.skip("Skipping base test class") + + from ssl import TLSVersion + + min_max_versions = [ + (self.tls_version(), self.tls_version()), + (TLSVersion.MINIMUM_SUPPORTED, self.tls_version()), + (TLSVersion.MINIMUM_SUPPORTED, TLSVersion.MAXIMUM_SUPPORTED), + ] - @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python 3.8+") - def test_sslkeylogfile(self, tmpdir, monkeypatch): + for minimum_version, maximum_version in min_max_versions: + with HTTPSConnectionPool( + self.host, + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=minimum_version, + ssl_maximum_version=maximum_version, + ) as https_pool: + conn = https_pool._get_conn() + try: + conn.connect() + if maximum_version == TLSVersion.MAXIMUM_SUPPORTED: + # A higher protocol than tls_protocol_name could be negotiated + assert conn.sock.version() >= self.tls_protocol_name # type: ignore[attr-defined] + else: + assert conn.sock.version() == self.tls_protocol_name # type: ignore[attr-defined] + finally: + conn.close() + + def test_sslkeylogfile( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: if not hasattr(util.SSLContext, "keylog_filename"): pytest.skip("requires OpenSSL 1.1.1+") - keylog_file = tmpdir.join("keylogfile.txt") + + keylog_file = tmp_path / "keylogfile.txt" monkeypatch.setenv("SSLKEYLOGFILE", str(keylog_file)) + with HTTPSConnectionPool( - self.host, self.port, ca_certs=DEFAULT_CA + self.host, + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), ) as https_pool: r = https_pool.request("GET", "/") assert r.status == 200, r.data - assert keylog_file.check(file=1), "keylogfile '%s' should exist" % str( + assert keylog_file.is_file(), "keylogfile '%s' should exist" % str( keylog_file ) - assert keylog_file.read().startswith( + assert keylog_file.read_text().startswith( "# TLS secrets log file" ), "keylogfile '%s' should start with '# TLS secrets log file'" % str( keylog_file ) @pytest.mark.parametrize("sslkeylogfile", [None, ""]) - def test_sslkeylogfile_empty(self, monkeypatch, sslkeylogfile): + def test_sslkeylogfile_empty( + self, monkeypatch: pytest.MonkeyPatch, sslkeylogfile: str | None + ) -> None: # Assert that an HTTPS connection doesn't error out when given # no SSLKEYLOGFILE or an empty value (ie 'SSLKEYLOGFILE=') if sslkeylogfile is not None: monkeypatch.setenv("SSLKEYLOGFILE", sslkeylogfile) else: monkeypatch.delenv("SSLKEYLOGFILE", raising=False) - with HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA) as pool: + with HTTPSConnectionPool( + self.host, + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), + ) as pool: r = pool.request("GET", "/") assert r.status == 200, r.data - def test_alpn_default(self): + def test_alpn_default(self, http_version: str) -> None: """Default ALPN protocols are sent by default.""" - if not has_alpn() or not has_alpn(ssl.SSLContext): - pytest.skip("ALPN-support not available") - with HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA) as pool: + with HTTPSConnectionPool( + self.host, + self.port, + ca_certs=DEFAULT_CA, + ssl_minimum_version=self.tls_version(), + ) as pool: r = pool.request("GET", "/alpn_protocol", retries=0) assert r.status == 200 assert r.data.decode("utf-8") == util.ALPN_PROTOCOLS[0] + assert ( + r.data.decode("utf-8") == {"h11": "http/1.1", "h2": "h2"}[http_version] + ) + + def test_http2_probe_result_is_cached(self, http_version: str) -> None: + assert http2_probe._values() == {} + + for i in range(2): # Do this twice to exercise the cache path + with HTTPSConnectionPool( + self.host, + self.port, + ca_certs=DEFAULT_CA, + ) as pool: + r = pool.request("GET", "/alpn_protocol", retries=0) + assert r.status == 200 + + if http_version == "h2": + # This means the probe was successful. + assert http2_probe._values() == {(self.host, self.port): True} + else: + # This means the probe wasn't attempted, otherwise would have a value. + assert http_version == "h11" + assert http2_probe._values() == {} + + @pytest.mark.xfail(reason="Hypercorn always supports both HTTP/2 and HTTP/1.1") + def test_http2_probe_result_failed(self, http_version: str) -> None: + if http_version == "h2": + pytest.skip("Test must have server in HTTP/1.1 mode") + assert http2_probe._values() == {} + + urllib3.http2.inject_into_urllib3() + try: + with HTTPSConnectionPool( + self.host, + self.port, + ca_certs=DEFAULT_CA, + ) as pool: + r = pool.request("GET", "/", retries=0) + assert r.status == 200 + + # The probe was a failure because Hypercorn didn't support HTTP/2. + assert http2_probe._values() == {(self.host, self.port): False} + finally: + urllib3.http2.extract_from_urllib3() + + def test_http2_probe_no_result_in_connect_error(self) -> None: + assert http2_probe._values() == {} + + urllib3.http2.inject_into_urllib3() + try: + with HTTPSConnectionPool( + TARPIT_HOST, + self.port, + ca_certs=DEFAULT_CA, + timeout=SHORT_TIMEOUT, + ) as pool: + with pytest.raises(ConnectTimeoutError): + pool.request("GET", "/", retries=False) + + # The probe was inconclusive since an error occurred during connection. + assert http2_probe._values() == {(TARPIT_HOST, self.port): None} + finally: + urllib3.http2.extract_from_urllib3() + + def test_http2_probe_no_result_in_ssl_error(self) -> None: + urllib3.http2.inject_into_urllib3() + try: + with HTTPSConnectionPool( + self.host, + self.port, + ca_certs=None, + timeout=LONG_TIMEOUT, + ) as pool: + with pytest.raises(SSLError): + pool.request("GET", "/", retries=False) + + # The probe was inconclusive since an error occurred during connection. + assert http2_probe._values() == {(self.host, self.port): None} + finally: + urllib3.http2.extract_from_urllib3() + + def test_http2_probe_blocked_per_thread(self) -> None: + state, current_thread, last_action = None, None, time.perf_counter() + + def connect_callback(label: str, thread_id: int, **kwargs: typing.Any) -> None: + nonlocal state, current_thread, last_action + + if label in ("before connect", "after connect failure"): + # We don't know if the target supports HTTP/2 as connections fail + assert kwargs["target_supports_http2"] is None + + # Since we're trying to connect to TARPIT_HOST, all connections will + # fail, but they should be tried one after the other + now = time.perf_counter() + assert now >= last_action + last_action = now + + if label == "before connect": + assert state is None + state = "connect" + assert current_thread != thread_id + current_thread = thread_id + elif label == "after connect failure": + assert state == "connect" + assert current_thread == thread_id + state = None + + assert http2_probe._values() == {} + + connect_timeout = LONG_TIMEOUT + total_threads = 3 + urllib3.http2.inject_into_urllib3() + try: + + def try_connect(_: typing.Any) -> tuple[float, float]: + with HTTPSConnectionPool( + TARPIT_HOST, + self.port, + ca_certs=DEFAULT_CA, + timeout=connect_timeout, + ) as pool: + start_time = time.time() + conn = pool._get_conn() + assert isinstance(conn, HTTPSConnection) + conn._connect_callback = connect_callback + with pytest.raises(ConnectTimeoutError): + conn.connect() + end_time = time.time() + return start_time, end_time + + threadpool = concurrent.futures.ThreadPoolExecutor(total_threads) + list(threadpool.map(try_connect, range(total_threads))) + + # The probe was inconclusive since an error occurred during connection. + assert http2_probe._values() == {(TARPIT_HOST, self.port): None} + finally: + urllib3.http2.extract_from_urllib3() + + def test_default_ssl_context_ssl_min_max_versions(self) -> None: + ctx = urllib3.util.ssl_.create_urllib3_context() + assert ctx.minimum_version == ssl.TLSVersion.TLSv1_2 + # urllib3 sets a default maximum version only when it is + # injected with PyOpenSSL SSL-support. + # Otherwise, the default maximum version is set by Python's + # `ssl.SSLContext`. The value respects OpenSSL configuration and + # can be different from `ssl.TLSVersion.MAXIMUM_SUPPORTED`. + # https://github.com/urllib3/urllib3/issues/2477#issuecomment-1151452150 + if util.IS_PYOPENSSL: + expected_maximum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED + else: + expected_maximum_version = ssl.SSLContext( + ssl.PROTOCOL_TLS_CLIENT + ).maximum_version + assert ctx.maximum_version == expected_maximum_version + + def test_ssl_context_ssl_version_uses_ssl_min_max_versions(self) -> None: + if self.ssl_version() == ssl.PROTOCOL_TLS_CLIENT: + pytest.skip( + "Skipping because ssl_version=ssl.PROTOCOL_TLS_CLIENT is not deprecated" + ) + + with pytest.warns( + DeprecationWarning, + match=r"'ssl_version' option is deprecated and will be removed in " + r"urllib3 v2\.1\.0\. Instead use 'ssl_minimum_version'", + ): + ctx = urllib3.util.ssl_.create_urllib3_context( + ssl_version=self.ssl_version() + ) + assert ctx.minimum_version == self.tls_version() + assert ctx.maximum_version == self.tls_version() + + def test_assert_missing_hashfunc(self, monkeypatch: pytest.MonkeyPatch) -> None: + fingerprint = "55:39:BF:70:05:12:43:FA:1F:D1:BF:4E:E8:1B:07:1D" + with HTTPSConnectionPool( + "localhost", + self.port, + cert_reqs="CERT_REQUIRED", + ca_certs=DEFAULT_CA, + assert_fingerprint=(fingerprint), + ssl_minimum_version=self.tls_version(), + ) as https_pool: + digest_length = len(fingerprint.replace(":", "").lower()) + monkeypatch.setitem(urllib3.util.ssl_.HASHFUNC_MAP, digest_length, None) + with pytest.raises(MaxRetryError) as cm: + https_pool.request("GET", "/", retries=0) + assert type(cm.value.reason) is SSLError + assert ( + f"Hash function implementation unavailable for fingerprint length: {digest_length}" + == str(cm.value.reason) + ) @pytest.mark.usefixtures("requires_tlsv1") -class TestHTTPS_TLSv1(TestHTTPS): +class TestHTTPS_TLSv1(BaseTestHTTPS): tls_protocol_name = "TLSv1" certs = TLSv1_CERTS @pytest.mark.usefixtures("requires_tlsv1_1") -class TestHTTPS_TLSv1_1(TestHTTPS): +class TestHTTPS_TLSv1_1(BaseTestHTTPS): tls_protocol_name = "TLSv1.1" certs = TLSv1_1_CERTS @pytest.mark.usefixtures("requires_tlsv1_2") -class TestHTTPS_TLSv1_2(TestHTTPS): +class TestHTTPS_TLSv1_2(BaseTestHTTPS): tls_protocol_name = "TLSv1.2" certs = TLSv1_2_CERTS @pytest.mark.usefixtures("requires_tlsv1_3") -class TestHTTPS_TLSv1_3(TestHTTPS): +class TestHTTPS_TLSv1_3(BaseTestHTTPS): tls_protocol_name = "TLSv1.3" certs = TLSv1_3_CERTS -class TestHTTPS_NoSAN: - def test_warning_for_certs_without_a_san(self, no_san_server): - """Ensure that a warning is raised when the cert from the server has - no Subject Alternative Name.""" - with mock.patch("warnings.warn") as warn: +class TestHTTPS_Hostname: + def test_can_validate_san(self, san_server: ServerConfig) -> None: + """Ensure that urllib3 can validate SANs with IP addresses in them.""" + with HTTPSConnectionPool( + san_server.host, + san_server.port, + cert_reqs="CERT_REQUIRED", + ca_certs=san_server.ca_certs, + ) as https_pool: + r = https_pool.request("GET", "/") + assert r.status == 200 + + def test_common_name_without_san_fails(self, no_san_server: ServerConfig) -> None: + with HTTPSConnectionPool( + no_san_server.host, + no_san_server.port, + cert_reqs="CERT_REQUIRED", + ca_certs=no_san_server.ca_certs, + ) as https_pool: + with pytest.raises( + MaxRetryError, + ) as e: + https_pool.request("GET", "/") + assert "mismatch, certificate is not valid" in str( + e.value + ) or "no appropriate subjectAltName" in str(e.value) + + def test_common_name_without_san_with_different_common_name( + self, no_san_server_with_different_commmon_name: ServerConfig + ) -> None: + ctx = urllib3.util.ssl_.create_urllib3_context() + try: + ctx.hostname_checks_common_name = True + except AttributeError: + pytest.skip("Couldn't set 'SSLContext.hostname_checks_common_name'") + + with HTTPSConnectionPool( + no_san_server_with_different_commmon_name.host, + no_san_server_with_different_commmon_name.port, + cert_reqs="CERT_REQUIRED", + ca_certs=no_san_server_with_different_commmon_name.ca_certs, + ssl_context=ctx, + ) as https_pool: + with pytest.raises(MaxRetryError) as e: + https_pool.request("GET", "/") + assert "mismatch, certificate is not valid for 'localhost'" in str( + e.value + ) or "hostname 'localhost' doesn't match 'example.com'" in str(e.value) + + @pytest.mark.parametrize("use_assert_hostname", [True, False]) + def test_hostname_checks_common_name_respected( + self, no_san_server: ServerConfig, use_assert_hostname: bool + ) -> None: + ctx = urllib3.util.ssl_.create_urllib3_context() + if not hasattr(ctx, "hostname_checks_common_name"): + pytest.skip("Test requires 'SSLContext.hostname_checks_common_name'") + ctx.load_verify_locations(no_san_server.ca_certs) + try: + ctx.hostname_checks_common_name = True + except AttributeError: + pytest.skip("Couldn't set 'SSLContext.hostname_checks_common_name'") + + err: MaxRetryError | None + try: with HTTPSConnectionPool( no_san_server.host, no_san_server.port, cert_reqs="CERT_REQUIRED", - ca_certs=no_san_server.ca_certs, + ssl_context=ctx, + assert_hostname=no_san_server.host if use_assert_hostname else None, ) as https_pool: - r = https_pool.request("GET", "/") - assert r.status == 200 - assert warn.called + https_pool.request("GET", "/") + except MaxRetryError as e: + err = e + else: + err = None - def test_common_name_without_san_with_different_common_name( - self, no_san_server_with_different_commmon_name - ): + # commonName is only valid for DNS names, not IP addresses. + if no_san_server.host == "localhost": + assert err is None + + # IP addresses should fail for commonName. + else: + assert err is not None + assert type(err.reason) is SSLError + assert isinstance( + err.reason.args[0], (ssl.SSLCertVerificationError, CertificateError) + ) + + def test_assert_hostname_invalid_san( + self, no_localhost_san_server: ServerConfig + ) -> None: + """Ensure SAN errors are not raised while assert_hostname is false""" + with HTTPSConnectionPool( + no_localhost_san_server.host, + no_localhost_san_server.port, + cert_reqs="CERT_REQUIRED", + ca_certs=no_localhost_san_server.ca_certs, + assert_hostname=False, + ) as https_pool: + https_pool.request("GET", "/") + + def test_assert_hostname_invalid_cn( + self, no_san_server_with_different_commmon_name: ServerConfig + ) -> None: + """Ensure CN errors are not raised while assert_hostname is false""" with HTTPSConnectionPool( no_san_server_with_different_commmon_name.host, no_san_server_with_different_commmon_name.port, cert_reqs="CERT_REQUIRED", ca_certs=no_san_server_with_different_commmon_name.ca_certs, + assert_hostname=False, ) as https_pool: - with pytest.raises(MaxRetryError) as cm: - https_pool.request("GET", "/") - assert isinstance(cm.value.reason, SSLError) + https_pool.request("GET", "/") class TestHTTPS_IPV4SAN: - def test_can_validate_ip_san(self, ipv4_san_server): + def test_can_validate_ip_san(self, ipv4_san_server: ServerConfig) -> None: """Ensure that urllib3 can validate SANs with IP addresses in them.""" with HTTPSConnectionPool( ipv4_san_server.host, @@ -899,23 +1314,11 @@ def test_can_validate_ip_san(self, ipv4_san_server): assert r.status == 200 -class TestHTTPS_IPv6Addr: - @pytest.mark.parametrize("host", ["::1", "[::1]"]) - def test_strip_square_brackets_before_validating(self, ipv6_addr_server, host): - """Test that the fix for #760 works.""" - with HTTPSConnectionPool( - host, - ipv6_addr_server.port, - cert_reqs="CERT_REQUIRED", - ca_certs=ipv6_addr_server.ca_certs, - ) as https_pool: - r = https_pool.request("GET", "/") - assert r.status == 200 - - class TestHTTPS_IPV6SAN: @pytest.mark.parametrize("host", ["::1", "[::1]"]) - def test_can_validate_ipv6_san(self, ipv6_san_server, host): + def test_can_validate_ipv6_san( + self, ipv6_san_server: ServerConfig, host: str, http_version: str + ) -> None: """Ensure that urllib3 can validate SANs with IPv6 addresses in them.""" with HTTPSConnectionPool( host, @@ -925,3 +1328,4 @@ def test_can_validate_ipv6_san(self, ipv6_san_server, host): ) as https_pool: r = https_pool.request("GET", "/") assert r.status == 200 + assert r.headers["server"] == f"hypercorn-{http_version}" diff --git a/test/with_dummyserver/test_no_ssl.py b/test/with_dummyserver/test_no_ssl.py index 43e79b7..733a0d3 100644 --- a/test/with_dummyserver/test_no_ssl.py +++ b/test/with_dummyserver/test_no_ssl.py @@ -3,30 +3,35 @@ Note: Import urllib3 inside the test functions to get the importblocker to work """ + +from __future__ import annotations + import pytest import urllib3 -from dummyserver.testcase import HTTPDummyServerTestCase, HTTPSDummyServerTestCase +from dummyserver.testcase import ( + HTTPSHypercornDummyServerTestCase, + HypercornDummyServerTestCase, +) +from urllib3.exceptions import InsecureRequestWarning from ..test_no_ssl import TestWithoutSSL -# Retry failed tests -pytestmark = pytest.mark.flaky - -class TestHTTPWithoutSSL(HTTPDummyServerTestCase, TestWithoutSSL): - def test_simple(self): +class TestHTTPWithoutSSL(HypercornDummyServerTestCase, TestWithoutSSL): + def test_simple(self) -> None: with urllib3.HTTPConnectionPool(self.host, self.port) as pool: r = pool.request("GET", "/") assert r.status == 200, r.data -class TestHTTPSWithoutSSL(HTTPSDummyServerTestCase, TestWithoutSSL): - def test_simple(self): +class TestHTTPSWithoutSSL(HTTPSHypercornDummyServerTestCase, TestWithoutSSL): + def test_simple(self) -> None: with urllib3.HTTPSConnectionPool( self.host, self.port, cert_reqs="NONE" ) as pool: - try: - pool.request("GET", "/") - except urllib3.exceptions.SSLError as e: - assert "SSL module is not available" in str(e) + with pytest.warns(InsecureRequestWarning): + try: + pool.request("GET", "/") + except urllib3.exceptions.SSLError as e: + assert "SSL module is not available" in str(e) diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index 509daf2..af77241 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -1,33 +1,37 @@ -import json +from __future__ import annotations + +import gzip +import typing from test import LONG_TIMEOUT +from unittest import mock import pytest -from dummyserver.server import HAS_IPV6 -from dummyserver.testcase import HTTPDummyServerTestCase, IPv6HTTPDummyServerTestCase -from urllib3._collections import HTTPHeaderDict +from dummyserver.socketserver import HAS_IPV6 +from dummyserver.testcase import ( + HypercornDummyServerTestCase, + IPv6HypercornDummyServerTestCase, +) +from urllib3 import HTTPHeaderDict, HTTPResponse, request from urllib3.connectionpool import port_by_scheme from urllib3.exceptions import MaxRetryError, URLSchemeUnknown from urllib3.poolmanager import PoolManager from urllib3.util.retry import Retry -# Retry failed tests -pytestmark = pytest.mark.flaky - -class TestPoolManager(HTTPDummyServerTestCase): +class TestPoolManager(HypercornDummyServerTestCase): @classmethod - def setup_class(cls): - super(TestPoolManager, cls).setup_class() - cls.base_url = "http://%s:%d" % (cls.host, cls.port) - cls.base_url_alt = "http://%s:%d" % (cls.host_alt, cls.port) + def setup_class(cls) -> None: + super().setup_class() + cls.base_url = f"http://{cls.host}:{cls.port}" + cls.base_url_alt = f"http://{cls.host_alt}:{cls.port}" - def test_redirect(self): + def test_redirect(self) -> None: with PoolManager() as http: r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/" % self.base_url}, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url}/"}, redirect=False, ) @@ -35,19 +39,19 @@ def test_redirect(self): r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/" % self.base_url}, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url}/"}, ) assert r.status == 200 assert r.data == b"Dummy server!" - def test_redirect_twice(self): + def test_redirect_twice(self) -> None: with PoolManager() as http: r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/redirect" % self.base_url}, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url}/redirect"}, redirect=False, ) @@ -55,20 +59,18 @@ def test_redirect_twice(self): r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={ - "target": "%s/redirect?target=%s/" % (self.base_url, self.base_url) - }, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url}/redirect?target={self.base_url}/"}, ) assert r.status == 200 assert r.data == b"Dummy server!" - def test_redirect_to_relative_url(self): + def test_redirect_to_relative_url(self) -> None: with PoolManager() as http: r = http.request( "GET", - "%s/redirect" % self.base_url, + f"{self.base_url}/redirect", fields={"target": "/redirect"}, redirect=False, ) @@ -76,19 +78,19 @@ def test_redirect_to_relative_url(self): assert r.status == 303 r = http.request( - "GET", "%s/redirect" % self.base_url, fields={"target": "/redirect"} + "GET", f"{self.base_url}/redirect", fields={"target": "/redirect"} ) assert r.status == 200 assert r.data == b"Dummy server!" - def test_cross_host_redirect(self): + def test_cross_host_redirect(self) -> None: with PoolManager() as http: - cross_host_location = "%s/echo?a=b" % self.base_url_alt + cross_host_location = f"{self.base_url_alt}/echo?a=b" with pytest.raises(MaxRetryError): http.request( "GET", - "%s/redirect" % self.base_url, + f"{self.base_url}/redirect", fields={"target": cross_host_location}, timeout=LONG_TIMEOUT, retries=0, @@ -96,23 +98,24 @@ def test_cross_host_redirect(self): r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/echo?a=b" % self.base_url_alt}, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url_alt}/echo?a=b"}, timeout=LONG_TIMEOUT, retries=1, ) + assert isinstance(r, HTTPResponse) + assert r._pool is not None assert r._pool.host == self.host_alt - def test_too_many_redirects(self): + def test_too_many_redirects(self) -> None: with PoolManager() as http: with pytest.raises(MaxRetryError): http.request( "GET", - "%s/redirect" % self.base_url, + f"{self.base_url}/redirect", fields={ - "target": "%s/redirect?target=%s/" - % (self.base_url, self.base_url) + "target": f"{self.base_url}/redirect?target={self.base_url}/" }, retries=1, preload_content=False, @@ -121,10 +124,9 @@ def test_too_many_redirects(self): with pytest.raises(MaxRetryError): http.request( "GET", - "%s/redirect" % self.base_url, + f"{self.base_url}/redirect", fields={ - "target": "%s/redirect?target=%s/" - % (self.base_url, self.base_url) + "target": f"{self.base_url}/redirect?target={self.base_url}/" }, retries=Retry(total=None, redirect=1), preload_content=False, @@ -136,64 +138,81 @@ def test_too_many_redirects(self): pool = http.connection_from_host(self.host, self.port) assert pool.num_connections == 1 - def test_redirect_cross_host_remove_headers(self): + def test_redirect_cross_host_remove_headers(self) -> None: with PoolManager() as http: r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, - headers={"Authorization": "foo", "Cookie": "foo=bar"}, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url_alt}/headers"}, + headers={ + "Authorization": "foo", + "Proxy-Authorization": "bar", + "Cookie": "foo=bar", + }, ) assert r.status == 200 - data = json.loads(r.data.decode("utf-8")) + data = r.json() assert "Authorization" not in data + assert "Proxy-Authorization" not in data assert "Cookie" not in data r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, - headers={"authorization": "foo", "cookie": "foo=bar"}, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url_alt}/headers"}, + headers={ + "authorization": "foo", + "proxy-authorization": "baz", + "cookie": "foo=bar", + }, ) assert r.status == 200 - data = json.loads(r.data.decode("utf-8")) + data = r.json() assert "authorization" not in data assert "Authorization" not in data + assert "proxy-authorization" not in data + assert "Proxy-Authorization" not in data assert "cookie" not in data assert "Cookie" not in data - def test_redirect_cross_host_no_remove_headers(self): + def test_redirect_cross_host_no_remove_headers(self) -> None: with PoolManager() as http: r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, - headers={"Authorization": "foo", "Cookie": "foo=bar"}, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url_alt}/headers"}, + headers={ + "Authorization": "foo", + "Proxy-Authorization": "bar", + "Cookie": "foo=bar", + }, retries=Retry(remove_headers_on_redirect=[]), ) assert r.status == 200 - data = json.loads(r.data.decode("utf-8")) + data = r.json() assert data["Authorization"] == "foo" + assert data["Proxy-Authorization"] == "bar" assert data["Cookie"] == "foo=bar" - def test_redirect_cross_host_set_removed_headers(self): + def test_redirect_cross_host_set_removed_headers(self) -> None: with PoolManager() as http: r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url_alt}/headers"}, headers={ "X-API-Secret": "foo", "Authorization": "bar", + "Proxy-Authorization": "baz", "Cookie": "foo=bar", }, retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), @@ -201,66 +220,78 @@ def test_redirect_cross_host_set_removed_headers(self): assert r.status == 200 - data = json.loads(r.data.decode("utf-8")) + data = r.json() assert "X-API-Secret" not in data assert data["Authorization"] == "bar" + assert data["Proxy-Authorization"] == "baz" assert data["Cookie"] == "foo=bar" + headers = { + "x-api-secret": "foo", + "authorization": "bar", + "proxy-authorization": "baz", + "cookie": "foo=bar", + } r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, - headers={ - "x-api-secret": "foo", - "authorization": "bar", - "cookie": "foo=bar", - }, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url_alt}/headers"}, + headers=headers, retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), ) assert r.status == 200 - data = json.loads(r.data.decode("utf-8")) + data = r.json() assert "x-api-secret" not in data assert "X-API-Secret" not in data assert data["Authorization"] == "bar" + assert data["Proxy-Authorization"] == "baz" assert data["Cookie"] == "foo=bar" - def test_redirect_without_preload_releases_connection(self): + # Ensure the header argument itself is not modified in-place. + assert headers == { + "x-api-secret": "foo", + "authorization": "bar", + "proxy-authorization": "baz", + "cookie": "foo=bar", + } + + def test_redirect_without_preload_releases_connection(self) -> None: with PoolManager(block=True, maxsize=2) as http: - r = http.request( - "GET", "%s/redirect" % self.base_url, preload_content=False - ) + r = http.request("GET", f"{self.base_url}/redirect", preload_content=False) + assert isinstance(r, HTTPResponse) + assert r._pool is not None assert r._pool.num_requests == 2 assert r._pool.num_connections == 1 assert len(http.pools) == 1 - def test_303_redirect_makes_request_lose_body(self): + def test_303_redirect_makes_request_lose_body(self) -> None: with PoolManager() as http: response = http.request( "POST", - "%s/redirect" % self.base_url, + f"{self.base_url}/redirect", fields={ - "target": "%s/headers_and_params" % self.base_url, + "target": f"{self.base_url}/headers_and_params", "status": "303 See Other", }, ) - data = json.loads(response.data) + data = response.json() assert data["params"] == {} assert "Content-Type" not in HTTPHeaderDict(data["headers"]) - def test_unknown_scheme(self): + def test_unknown_scheme(self) -> None: with PoolManager() as http: unknown_scheme = "unknown" - unknown_scheme_url = "%s://host" % unknown_scheme + unknown_scheme_url = f"{unknown_scheme}://host" with pytest.raises(URLSchemeUnknown) as e: r = http.request("GET", unknown_scheme_url) assert e.value.scheme == unknown_scheme r = http.request( "GET", - "%s/redirect" % self.base_url, + f"{self.base_url}/redirect", fields={"target": unknown_scheme_url}, redirect=False, ) @@ -269,31 +300,29 @@ def test_unknown_scheme(self): with pytest.raises(URLSchemeUnknown) as e: r = http.request( "GET", - "%s/redirect" % self.base_url, + f"{self.base_url}/redirect", fields={"target": unknown_scheme_url}, ) assert e.value.scheme == unknown_scheme - def test_raise_on_redirect(self): + def test_raise_on_redirect(self) -> None: with PoolManager() as http: r = http.request( "GET", - "%s/redirect" % self.base_url, - fields={ - "target": "%s/redirect?target=%s/" % (self.base_url, self.base_url) - }, + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url}/redirect?target={self.base_url}/"}, retries=Retry(total=None, redirect=1, raise_on_redirect=False), ) assert r.status == 303 - def test_raise_on_status(self): + def test_raise_on_status(self) -> None: with PoolManager() as http: with pytest.raises(MaxRetryError): # the default is to raise r = http.request( "GET", - "%s/status" % self.base_url, + f"{self.base_url}/status", fields={"status": "500 Internal Server Error"}, retries=Retry(total=1, status_forcelist=range(500, 600)), ) @@ -302,7 +331,7 @@ def test_raise_on_status(self): # raise explicitly r = http.request( "GET", - "%s/status" % self.base_url, + f"{self.base_url}/status", fields={"status": "500 Internal Server Error"}, retries=Retry( total=1, status_forcelist=range(500, 600), raise_on_status=True @@ -312,7 +341,7 @@ def test_raise_on_status(self): # don't raise r = http.request( "GET", - "%s/status" % self.base_url, + f"{self.base_url}/status", fields={"status": "500 Internal Server Error"}, retries=Retry( total=1, status_forcelist=range(500, 600), raise_on_status=False @@ -321,7 +350,7 @@ def test_raise_on_status(self): assert r.status == 500 - def test_missing_port(self): + def test_missing_port(self) -> None: # Can a URL that lacks an explicit port like ':80' succeed, or # will all such URLs fail with an error? @@ -331,87 +360,342 @@ def test_missing_port(self): # our test server happens to be listening. port_by_scheme["http"] = self.port try: - r = http.request("GET", "http://%s/" % self.host, retries=0) + r = http.request("GET", f"http://{self.host}/", retries=0) finally: port_by_scheme["http"] = 80 assert r.status == 200 assert r.data == b"Dummy server!" - def test_headers(self): + def test_headers(self) -> None: with PoolManager(headers={"Foo": "bar"}) as http: - r = http.request("GET", "%s/headers" % self.base_url) - returned_headers = json.loads(r.data.decode()) + r = http.request("GET", f"{self.base_url}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" - r = http.request("POST", "%s/headers" % self.base_url) - returned_headers = json.loads(r.data.decode()) + r = http.request("POST", f"{self.base_url}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" - r = http.request_encode_url("GET", "%s/headers" % self.base_url) - returned_headers = json.loads(r.data.decode()) + r = http.request_encode_url("GET", f"{self.base_url}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" - r = http.request_encode_body("POST", "%s/headers" % self.base_url) - returned_headers = json.loads(r.data.decode()) + r = http.request_encode_body("POST", f"{self.base_url}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" r = http.request_encode_url( - "GET", "%s/headers" % self.base_url, headers={"Baz": "quux"} + "GET", f"{self.base_url}/headers", headers={"Baz": "quux"} ) - returned_headers = json.loads(r.data.decode()) + returned_headers = r.json() assert returned_headers.get("Foo") is None assert returned_headers.get("Baz") == "quux" r = http.request_encode_body( - "GET", "%s/headers" % self.base_url, headers={"Baz": "quux"} + "GET", f"{self.base_url}/headers", headers={"Baz": "quux"} ) - returned_headers = json.loads(r.data.decode()) + returned_headers = r.json() assert returned_headers.get("Foo") is None assert returned_headers.get("Baz") == "quux" - def test_http_with_ssl_keywords(self): + def test_headers_http_header_dict(self) -> None: + # Test uses a list of headers to assert the order + # that headers are sent in the request too. + + headers = HTTPHeaderDict() + headers.add("Foo", "bar") + headers.add("Multi", "1") + headers.add("Baz", "quux") + headers.add("Multi", "2") + + with PoolManager(headers=headers) as http: + r = http.request("GET", f"{self.base_url}/multi_headers") + returned_headers = r.json()["headers"] + assert returned_headers[-4:] == [ + ["Foo", "bar"], + ["Multi", "1"], + ["Multi", "2"], + ["Baz", "quux"], + ] + + r = http.request( + "GET", + f"{self.base_url}/multi_headers", + headers={ + **headers, + "Extra": "extra", + "Foo": "new", + }, + ) + returned_headers = r.json()["headers"] + assert returned_headers[-4:] == [ + ["Foo", "new"], + ["Multi", "1, 2"], + ["Baz", "quux"], + ["Extra", "extra"], + ] + + def test_merge_headers_with_pool_manager_headers(self) -> None: + headers = HTTPHeaderDict() + headers.add("Cookie", "choc-chip") + headers.add("Cookie", "oatmeal-raisin") + orig = headers.copy() + added_headers = {"Cookie": "tim-tam"} + + with PoolManager(headers=headers) as http: + r = http.request( + "GET", + f"{self.base_url}/multi_headers", + headers=typing.cast(HTTPHeaderDict, http.headers) | added_headers, + ) + returned_headers = r.json()["headers"] + assert returned_headers[-3:] == [ + ["Cookie", "choc-chip"], + ["Cookie", "oatmeal-raisin"], + ["Cookie", "tim-tam"], + ] + # make sure the pool headers weren't modified + assert http.headers == orig + + def test_headers_http_multi_header_multipart(self) -> None: + headers = HTTPHeaderDict() + headers.add("Multi", "1") + headers.add("Multi", "2") + old_headers = headers.copy() + + with PoolManager(headers=headers) as http: + r = http.request( + "POST", + f"{self.base_url}/multi_headers", + fields={"k": "v"}, + multipart_boundary="b", + encode_multipart=True, + ) + returned_headers = r.json()["headers"] + assert returned_headers[5:] == [ + ["Multi", "1"], + ["Multi", "2"], + ["Content-Type", "multipart/form-data; boundary=b"], + ] + # Assert that the previous headers weren't modified. + assert headers == old_headers + + # Set a default value for the Content-Type + headers["Content-Type"] = "multipart/form-data; boundary=b; field=value" + r = http.request( + "POST", + f"{self.base_url}/multi_headers", + fields={"k": "v"}, + multipart_boundary="b", + encode_multipart=True, + ) + returned_headers = r.json()["headers"] + assert returned_headers[5:] == [ + ["Multi", "1"], + ["Multi", "2"], + # Uses the set value, not the one that would be generated. + ["Content-Type", "multipart/form-data; boundary=b; field=value"], + ] + + def test_body(self) -> None: + with PoolManager() as http: + r = http.request("POST", f"{self.base_url}/echo", body=b"test") + assert r.data == b"test" + + def test_http_with_ssl_keywords(self) -> None: with PoolManager(ca_certs="REQUIRED") as http: - r = http.request("GET", "http://%s:%s/" % (self.host, self.port)) + r = http.request("GET", f"http://{self.host}:{self.port}/") assert r.status == 200 - def test_http_with_server_hostname(self): + def test_http_with_server_hostname(self) -> None: with PoolManager(server_hostname="example.com") as http: - r = http.request("GET", "http://%s:%s/" % (self.host, self.port)) + r = http.request("GET", f"http://{self.host}:{self.port}/") assert r.status == 200 - def test_http_with_ca_cert_dir(self): + def test_http_with_ca_cert_dir(self) -> None: with PoolManager(ca_certs="REQUIRED", ca_cert_dir="/nosuchdir") as http: - r = http.request("GET", "http://%s:%s/" % (self.host, self.port)) + r = http.request("GET", f"http://{self.host}:{self.port}/") assert r.status == 200 @pytest.mark.parametrize( ["target", "expected_target"], [ + # annoyingly quart.request.full_path adds a stray `?` + ("/echo_uri", b"/echo_uri?"), ("/echo_uri?q=1#fragment", b"/echo_uri?q=1"), ("/echo_uri?#", b"/echo_uri?"), - ("/echo_uri#?", b"/echo_uri"), - ("/echo_uri#?#", b"/echo_uri"), + ("/echo_uri#!", b"/echo_uri?"), + ("/echo_uri#!#", b"/echo_uri?"), ("/echo_uri??#", b"/echo_uri??"), ("/echo_uri?%3f#", b"/echo_uri?%3F"), ("/echo_uri?%3F#", b"/echo_uri?%3F"), ("/echo_uri?[]", b"/echo_uri?%5B%5D"), ], ) - def test_encode_http_target(self, target, expected_target): + def test_encode_http_target(self, target: str, expected_target: bytes) -> None: with PoolManager() as http: - url = "http://%s:%d%s" % (self.host, self.port, target) + url = f"http://{self.host}:{self.port}{target}" r = http.request("GET", url) assert r.data == expected_target + def test_top_level_request(self) -> None: + r = request("GET", f"{self.base_url}/") + assert r.status == 200 + assert r.data == b"Dummy server!" + + def test_top_level_request_without_keyword_args(self) -> None: + body = "" + with pytest.raises(TypeError): + request("GET", f"{self.base_url}/", body) # type: ignore[misc] + + def test_top_level_request_with_body(self) -> None: + r = request("POST", f"{self.base_url}/echo", body=b"test") + assert r.status == 200 + assert r.data == b"test" + + def test_top_level_request_with_preload_content(self) -> None: + r = request("GET", f"{self.base_url}/echo", preload_content=False) + assert r.status == 200 + assert r.connection is not None + r.data + assert r.connection is None + + def test_top_level_request_with_decode_content(self) -> None: + r = request( + "GET", + f"{self.base_url}/encodingrequest", + headers={"accept-encoding": "gzip"}, + decode_content=False, + ) + assert r.status == 200 + assert gzip.decompress(r.data) == b"hello, world!" + + r = request( + "GET", + f"{self.base_url}/encodingrequest", + headers={"accept-encoding": "gzip"}, + decode_content=True, + ) + assert r.status == 200 + assert r.data == b"hello, world!" + + def test_top_level_request_with_redirect(self) -> None: + r = request( + "GET", + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url}/"}, + redirect=False, + ) + + assert r.status == 303 + + r = request( + "GET", + f"{self.base_url}/redirect", + fields={"target": f"{self.base_url}/"}, + redirect=True, + ) + + assert r.status == 200 + assert r.data == b"Dummy server!" + + def test_top_level_request_with_retries(self) -> None: + r = request("GET", f"{self.base_url}/redirect", retries=False) + assert r.status == 303 + + r = request("GET", f"{self.base_url}/redirect", retries=3) + assert r.status == 200 + + def test_top_level_request_with_timeout(self) -> None: + with mock.patch("urllib3.poolmanager.RequestMethods.request") as mockRequest: + mockRequest.return_value = HTTPResponse(status=200) + + r = request("GET", f"{self.base_url}/redirect", timeout=2.5) + + assert r.status == 200 + + mockRequest.assert_called_with( + "GET", + f"{self.base_url}/redirect", + body=None, + fields=None, + headers=None, + preload_content=True, + decode_content=True, + redirect=True, + retries=None, + timeout=2.5, + json=None, + ) + + @pytest.mark.parametrize( + "headers", + [ + None, + {"content-Type": "application/json"}, + {"content-Type": "text/plain"}, + {"attribute": "value", "CONTENT-TYPE": "application/json"}, + HTTPHeaderDict(cookie="foo, bar"), + ], + ) + def test_request_with_json(self, headers: HTTPHeaderDict) -> None: + old_headers = None if headers is None else headers.copy() + body = {"attribute": "value"} + r = request( + method="POST", url=f"{self.base_url}/echo_json", headers=headers, json=body + ) + assert r.status == 200 + assert r.json() == body + content_type = HTTPHeaderDict(old_headers).get( + "Content-Type", "application/json" + ) + assert content_type in r.headers["Content-Type"].replace(" ", "").split(",") + + # Ensure the header argument itself is not modified in-place. + assert headers == old_headers + + def test_top_level_request_with_json_with_httpheaderdict(self) -> None: + body = {"attribute": "value"} + header = HTTPHeaderDict(cookie="foo, bar") + with PoolManager(headers=header) as http: + r = http.request(method="POST", url=f"{self.base_url}/echo_json", json=body) + assert r.status == 200 + assert r.json() == body + assert "application/json" in r.headers["Content-Type"].replace( + " ", "" + ).split(",") + + def test_top_level_request_with_body_and_json(self) -> None: + match = "request got values for both 'body' and 'json' parameters which are mutually exclusive" + with pytest.raises(TypeError, match=match): + body = {"attribute": "value"} + request(method="POST", url=f"{self.base_url}/echo", body="", json=body) + + def test_top_level_request_with_invalid_body(self) -> None: + class BadBody: + def __repr__(self) -> str: + return "" + + with pytest.raises(TypeError) as e: + request( + method="POST", + url=f"{self.base_url}/echo", + body=BadBody(), # type: ignore[arg-type] + ) + assert str(e.value) == ( + "'body' must be a bytes-like object, file-like " + "object, or iterable. Instead was " + ) + @pytest.mark.skipif(not HAS_IPV6, reason="IPv6 is not supported on this system") -class TestIPv6PoolManager(IPv6HTTPDummyServerTestCase): +class TestIPv6PoolManager(IPv6HypercornDummyServerTestCase): @classmethod - def setup_class(cls): - super(TestIPv6PoolManager, cls).setup_class() - cls.base_url = "http://[%s]:%d" % (cls.host, cls.port) + def setup_class(cls) -> None: + super().setup_class() + cls.base_url = f"http://[{cls.host}]:{cls.port}" - def test_ipv6(self): + def test_ipv6(self) -> None: with PoolManager() as http: http.request("GET", self.base_url) diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 7b292a2..b3e09b5 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -1,27 +1,31 @@ -import json +from __future__ import annotations + +import binascii +import contextlib +import hashlib +import ipaddress import os.path +import pathlib import shutil import socket import ssl -import sys import tempfile -import warnings -from test import ( - LONG_TIMEOUT, - SHORT_TIMEOUT, - onlyPy2, - onlyPy3, - onlySecureTransport, - withPyOpenSSL, -) +from test import LONG_TIMEOUT, SHORT_TIMEOUT, resolvesLocalhostFQDN, withPyOpenSSL +from test.conftest import ServerConfig import pytest import trustme -from dummyserver.server import DEFAULT_CA, HAS_IPV6, get_unreachable_address -from dummyserver.testcase import HTTPDummyProxyTestCase, IPv6HTTPDummyProxyTestCase +import urllib3.exceptions +from dummyserver.socketserver import DEFAULT_CA, HAS_IPV6, get_unreachable_address +from dummyserver.testcase import ( + HypercornDummyProxyTestCase, + IPv6HypercornDummyProxyTestCase, +) +from urllib3 import HTTPResponse from urllib3._collections import HTTPHeaderDict -from urllib3.connectionpool import VerifiedHTTPSConnection, connection_from_url +from urllib3.connection import VerifiedHTTPSConnection +from urllib3.connectionpool import connection_from_url from urllib3.exceptions import ( ConnectTimeoutError, InsecureRequestWarning, @@ -31,31 +35,36 @@ ProxySchemeUnsupported, ReadTimeoutError, SSLError, - SubjectAltNameWarning, ) from urllib3.poolmanager import ProxyManager, proxy_from_url -from urllib3.util import Timeout from urllib3.util.ssl_ import create_urllib3_context +from urllib3.util.timeout import Timeout from .. import TARPIT_HOST, requires_network -# Retry failed tests -pytestmark = pytest.mark.flaky +def assert_is_verified(pm: ProxyManager, *, proxy: bool, target: bool) -> None: + pool = list(pm.pools._container.values())[-1] # retrieve last pool entry + connection = ( + pool.pool.queue[-1] if pool.pool is not None else None + ) # retrieve last connection entry -class TestHTTPProxyManager(HTTPDummyProxyTestCase): + assert connection is not None + assert connection.proxy_is_verified is proxy + assert connection.is_verified is target + + +class TestHTTPProxyManager(HypercornDummyProxyTestCase): @classmethod - def setup_class(cls): - super(TestHTTPProxyManager, cls).setup_class() - cls.http_url = "http://%s:%d" % (cls.http_host, cls.http_port) - cls.http_url_alt = "http://%s:%d" % (cls.http_host_alt, cls.http_port) - cls.https_url = "https://%s:%d" % (cls.https_host, cls.https_port) - cls.https_url_alt = "https://%s:%d" % (cls.https_host_alt, cls.https_port) - cls.proxy_url = "http://%s:%d" % (cls.proxy_host, cls.proxy_port) - cls.https_proxy_url = "https://%s:%d" % ( - cls.proxy_host, - cls.https_proxy_port, - ) + def setup_class(cls) -> None: + super().setup_class() + cls.http_url = f"http://{cls.http_host}:{int(cls.http_port)}" + cls.http_url_alt = f"http://{cls.http_host_alt}:{int(cls.http_port)}" + cls.https_url = f"https://{cls.https_host}:{int(cls.https_port)}" + cls.https_url_alt = f"https://{cls.https_host_alt}:{int(cls.https_port)}" + cls.https_url_fqdn = f"https://{cls.https_host}.:{int(cls.https_port)}" + cls.proxy_url = f"http://{cls.proxy_host}:{int(cls.proxy_port)}" + cls.https_proxy_url = f"https://{cls.proxy_host}:{int(cls.https_proxy_port)}" # Generate another CA to test verification failure cls.certs_dir = tempfile.mkdtemp() @@ -65,97 +74,105 @@ def setup_class(cls): bad_ca.cert_pem.write_to_path(cls.bad_ca_path) @classmethod - def teardown_class(cls): - super(TestHTTPProxyManager, cls).teardown_class() + def teardown_class(cls) -> None: + super().teardown_class() shutil.rmtree(cls.certs_dir) - def test_basic_proxy(self): + def test_basic_proxy(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: - r = http.request("GET", "%s/" % self.http_url) + r = http.request("GET", f"{self.http_url}/") assert r.status == 200 - r = http.request("GET", "%s/" % self.https_url) + r = http.request("GET", f"{self.https_url}/") assert r.status == 200 - @onlyPy3 - def test_https_proxy(self): + def test_https_proxy(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: - r = https.request("GET", "%s/" % self.https_url) + r = https.request("GET", f"{self.https_url}/") assert r.status == 200 - r = https.request("GET", "%s/" % self.http_url) + r = https.request("GET", f"{self.http_url}/") assert r.status == 200 - @onlyPy3 - def test_https_proxy_with_proxy_ssl_context(self): - proxy_ssl_context = create_urllib3_context() - proxy_ssl_context.load_verify_locations(DEFAULT_CA) - with proxy_from_url( - self.https_proxy_url, - proxy_ssl_context=proxy_ssl_context, - ca_certs=DEFAULT_CA, - ) as https: - r = https.request("GET", "%s/" % self.https_url) + def test_is_verified_http_proxy_to_http_target(self) -> None: + with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: + r = http.request("GET", f"{self.http_url}/") assert r.status == 200 + assert_is_verified(http, proxy=False, target=False) - r = https.request("GET", "%s/" % self.http_url) + def test_is_verified_http_proxy_to_https_target(self) -> None: + with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: + r = http.request("GET", f"{self.https_url}/") assert r.status == 200 + assert_is_verified(http, proxy=False, target=True) - @onlyPy2 - def test_https_proxy_not_supported(self): + def test_is_verified_https_proxy_to_http_target(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: - r = https.request("GET", "%s/" % self.http_url) + r = https.request("GET", f"{self.http_url}/") assert r.status == 200 + assert_is_verified(https, proxy=True, target=False) - with pytest.raises(ProxySchemeUnsupported) as excinfo: - https.request("GET", "%s/" % self.https_url) + def test_is_verified_https_proxy_to_https_target(self) -> None: + with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: + r = https.request("GET", f"{self.https_url}/") + assert r.status == 200 + assert_is_verified(https, proxy=True, target=True) - assert "is not supported in Python 2" in str(excinfo.value) + def test_http_and_https_kwarg_ca_cert_data_proxy(self) -> None: + with open(DEFAULT_CA) as pem_file: + pem_file_data = pem_file.read() + with proxy_from_url(self.https_proxy_url, ca_cert_data=pem_file_data) as https: + r = https.request("GET", f"{self.https_url}/") + assert r.status == 200 - @withPyOpenSSL - @onlyPy3 - def test_https_proxy_pyopenssl_not_supported(self): - with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: - r = https.request("GET", "%s/" % self.http_url) + r = https.request("GET", f"{self.http_url}/") assert r.status == 200 - with pytest.raises(ProxySchemeUnsupported) as excinfo: - https.request("GET", "%s/" % self.https_url) + def test_https_proxy_with_proxy_ssl_context(self) -> None: + proxy_ssl_context = create_urllib3_context() + proxy_ssl_context.load_verify_locations(DEFAULT_CA) + with proxy_from_url( + self.https_proxy_url, + proxy_ssl_context=proxy_ssl_context, + ca_certs=DEFAULT_CA, + ) as https: + r = https.request("GET", f"{self.https_url}/") + assert r.status == 200 - assert "isn't available on non-native SSLContext" in str(excinfo.value) + r = https.request("GET", f"{self.http_url}/") + assert r.status == 200 - @onlySecureTransport - @onlyPy3 - def test_https_proxy_securetransport_not_supported(self): + @withPyOpenSSL + def test_https_proxy_pyopenssl_not_supported(self) -> None: with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: - r = https.request("GET", "%s/" % self.http_url) + r = https.request("GET", f"{self.http_url}/") assert r.status == 200 - with pytest.raises(ProxySchemeUnsupported) as excinfo: - https.request("GET", "%s/" % self.https_url) - - assert "isn't available on non-native SSLContext" in str(excinfo.value) + with pytest.raises( + ProxySchemeUnsupported, match="isn't available on non-native SSLContext" + ): + https.request("GET", f"{self.https_url}/") - def test_https_proxy_forwarding_for_https(self): + def test_https_proxy_forwarding_for_https(self) -> None: with proxy_from_url( self.https_proxy_url, ca_certs=DEFAULT_CA, use_forwarding_for_https=True, ) as https: - r = https.request("GET", "%s/" % self.http_url) + r = https.request("GET", f"{self.http_url}/") assert r.status == 200 - r = https.request("GET", "%s/" % self.https_url) + r = https.request("GET", f"{self.https_url}/") assert r.status == 200 - def test_nagle_proxy(self): + def test_nagle_proxy(self) -> None: """Test that proxy connections do not have TCP_NODELAY turned on""" with ProxyManager(self.proxy_url) as http: hc2 = http.connection_from_host(self.http_host, self.http_port) conn = hc2._get_conn() try: - hc2._make_request(conn, "GET", "/") - tcp_nodelay_setting = conn.sock.getsockopt( + hc2._make_request(conn, "GET", f"{self.http_url}/") + tcp_nodelay_setting = conn.sock.getsockopt( # type: ignore[attr-defined] socket.IPPROTO_TCP, socket.TCP_NODELAY ) assert tcp_nodelay_setting == 0, ( @@ -165,85 +182,89 @@ def test_nagle_proxy(self): finally: conn.close() - def test_proxy_conn_fail(self): + @pytest.mark.parametrize("proxy_scheme", ["http", "https"]) + @pytest.mark.parametrize("target_scheme", ["http", "https"]) + def test_proxy_conn_fail_from_dns( + self, proxy_scheme: str, target_scheme: str + ) -> None: host, port = get_unreachable_address() with proxy_from_url( - "http://%s:%s/" % (host, port), retries=1, timeout=LONG_TIMEOUT + f"{proxy_scheme}://{host}:{port}/", retries=1, timeout=LONG_TIMEOUT ) as http: - with pytest.raises(MaxRetryError): - http.request("GET", "%s/" % self.https_url) - with pytest.raises(MaxRetryError): - http.request("GET", "%s/" % self.http_url) + if target_scheme == "https": + target_url = self.https_url + else: + target_url = self.http_url with pytest.raises(MaxRetryError) as e: - http.request("GET", "%s/" % self.http_url) - assert type(e.value.reason) == ProxyError + http.request("GET", f"{target_url}/") + assert isinstance(e.value.reason, ProxyError) + assert isinstance( + e.value.reason.original_error, urllib3.exceptions.NameResolutionError + ) - def test_oldapi(self): + def test_oldapi(self) -> None: with ProxyManager( - connection_from_url(self.proxy_url), ca_certs=DEFAULT_CA + connection_from_url(self.proxy_url), ca_certs=DEFAULT_CA # type: ignore[arg-type] ) as http: - r = http.request("GET", "%s/" % self.http_url) + r = http.request("GET", f"{self.http_url}/") assert r.status == 200 - r = http.request("GET", "%s/" % self.https_url) + r = http.request("GET", f"{self.https_url}/") + assert r.status == 200 + + @resolvesLocalhostFQDN() + def test_proxy_https_fqdn(self) -> None: + with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: + r = http.request("GET", f"{self.https_url_fqdn}/") assert r.status == 200 - def test_proxy_verified(self): + def test_proxy_verified(self) -> None: with proxy_from_url( self.proxy_url, cert_reqs="REQUIRED", ca_certs=self.bad_ca_path ) as http: - https_pool = http._new_pool("https", self.https_host, self.https_port) - with pytest.raises(MaxRetryError) as e: - https_pool.request("GET", "/", retries=0) + with http._new_pool( + "https", self.https_host, self.https_port + ) as https_pool: + with pytest.raises(MaxRetryError) as e: + https_pool.request("GET", "/", retries=0) assert isinstance(e.value.reason, SSLError) - assert "certificate verify failed" in str(e.value.reason), ( - "Expected 'certificate verify failed', instead got: %r" % e.value.reason - ) + assert ( + "certificate verify failed" in str(e.value.reason) + # PyPy is more specific + or "self signed certificate in certificate chain" in str(e.value.reason) + ), f"Expected 'certificate verify failed', instead got: {e.value.reason!r}" http = proxy_from_url( self.proxy_url, cert_reqs="REQUIRED", ca_certs=DEFAULT_CA ) - https_pool = http._new_pool("https", self.https_host, self.https_port) - - conn = https_pool._new_conn() - assert conn.__class__ == VerifiedHTTPSConnection - https_pool.request("GET", "/") # Should succeed without exceptions. + with http._new_pool( + "https", self.https_host, self.https_port + ) as https_pool2: + with contextlib.closing(https_pool._new_conn()) as conn: + assert conn.__class__ == VerifiedHTTPSConnection + https_pool2.request( + "GET", "/" + ) # Should succeed without exceptions. http = proxy_from_url( self.proxy_url, cert_reqs="REQUIRED", ca_certs=DEFAULT_CA ) - https_fail_pool = http._new_pool("https", "127.0.0.1", self.https_port) - - with pytest.raises(MaxRetryError) as e: - https_fail_pool.request("GET", "/", retries=0) - assert isinstance(e.value.reason, SSLError) - assert "doesn't match" in str(e.value.reason) - - @onlyPy3 - def test_proxy_verified_warning(self): - """Skip proxy verification to validate warnings are generated""" - with warnings.catch_warnings(record=True) as w: - with proxy_from_url(self.https_proxy_url, cert_reqs="NONE") as https: - r = https.request("GET", "%s/" % self.https_url) - assert r.status == 200 - assert len(w) == 2 # We expect two warnings (proxy, destination) - assert w[0].category == InsecureRequestWarning - assert w[1].category == InsecureRequestWarning - messages = set(str(x.message) for x in w) - expected = [ - "Unverified HTTPS request is being made to host 'localhost'", - "Unverified HTTPS connection done to an HTTPS proxy.", - ] - for warn_message in expected: - assert [msg for msg in messages if warn_message in expected] - - def test_redirect(self): + with http._new_pool( + "https", "127.0.0.1", self.https_port + ) as https_fail_pool: + with pytest.raises( + MaxRetryError, match="doesn't match|IP address mismatch" + ) as e: + https_fail_pool.request("GET", "/", retries=0) + assert isinstance(e.value.reason, SSLError) + + def test_redirect(self) -> None: with proxy_from_url(self.proxy_url) as http: r = http.request( "GET", - "%s/redirect" % self.http_url, - fields={"target": "%s/" % self.http_url}, + f"{self.http_url}/redirect", + fields={"target": f"{self.http_url}/"}, redirect=False, ) @@ -251,183 +272,163 @@ def test_redirect(self): r = http.request( "GET", - "%s/redirect" % self.http_url, - fields={"target": "%s/" % self.http_url}, + f"{self.http_url}/redirect", + fields={"target": f"{self.http_url}/"}, ) assert r.status == 200 assert r.data == b"Dummy server!" - def test_cross_host_redirect(self): + def test_cross_host_redirect(self) -> None: with proxy_from_url(self.proxy_url) as http: - cross_host_location = "%s/echo?a=b" % self.http_url_alt + cross_host_location = f"{self.http_url_alt}/echo?a=b" with pytest.raises(MaxRetryError): http.request( "GET", - "%s/redirect" % self.http_url, + f"{self.http_url}/redirect", fields={"target": cross_host_location}, retries=0, ) r = http.request( "GET", - "%s/redirect" % self.http_url, - fields={"target": "%s/echo?a=b" % self.http_url_alt}, + f"{self.http_url}/redirect", + fields={"target": f"{self.http_url_alt}/echo?a=b"}, retries=1, ) + assert isinstance(r, HTTPResponse) + assert r._pool is not None assert r._pool.host != self.http_host_alt - def test_cross_protocol_redirect(self): + def test_cross_protocol_redirect(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: - cross_protocol_location = "%s/echo?a=b" % self.https_url + cross_protocol_location = f"{self.https_url}/echo?a=b" with pytest.raises(MaxRetryError): http.request( "GET", - "%s/redirect" % self.http_url, + f"{self.http_url}/redirect", fields={"target": cross_protocol_location}, retries=0, ) r = http.request( "GET", - "%s/redirect" % self.http_url, - fields={"target": "%s/echo?a=b" % self.https_url}, + f"{self.http_url}/redirect", + fields={"target": f"{self.https_url}/echo?a=b"}, retries=1, ) + assert isinstance(r, HTTPResponse) + assert r._pool is not None assert r._pool.host == self.https_host - def test_headers(self): + def test_headers(self) -> None: with proxy_from_url( self.proxy_url, headers={"Foo": "bar"}, proxy_headers={"Hickory": "dickory"}, ca_certs=DEFAULT_CA, ) as http: - - r = http.request_encode_url("GET", "%s/headers" % self.http_url) - returned_headers = json.loads(r.data.decode()) + r = http.request_encode_url("GET", f"{self.http_url}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Hickory") == "dickory" - assert returned_headers.get("Host") == "%s:%s" % ( - self.http_host, - self.http_port, - ) + assert returned_headers.get("Host") == f"{self.http_host}:{self.http_port}" - r = http.request_encode_url("GET", "%s/headers" % self.http_url_alt) - returned_headers = json.loads(r.data.decode()) + r = http.request_encode_url("GET", f"{self.http_url_alt}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Hickory") == "dickory" - assert returned_headers.get("Host") == "%s:%s" % ( - self.http_host_alt, - self.http_port, + assert ( + returned_headers.get("Host") == f"{self.http_host_alt}:{self.http_port}" ) - r = http.request_encode_url("GET", "%s/headers" % self.https_url) - returned_headers = json.loads(r.data.decode()) + r = http.request_encode_url("GET", f"{self.https_url}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Hickory") is None - assert returned_headers.get("Host") == "%s:%s" % ( - self.https_host, - self.https_port, + assert ( + returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) - r = http.request_encode_body("POST", "%s/headers" % self.http_url) - returned_headers = json.loads(r.data.decode()) + r = http.request_encode_body("POST", f"{self.http_url}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Hickory") == "dickory" - assert returned_headers.get("Host") == "%s:%s" % ( - self.http_host, - self.http_port, - ) + assert returned_headers.get("Host") == f"{self.http_host}:{self.http_port}" r = http.request_encode_url( - "GET", "%s/headers" % self.http_url, headers={"Baz": "quux"} + "GET", f"{self.http_url}/headers", headers={"Baz": "quux"} ) - returned_headers = json.loads(r.data.decode()) + returned_headers = r.json() assert returned_headers.get("Foo") is None assert returned_headers.get("Baz") == "quux" assert returned_headers.get("Hickory") == "dickory" - assert returned_headers.get("Host") == "%s:%s" % ( - self.http_host, - self.http_port, - ) + assert returned_headers.get("Host") == f"{self.http_host}:{self.http_port}" r = http.request_encode_url( - "GET", "%s/headers" % self.https_url, headers={"Baz": "quux"} + "GET", f"{self.https_url}/headers", headers={"Baz": "quux"} ) - returned_headers = json.loads(r.data.decode()) + returned_headers = r.json() assert returned_headers.get("Foo") is None assert returned_headers.get("Baz") == "quux" assert returned_headers.get("Hickory") is None - assert returned_headers.get("Host") == "%s:%s" % ( - self.https_host, - self.https_port, + assert ( + returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) r = http.request_encode_body( - "GET", "%s/headers" % self.http_url, headers={"Baz": "quux"} + "GET", f"{self.http_url}/headers", headers={"Baz": "quux"} ) - returned_headers = json.loads(r.data.decode()) + returned_headers = r.json() assert returned_headers.get("Foo") is None assert returned_headers.get("Baz") == "quux" assert returned_headers.get("Hickory") == "dickory" - assert returned_headers.get("Host") == "%s:%s" % ( - self.http_host, - self.http_port, - ) + assert returned_headers.get("Host") == f"{self.http_host}:{self.http_port}" r = http.request_encode_body( - "GET", "%s/headers" % self.https_url, headers={"Baz": "quux"} + "GET", f"{self.https_url}/headers", headers={"Baz": "quux"} ) - returned_headers = json.loads(r.data.decode()) + returned_headers = r.json() assert returned_headers.get("Foo") is None assert returned_headers.get("Baz") == "quux" assert returned_headers.get("Hickory") is None - assert returned_headers.get("Host") == "%s:%s" % ( - self.https_host, - self.https_port, + assert ( + returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) - @onlyPy3 - def test_https_headers(self): + def test_https_headers(self) -> None: with proxy_from_url( self.https_proxy_url, headers={"Foo": "bar"}, proxy_headers={"Hickory": "dickory"}, ca_certs=DEFAULT_CA, ) as http: - - r = http.request_encode_url("GET", "%s/headers" % self.http_url) - returned_headers = json.loads(r.data.decode()) + r = http.request_encode_url("GET", f"{self.http_url}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Hickory") == "dickory" - assert returned_headers.get("Host") == "%s:%s" % ( - self.http_host, - self.http_port, - ) + assert returned_headers.get("Host") == f"{self.http_host}:{self.http_port}" - r = http.request_encode_url("GET", "%s/headers" % self.http_url_alt) - returned_headers = json.loads(r.data.decode()) + r = http.request_encode_url("GET", f"{self.http_url_alt}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Hickory") == "dickory" - assert returned_headers.get("Host") == "%s:%s" % ( - self.http_host_alt, - self.http_port, + assert ( + returned_headers.get("Host") == f"{self.http_host_alt}:{self.http_port}" ) r = http.request_encode_body( - "GET", "%s/headers" % self.https_url, headers={"Baz": "quux"} + "GET", f"{self.https_url}/headers", headers={"Baz": "quux"} ) - returned_headers = json.loads(r.data.decode()) + returned_headers = r.json() assert returned_headers.get("Foo") is None assert returned_headers.get("Baz") == "quux" assert returned_headers.get("Hickory") is None - assert returned_headers.get("Host") == "%s:%s" % ( - self.https_host, - self.https_port, + assert ( + returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) - def test_https_headers_forwarding_for_https(self): + def test_https_headers_forwarding_for_https(self) -> None: with proxy_from_url( self.https_proxy_url, headers={"Foo": "bar"}, @@ -435,17 +436,15 @@ def test_https_headers_forwarding_for_https(self): ca_certs=DEFAULT_CA, use_forwarding_for_https=True, ) as http: - - r = http.request_encode_url("GET", "%s/headers" % self.https_url) - returned_headers = json.loads(r.data.decode()) + r = http.request_encode_url("GET", f"{self.https_url}/headers") + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Hickory") == "dickory" - assert returned_headers.get("Host") == "%s:%s" % ( - self.https_host, - self.https_port, + assert ( + returned_headers.get("Host") == f"{self.https_host}:{self.https_port}" ) - def test_headerdict(self): + def test_headerdict(self) -> None: default_headers = HTTPHeaderDict(a="b") proxy_headers = HTTPHeaderDict() proxy_headers.add("foo", "bar") @@ -454,14 +453,12 @@ def test_headerdict(self): self.proxy_url, headers=default_headers, proxy_headers=proxy_headers ) as http: request_headers = HTTPHeaderDict(baz="quux") - r = http.request( - "GET", "%s/headers" % self.http_url, headers=request_headers - ) - returned_headers = json.loads(r.data.decode()) + r = http.request("GET", f"{self.http_url}/headers", headers=request_headers) + returned_headers = r.json() assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Baz") == "quux" - def test_proxy_pooling(self): + def test_proxy_pooling(self) -> None: with proxy_from_url(self.proxy_url, cert_reqs="NONE") as http: for x in range(2): http.urlopen("GET", self.http_url) @@ -472,14 +469,16 @@ def test_proxy_pooling(self): assert len(http.pools) == 1 for x in range(2): - http.urlopen("GET", self.https_url) + with pytest.warns(InsecureRequestWarning): + http.urlopen("GET", self.https_url) assert len(http.pools) == 2 for x in range(2): - http.urlopen("GET", self.https_url_alt) + with pytest.warns(InsecureRequestWarning): + http.urlopen("GET", self.https_url_alt) assert len(http.pools) == 3 - def test_proxy_pooling_ext(self): + def test_proxy_pooling_ext(self) -> None: with proxy_from_url(self.proxy_url) as http: hc1 = http.connection_from_url(self.http_url) hc2 = http.connection_from_host(self.http_host, self.http_port) @@ -501,7 +500,7 @@ def test_proxy_pooling_ext(self): assert sc2 != sc3 assert sc3 == sc4 - @requires_network + @requires_network() @pytest.mark.parametrize( ["proxy_scheme", "target_scheme", "use_forwarding_for_https"], [ @@ -512,14 +511,10 @@ def test_proxy_pooling_ext(self): ], ) def test_forwarding_proxy_request_timeout( - self, proxy_scheme, target_scheme, use_forwarding_for_https - ): - _should_skip_https_in_https( - proxy_scheme, target_scheme, use_forwarding_for_https - ) - + self, proxy_scheme: str, target_scheme: str, use_forwarding_for_https: bool + ) -> None: proxy_url = self.https_proxy_url if proxy_scheme == "https" else self.proxy_url - target_url = "%s://%s" % (target_scheme, TARPIT_HOST) + target_url = f"{target_scheme}://{TARPIT_HOST}" with proxy_from_url( proxy_url, @@ -533,17 +528,17 @@ def test_forwarding_proxy_request_timeout( # We sent the request to the proxy but didn't get any response # so we're not sure if that's being caused by the proxy or the # target so we put the blame on the target. - assert type(e.value.reason) == ReadTimeoutError + assert isinstance(e.value.reason, ReadTimeoutError) - @requires_network + @requires_network() @pytest.mark.parametrize( ["proxy_scheme", "target_scheme"], [("http", "https"), ("https", "https")] ) - def test_tunneling_proxy_request_timeout(self, proxy_scheme, target_scheme): - _should_skip_https_in_https(proxy_scheme, target_scheme) - + def test_tunneling_proxy_request_timeout( + self, proxy_scheme: str, target_scheme: str + ) -> None: proxy_url = self.https_proxy_url if proxy_scheme == "https" else self.proxy_url - target_url = "%s://%s" % (target_scheme, TARPIT_HOST) + target_url = f"{target_scheme}://{TARPIT_HOST}" with proxy_from_url( proxy_url, @@ -553,10 +548,9 @@ def test_tunneling_proxy_request_timeout(self, proxy_scheme, target_scheme): timeout = Timeout(connect=LONG_TIMEOUT, read=SHORT_TIMEOUT) proxy.request("GET", target_url, timeout=timeout) - assert type(e.value.reason) == ProxyError - assert type(e.value.reason.original_error) == socket.timeout + assert isinstance(e.value.reason, ReadTimeoutError) - @requires_network + @requires_network() @pytest.mark.parametrize( ["proxy_scheme", "target_scheme", "use_forwarding_for_https"], [ @@ -567,13 +561,9 @@ def test_tunneling_proxy_request_timeout(self, proxy_scheme, target_scheme): ], ) def test_forwarding_proxy_connect_timeout( - self, proxy_scheme, target_scheme, use_forwarding_for_https - ): - _should_skip_https_in_https( - proxy_scheme, target_scheme, use_forwarding_for_https - ) - - proxy_url = "%s://%s" % (proxy_scheme, TARPIT_HOST) + self, proxy_scheme: str, target_scheme: str, use_forwarding_for_https: bool + ) -> None: + proxy_url = f"{proxy_scheme}://{TARPIT_HOST}" target_url = self.https_url if target_scheme == "https" else self.http_url with proxy_from_url( @@ -585,16 +575,17 @@ def test_forwarding_proxy_connect_timeout( with pytest.raises(MaxRetryError) as e: proxy.request("GET", target_url) - assert type(e.value.reason) == ConnectTimeoutError + assert isinstance(e.value.reason, ProxyError) + assert isinstance(e.value.reason.original_error, ConnectTimeoutError) - @requires_network + @requires_network() @pytest.mark.parametrize( ["proxy_scheme", "target_scheme"], [("http", "https"), ("https", "https")] ) - def test_tunneling_proxy_connect_timeout(self, proxy_scheme, target_scheme): - _should_skip_https_in_https(proxy_scheme, target_scheme) - - proxy_url = "%s://%s" % (proxy_scheme, TARPIT_HOST) + def test_tunneling_proxy_connect_timeout( + self, proxy_scheme: str, target_scheme: str + ) -> None: + proxy_url = f"{proxy_scheme}://{TARPIT_HOST}" target_url = self.https_url if target_scheme == "https" else self.http_url with proxy_from_url( @@ -603,9 +594,10 @@ def test_tunneling_proxy_connect_timeout(self, proxy_scheme, target_scheme): with pytest.raises(MaxRetryError) as e: proxy.request("GET", target_url) - assert type(e.value.reason) == ConnectTimeoutError + assert isinstance(e.value.reason, ProxyError) + assert isinstance(e.value.reason.original_error, ConnectTimeoutError) - @requires_network + @requires_network() @pytest.mark.parametrize( ["target_scheme", "use_forwarding_for_https"], [ @@ -614,9 +606,9 @@ def test_tunneling_proxy_connect_timeout(self, proxy_scheme, target_scheme): ("https", True), ], ) - def test_https_proxy_tls_error(self, target_scheme, use_forwarding_for_https): - _should_skip_https_in_https("https", target_scheme, use_forwarding_for_https) - + def test_https_proxy_tls_error( + self, target_scheme: str, use_forwarding_for_https: str + ) -> None: target_url = self.https_url if target_scheme == "https" else self.http_url proxy_ctx = ssl.create_default_context() with proxy_from_url( @@ -626,9 +618,10 @@ def test_https_proxy_tls_error(self, target_scheme, use_forwarding_for_https): ) as proxy: with pytest.raises(MaxRetryError) as e: proxy.request("GET", target_url) - assert type(e.value.reason) == SSLError + assert isinstance(e.value.reason, ProxyError) + assert isinstance(e.value.reason.original_error, SSLError) - @requires_network + @requires_network() @pytest.mark.parametrize( ["proxy_scheme", "use_forwarding_for_https"], [ @@ -637,8 +630,11 @@ def test_https_proxy_tls_error(self, target_scheme, use_forwarding_for_https): ("https", True), ], ) - def test_proxy_https_target_tls_error(self, proxy_scheme, use_forwarding_for_https): - _should_skip_https_in_https(proxy_scheme, "https") + def test_proxy_https_target_tls_error( + self, proxy_scheme: str, use_forwarding_for_https: str + ) -> None: + if proxy_scheme == "https" and use_forwarding_for_https: + pytest.skip("Test is expected to fail due to urllib3/urllib3#2577") proxy_url = self.https_proxy_url if proxy_scheme == "https" else self.proxy_url proxy_ctx = ssl.create_default_context() @@ -653,15 +649,15 @@ def test_proxy_https_target_tls_error(self, proxy_scheme, use_forwarding_for_htt ) as proxy: with pytest.raises(MaxRetryError) as e: proxy.request("GET", self.https_url) - assert type(e.value.reason) == SSLError + assert isinstance(e.value.reason, SSLError) - def test_scheme_host_case_insensitive(self): + def test_scheme_host_case_insensitive(self) -> None: """Assert that upper-case schemes and hosts are normalized.""" with proxy_from_url(self.proxy_url.upper(), ca_certs=DEFAULT_CA) as http: - r = http.request("GET", "%s/" % self.http_url.upper()) + r = http.request("GET", f"{self.http_url.upper()}/") assert r.status == 200 - r = http.request("GET", "%s/" % self.https_url.upper()) + r = http.request("GET", f"{self.https_url.upper()}/") assert r.status == 200 @pytest.mark.parametrize( @@ -681,92 +677,209 @@ def test_scheme_host_case_insensitive(self): ), ], ) - def test_invalid_schema(self, url, error_msg): + def test_invalid_schema(self, url: str, error_msg: str) -> None: with pytest.raises(ProxySchemeUnknown, match=error_msg): proxy_from_url(url) @pytest.mark.skipif(not HAS_IPV6, reason="Only runs on IPv6 systems") -class TestIPv6HTTPProxyManager(IPv6HTTPDummyProxyTestCase): +class TestIPv6HTTPProxyManager(IPv6HypercornDummyProxyTestCase): @classmethod - def setup_class(cls): - HTTPDummyProxyTestCase.setup_class() - cls.http_url = "http://%s:%d" % (cls.http_host, cls.http_port) - cls.http_url_alt = "http://%s:%d" % (cls.http_host_alt, cls.http_port) - cls.https_url = "https://%s:%d" % (cls.https_host, cls.https_port) - cls.https_url_alt = "https://%s:%d" % (cls.https_host_alt, cls.https_port) - cls.proxy_url = "http://[%s]:%d" % (cls.proxy_host, cls.proxy_port) - - def test_basic_ipv6_proxy(self): + def setup_class(cls) -> None: + super().setup_class() + cls.http_url = f"http://{cls.http_host}:{int(cls.http_port)}" + cls.http_url_alt = f"http://{cls.http_host_alt}:{int(cls.http_port)}" + cls.https_url = f"https://{cls.https_host}:{int(cls.https_port)}" + cls.https_url_alt = f"https://{cls.https_host_alt}:{int(cls.https_port)}" + cls.proxy_url = f"http://[{cls.proxy_host}]:{int(cls.proxy_port)}" + + def test_basic_ipv6_proxy(self) -> None: with proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA) as http: - r = http.request("GET", "%s/" % self.http_url) + r = http.request("GET", f"{self.http_url}/") assert r.status == 200 - r = http.request("GET", "%s/" % self.https_url) + r = http.request("GET", f"{self.https_url}/") assert r.status == 200 class TestHTTPSProxyVerification: - @onlyPy3 - def test_https_proxy_hostname_verification(self, no_localhost_san_server): + @staticmethod + def _get_proxy_fingerprint_md5(ca_path: str) -> str: + proxy_pem_path = pathlib.Path(ca_path).parent / "proxy.pem" + proxy_der = ssl.PEM_cert_to_DER_cert(proxy_pem_path.read_text()) + proxy_hashed = hashlib.md5(proxy_der).digest() + fingerprint = binascii.hexlify(proxy_hashed).decode("ascii") + return fingerprint + + @staticmethod + def _get_certificate_formatted_proxy_host(host: str) -> str: + try: + addr = ipaddress.ip_address(host) + except ValueError: + return host + + if addr.version != 6: + return host + + # Transform ipv6 like '::1' to 0:0:0:0:0:0:0:1 via '0000:0000:0000:0000:0000:0000:0000:0001' + return addr.exploded.replace("0000", "0").replace("000", "") + + def test_https_proxy_assert_fingerprint_md5( + self, no_san_proxy_with_server: tuple[ServerConfig, ServerConfig] + ) -> None: + proxy, server = no_san_proxy_with_server + proxy_url = f"https://{proxy.host}:{proxy.port}" + destination_url = f"https://{server.host}:{server.port}" + + proxy_fingerprint = self._get_proxy_fingerprint_md5(proxy.ca_certs) + with proxy_from_url( + proxy_url, + ca_certs=proxy.ca_certs, + proxy_assert_fingerprint=proxy_fingerprint, + ) as https: + https.request("GET", destination_url) + + def test_https_proxy_assert_fingerprint_md5_non_matching( + self, no_san_proxy_with_server: tuple[ServerConfig, ServerConfig] + ) -> None: + proxy, server = no_san_proxy_with_server + proxy_url = f"https://{proxy.host}:{proxy.port}" + destination_url = f"https://{server.host}:{server.port}" + + proxy_fingerprint = self._get_proxy_fingerprint_md5(proxy.ca_certs) + new_char = "b" if proxy_fingerprint[5] == "a" else "a" + proxy_fingerprint = proxy_fingerprint[:5] + new_char + proxy_fingerprint[6:] + + with proxy_from_url( + proxy_url, + ca_certs=proxy.ca_certs, + proxy_assert_fingerprint=proxy_fingerprint, + ) as https: + with pytest.raises(MaxRetryError) as e: + https.request("GET", destination_url) + + assert "Fingerprints did not match" in str(e) + + def test_https_proxy_assert_hostname( + self, san_proxy_with_server: tuple[ServerConfig, ServerConfig] + ) -> None: + proxy, server = san_proxy_with_server + destination_url = f"https://{server.host}:{server.port}" + + with proxy_from_url( + proxy.base_url, ca_certs=proxy.ca_certs, proxy_assert_hostname=proxy.host + ) as https: + https.request("GET", destination_url) + + def test_https_proxy_assert_hostname_non_matching( + self, san_proxy_with_server: tuple[ServerConfig, ServerConfig] + ) -> None: + proxy, server = san_proxy_with_server + destination_url = f"https://{server.host}:{server.port}" + + proxy_hostname = "example.com" + with proxy_from_url( + proxy.base_url, + ca_certs=proxy.ca_certs, + proxy_assert_hostname=proxy_hostname, + ) as https: + with pytest.raises(MaxRetryError) as e: + https.request("GET", destination_url) + + proxy_host = self._get_certificate_formatted_proxy_host(proxy.host) + msg = f"hostname \\'{proxy_hostname}\\' doesn\\'t match \\'{proxy_host}\\'" + assert msg in str(e) + + def test_https_proxy_hostname_verification( + self, no_localhost_san_server: ServerConfig + ) -> None: bad_server = no_localhost_san_server - bad_proxy_url = "https://%s:%s" % (bad_server.host, bad_server.port) + bad_proxy_url = f"https://{bad_server.host}:{bad_server.port}" # An exception will be raised before we contact the destination domain. test_url = "testing.com" with proxy_from_url(bad_proxy_url, ca_certs=bad_server.ca_certs) as https: with pytest.raises(MaxRetryError) as e: https.request("GET", "http://%s/" % test_url) - assert isinstance(e.value.reason, SSLError) - assert "hostname 'localhost' doesn't match" in str(e.value.reason) + assert isinstance(e.value.reason, ProxyError) + + ssl_error = e.value.reason.original_error + assert isinstance(ssl_error, SSLError) + assert "hostname 'localhost' doesn't match" in str( + ssl_error + ) or "Hostname mismatch" in str(ssl_error) with pytest.raises(MaxRetryError) as e: https.request("GET", "https://%s/" % test_url) - assert isinstance(e.value.reason, SSLError) + assert isinstance(e.value.reason, ProxyError) + + ssl_error = e.value.reason.original_error + assert isinstance(ssl_error, SSLError) assert "hostname 'localhost' doesn't match" in str( - e.value.reason - ) or "Hostname mismatch" in str(e.value.reason) - - @onlyPy3 - def test_https_proxy_ipv4_san(self, ipv4_san_proxy): - proxy, server = ipv4_san_proxy - proxy_url = "https://%s:%s" % (proxy.host, proxy.port) - destination_url = "https://%s:%s" % (server.host, server.port) + ssl_error + ) or "Hostname mismatch" in str(ssl_error) + + def test_https_proxy_ipv4_san( + self, ipv4_san_proxy_with_server: tuple[ServerConfig, ServerConfig] + ) -> None: + proxy, server = ipv4_san_proxy_with_server + proxy_url = f"https://{proxy.host}:{proxy.port}" + destination_url = f"https://{server.host}:{server.port}" with proxy_from_url(proxy_url, ca_certs=proxy.ca_certs) as https: r = https.request("GET", destination_url) assert r.status == 200 - @onlyPy3 - def test_https_proxy_ipv6_san(self, ipv6_san_proxy): - proxy, server = ipv6_san_proxy - proxy_url = "https://[%s]:%s" % (proxy.host, proxy.port) - destination_url = "https://%s:%s" % (server.host, server.port) + def test_https_proxy_ipv6_san( + self, ipv6_san_proxy_with_server: tuple[ServerConfig, ServerConfig] + ) -> None: + proxy, server = ipv6_san_proxy_with_server + proxy_url = f"https://[{proxy.host}]:{proxy.port}" + destination_url = f"https://{server.host}:{server.port}" with proxy_from_url(proxy_url, ca_certs=proxy.ca_certs) as https: r = https.request("GET", destination_url) assert r.status == 200 - @onlyPy3 - def test_https_proxy_common_name_warning(self, no_san_proxy): - proxy, server = no_san_proxy - proxy_url = "https://%s:%s" % (proxy.host, proxy.port) - destination_url = "https://%s:%s" % (server.host, server.port) - - with warnings.catch_warnings(record=True) as w: - with proxy_from_url(proxy_url, ca_certs=proxy.ca_certs) as https: - r = https.request("GET", destination_url) - assert r.status == 200 - - assert len(w) == 1 - assert w[0].category == SubjectAltNameWarning - - -def _should_skip_https_in_https( - proxy_scheme, target_scheme, use_forwarding_for_https=False -): - if ( - sys.version_info[0] == 2 - and proxy_scheme == "https" - and target_scheme == "https" - and use_forwarding_for_https is False - ): - pytest.skip("HTTPS-in-HTTPS isn't supported on Python 2") + @pytest.mark.parametrize("target_scheme", ["http", "https"]) + def test_https_proxy_no_san( + self, + no_san_proxy_with_server: tuple[ServerConfig, ServerConfig], + target_scheme: str, + ) -> None: + proxy, server = no_san_proxy_with_server + proxy_url = f"https://{proxy.host}:{proxy.port}" + destination_url = f"{target_scheme}://{server.host}:{server.port}" + + with proxy_from_url(proxy_url, ca_certs=proxy.ca_certs) as https: + with pytest.raises(MaxRetryError) as e: + https.request("GET", destination_url) + assert isinstance(e.value.reason, ProxyError) + + ssl_error = e.value.reason.original_error + assert isinstance(ssl_error, SSLError) + assert "no appropriate subjectAltName fields were found" in str( + ssl_error + ) or "Hostname mismatch, certificate is not valid for 'localhost'" in str( + ssl_error + ) + + def test_https_proxy_no_san_hostname_checks_common_name( + self, no_san_proxy_with_server: tuple[ServerConfig, ServerConfig] + ) -> None: + proxy, server = no_san_proxy_with_server + proxy_url = f"https://{proxy.host}:{proxy.port}" + destination_url = f"https://{server.host}:{server.port}" + + proxy_ctx = urllib3.util.ssl_.create_urllib3_context() + try: + proxy_ctx.hostname_checks_common_name = True + # PyPy doesn't like us setting 'hostname_checks_common_name' + # but also has it enabled by default so we need to handle that. + except AttributeError: + pass + if getattr(proxy_ctx, "hostname_checks_common_name", False) is not True: + pytest.skip("Test requires 'SSLContext.hostname_checks_common_name=True'") + + with proxy_from_url( + proxy_url, ca_certs=proxy.ca_certs, proxy_ssl_context=proxy_ctx + ) as https: + https.request("GET", destination_url) diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 9ee3dff..89db60b 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1,70 +1,72 @@ # TODO: Break this module up into pieces. Maybe group by functionality tested # rather than the socket level-ness of it. -from dummyserver.server import ( +from __future__ import annotations + +import contextlib +import errno +import http.client +import io +import os +import os.path +import select +import shutil +import socket +import ssl +import tempfile +import threading +import time +import typing +import zlib +from collections import OrderedDict +from pathlib import Path +from test import LONG_TIMEOUT, SHORT_TIMEOUT, notWindows, resolvesLocalhostFQDN +from threading import Event +from unittest import mock + +import pytest +import trustme + +from dummyserver.socketserver import ( DEFAULT_CA, DEFAULT_CERTS, encrypt_key_pem, get_unreachable_address, ) from dummyserver.testcase import SocketDummyServerTestCase, consume_socket -from urllib3 import HTTPConnectionPool, HTTPSConnectionPool, ProxyManager, util +from urllib3 import ( + BaseHTTPResponse, + HTTPConnectionPool, + HTTPSConnectionPool, + ProxyManager, + util, +) from urllib3._collections import HTTPHeaderDict from urllib3.connection import HTTPConnection, _get_default_user_agent +from urllib3.connectionpool import _url_from_pool from urllib3.exceptions import ( + InsecureRequestWarning, MaxRetryError, ProtocolError, ProxyError, ReadTimeoutError, SSLError, ) -from urllib3.packages.six.moves import http_client as httplib from urllib3.poolmanager import proxy_from_url from urllib3.util import ssl_, ssl_wrap_socket from urllib3.util.retry import Retry from urllib3.util.timeout import Timeout -from .. import LogRecorder, has_alpn, onlyPy3 - -try: - from mimetools import Message as MimeToolMessage -except ImportError: +from .. import LogRecorder - class MimeToolMessage(object): - pass - - -import os -import os.path -import platform -import select -import shutil -import socket -import ssl -import sys -import tempfile -from collections import OrderedDict -from test import ( - LONG_TIMEOUT, - SHORT_TIMEOUT, - notPyPy2, - notSecureTransport, - notWindows, - requires_ssl_context_keyfile_password, - resolvesLocalhostFQDN, -) -from threading import Event - -import mock -import pytest -import trustme - -# Retry failed tests -pytestmark = pytest.mark.flaky +if typing.TYPE_CHECKING: + from _typeshed import StrOrBytesPath +else: + StrOrBytesPath = object class TestCookies(SocketDummyServerTestCase): - def test_multi_setcookie(self): - def multicookie_response_handler(listener): + def test_multi_setcookie(self) -> None: + def multicookie_response_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -87,13 +89,11 @@ def multicookie_response_handler(listener): class TestSNI(SocketDummyServerTestCase): - def test_hostname_in_first_request_packet(self): - if not util.HAS_SNI: - pytest.skip("SNI-support not available") + def test_hostname_in_first_request_packet(self) -> None: done_receiving = Event() self.buf = b"" - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] self.buf = sock.recv(65536) # We only accept one packet @@ -114,14 +114,11 @@ def socket_handler(listener): class TestALPN(SocketDummyServerTestCase): - def test_alpn_protocol_in_first_request_packet(self): - if not has_alpn(): - pytest.skip("ALPN-support not available") - + def test_alpn_protocol_in_first_request_packet(self) -> None: done_receiving = Event() self.buf = b"" - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] self.buf = sock.recv(65536) # We only accept one packet @@ -142,16 +139,48 @@ def socket_handler(listener): ), "missing ALPN protocol in SSL handshake" +def original_ssl_wrap_socket( + sock: socket.socket, + keyfile: StrOrBytesPath | None = None, + certfile: StrOrBytesPath | None = None, + server_side: bool = False, + cert_reqs: ssl.VerifyMode = ssl.CERT_NONE, + ssl_version: int = ssl.PROTOCOL_TLS, + ca_certs: str | None = None, + do_handshake_on_connect: bool = True, + suppress_ragged_eofs: bool = True, + ciphers: str | None = None, +) -> ssl.SSLSocket: + if server_side and not certfile: + raise ValueError("certfile must be specified for server-side operations") + if keyfile and not certfile: + raise ValueError("certfile must be specified") + context = ssl.SSLContext(ssl_version) + context.verify_mode = cert_reqs + if ca_certs: + context.load_verify_locations(ca_certs) + if certfile: + context.load_cert_chain(certfile, keyfile) + if ciphers: + context.set_ciphers(ciphers) + return context.wrap_socket( + sock=sock, + server_side=server_side, + do_handshake_on_connect=do_handshake_on_connect, + suppress_ragged_eofs=suppress_ragged_eofs, + ) + + class TestClientCerts(SocketDummyServerTestCase): """ Tests for client certificate support. """ @classmethod - def setup_class(cls): + def setup_class(cls) -> None: cls.tmpdir = tempfile.mkdtemp() ca = trustme.CA() - cert = ca.issue_cert(u"localhost") + cert = ca.issue_cert("localhost") encrypted_key = encrypt_key_pem(cert.private_key_pem, b"letmein") cls.ca_path = os.path.join(cls.tmpdir, "ca.pem") @@ -166,14 +195,15 @@ def setup_class(cls): cert.private_key_pem.write_to_path(cls.key_path) encrypted_key.write_to_path(cls.password_key_path) - def teardown_class(cls): + @classmethod + def teardown_class(cls) -> None: shutil.rmtree(cls.tmpdir) - def _wrap_in_ssl(self, sock): + def _wrap_in_ssl(self, sock: socket.socket) -> ssl.SSLSocket: """ Given a single socket, wraps it in TLS. """ - return ssl.wrap_socket( + return original_ssl_wrap_socket( sock, ssl_version=ssl.PROTOCOL_SSLv23, cert_reqs=ssl.CERT_REQUIRED, @@ -183,7 +213,7 @@ def _wrap_in_ssl(self, sock): server_side=True, ) - def test_client_certs_two_files(self): + def test_client_certs_two_files(self) -> None: """ Having a client cert in a separate file to its associated key works properly. @@ -191,7 +221,7 @@ def test_client_certs_two_files(self): done_receiving = Event() client_certs = [] - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] sock = self._wrap_in_ssl(sock) @@ -227,7 +257,7 @@ def socket_handler(listener): assert len(client_certs) == 1 - def test_client_certs_one_file(self): + def test_client_certs_one_file(self) -> None: """ Having a client cert and its associated private key in just one file works properly. @@ -235,7 +265,7 @@ def test_client_certs_one_file(self): done_receiving = Event() client_certs = [] - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] sock = self._wrap_in_ssl(sock) @@ -270,13 +300,13 @@ def socket_handler(listener): assert len(client_certs) == 1 - def test_missing_client_certs_raises_error(self): + def test_missing_client_certs_raises_error(self) -> None: """ Having client certs not be present causes an error. """ done_receiving = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] try: @@ -296,22 +326,20 @@ def socket_handler(listener): done_receiving.set() done_receiving.set() - @requires_ssl_context_keyfile_password - def test_client_cert_with_string_password(self): - self.run_client_cert_with_password_test(u"letmein") + def test_client_cert_with_string_password(self) -> None: + self.run_client_cert_with_password_test("letmein") - @requires_ssl_context_keyfile_password - def test_client_cert_with_bytes_password(self): + def test_client_cert_with_bytes_password(self) -> None: self.run_client_cert_with_password_test(b"letmein") - def run_client_cert_with_password_test(self, password): + def run_client_cert_with_password_test(self, password: bytes | str) -> None: """ Tests client certificate password functionality """ done_receiving = Event() client_certs = [] - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] sock = self._wrap_in_ssl(sock) @@ -334,6 +362,7 @@ def socket_handler(listener): sock.close() self._start_server(socket_handler) + assert ssl_.SSLContext is not None ssl_context = ssl_.SSLContext(ssl_.PROTOCOL_SSLv23) ssl_context.load_cert_chain( certfile=self.cert_path, keyfile=self.password_key_path, password=password @@ -351,35 +380,32 @@ def socket_handler(listener): assert len(client_certs) == 1 - @requires_ssl_context_keyfile_password - def test_load_keyfile_with_invalid_password(self): + def test_load_keyfile_with_invalid_password(self) -> None: + assert ssl_.SSLContext is not None context = ssl_.SSLContext(ssl_.PROTOCOL_SSLv23) - - # Different error is raised depending on context. - if ssl_.IS_PYOPENSSL: - from OpenSSL.SSL import Error - - expected_error = Error - else: - expected_error = ssl.SSLError - - with pytest.raises(expected_error): + with pytest.raises(ssl.SSLError): context.load_cert_chain( certfile=self.cert_path, keyfile=self.password_key_path, password=b"letmei", ) + def test_load_invalid_cert_file(self) -> None: + assert ssl_.SSLContext is not None + context = ssl_.SSLContext(ssl_.PROTOCOL_SSLv23) + with pytest.raises(ssl.SSLError): + context.load_cert_chain(certfile=self.password_key_path) + class TestSocketClosing(SocketDummyServerTestCase): - def test_recovery_when_server_closes_connection(self): + def test_recovery_when_server_closes_connection(self) -> None: # Does the pool work seamlessly if an open connection in the # connection pool gets hung up on by the server, then reaches # the front of the queue again? done_closing = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: for i in 0, 1: sock = listener.accept()[0] @@ -387,7 +413,7 @@ def socket_handler(listener): while not buf.endswith(b"\r\n\r\n"): buf = sock.recv(65536) - body = "Response %d" % i + body = f"Response {int(i)}" sock.send( ( "HTTP/1.1 200 OK\r\n" @@ -413,18 +439,19 @@ def socket_handler(listener): assert response.status == 200 assert response.data == b"Response 1" - def test_connection_refused(self): + def test_connection_refused(self) -> None: # Does the pool retry if there is no listener on the port? host, port = get_unreachable_address() with HTTPConnectionPool(host, port, maxsize=3, block=True) as http: with pytest.raises(MaxRetryError): http.request("GET", "/", retries=0, release_conn=False) + assert http.pool is not None assert http.pool.qsize() == http.pool.maxsize - def test_connection_read_timeout(self): + def test_connection_read_timeout(self) -> None: timed_out = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] while not sock.recv(65536).endswith(b"\r\n\r\n"): pass @@ -447,12 +474,13 @@ def socket_handler(listener): finally: timed_out.set() + assert http.pool is not None assert http.pool.qsize() == http.pool.maxsize - def test_read_timeout_dont_retry_method_not_in_allowlist(self): + def test_read_timeout_dont_retry_method_not_in_allowlist(self) -> None: timed_out = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] sock.recv(65536) timed_out.wait() @@ -468,11 +496,11 @@ def socket_handler(listener): finally: timed_out.set() - def test_https_connection_read_timeout(self): + def test_https_connection_read_timeout(self) -> None: """Handshake timeouts should fail with a Timeout""" timed_out = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] while not sock.recv(65536): pass @@ -480,6 +508,7 @@ def socket_handler(listener): timed_out.wait() sock.close() + # first ReadTimeoutError due to SocketTimeout self._start_server(socket_handler) with HTTPSConnectionPool( self.host, self.port, timeout=LONG_TIMEOUT, retries=False @@ -490,8 +519,15 @@ def socket_handler(listener): finally: timed_out.set() - def test_timeout_errors_cause_retries(self): - def socket_handler(listener): + # second ReadTimeoutError due to errno + with HTTPSConnectionPool(host=self.host): + err = OSError() + err.errno = errno.EAGAIN + with pytest.raises(ReadTimeoutError): + pool._raise_timeout(err, "", 0) + + def test_timeout_errors_cause_retries(self) -> None: + def socket_handler(listener: socket.socket) -> None: sock_timeout = listener.accept()[0] # Wait for a second request before closing the first socket. @@ -534,10 +570,10 @@ def socket_handler(listener): finally: socket.setdefaulttimeout(default_timeout) - def test_delayed_body_read_timeout(self): + def test_delayed_body_read_timeout(self) -> None: timed_out = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" body = "Hi" @@ -571,10 +607,10 @@ def socket_handler(listener): finally: timed_out.set() - def test_delayed_body_read_timeout_with_preload(self): + def test_delayed_body_read_timeout_with_preload(self) -> None: timed_out = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" body = "Hi" @@ -601,11 +637,11 @@ def socket_handler(listener): finally: timed_out.set() - def test_incomplete_response(self): + def test_incomplete_response(self) -> None: body = "Response" partial_body = body[:2] - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] # Consume request @@ -631,10 +667,10 @@ def socket_handler(listener): with pytest.raises(ProtocolError): response.read() - def test_retry_weird_http_version(self): + def test_retry_weird_http_version(self) -> None: """Retry class should handle httplib.BadStatusLine errors properly""" - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] # First request. # Pause before responding so the first request times out. @@ -681,10 +717,10 @@ def socket_handler(listener): assert response.status == 200 assert response.data == b"foo" - def test_connection_cleanup_on_read_timeout(self): + def test_connection_cleanup_on_read_timeout(self) -> None: timed_out = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" body = "Hi" @@ -704,6 +740,7 @@ def socket_handler(listener): self._start_server(socket_handler) with HTTPConnectionPool(self.host, self.port) as pool: + assert pool.pool is not None poolsize = pool.pool.qsize() response = pool.urlopen( "GET", "/", retries=0, preload_content=False, timeout=LONG_TIMEOUT @@ -715,11 +752,11 @@ def socket_handler(listener): finally: timed_out.set() - def test_connection_cleanup_on_protocol_error_during_read(self): + def test_connection_cleanup_on_protocol_error_during_read(self) -> None: body = "Response" partial_body = body[:2] - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] # Consume request @@ -741,6 +778,7 @@ def socket_handler(listener): self._start_server(socket_handler) with HTTPConnectionPool(self.host, self.port) as pool: + assert pool.pool is not None poolsize = pool.pool.qsize() response = pool.request("GET", "/", retries=0, preload_content=False) @@ -748,10 +786,10 @@ def socket_handler(listener): response.read() assert poolsize == pool.pool.qsize() - def test_connection_closed_on_read_timeout_preload_false(self): + def test_connection_closed_on_read_timeout_preload_false(self) -> None: timed_out = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] # Consume request @@ -761,14 +799,12 @@ def socket_handler(listener): # Send partial chunked response and then hang. sock.send( - ( - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/plain\r\n" - "Transfer-Encoding: chunked\r\n" - "\r\n" - "8\r\n" - "12345678\r\n" - ).encode("utf-8") + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + b"8\r\n" + b"12345678\r\n" ) timed_out.wait(5) @@ -776,7 +812,7 @@ def socket_handler(listener): # leaking it. Because we don't want to hang this thread, we # actually use select.select to confirm that a new request is # coming in: this lets us time the thread out. - rlist, _, _ = select.select([listener], [], []) + rlist, _, _ = select.select([listener], [], [], 1) assert rlist new_sock = listener.accept()[0] @@ -787,15 +823,13 @@ def socket_handler(listener): # Send complete chunked response. new_sock.send( - ( - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/plain\r\n" - "Transfer-Encoding: chunked\r\n" - "\r\n" - "8\r\n" - "12345678\r\n" - "0\r\n\r\n" - ).encode("utf-8") + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + b"8\r\n" + b"12345678\r\n" + b"0\r\n\r\n" ) new_sock.close() @@ -819,11 +853,11 @@ def socket_handler(listener): ) assert len(response.read()) == 8 - def test_closing_response_actually_closes_connection(self): + def test_closing_response_actually_closes_connection(self) -> None: done_closing = Event() complete = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -831,12 +865,10 @@ def socket_handler(listener): buf = sock.recv(65536) sock.send( - ( - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/plain\r\n" - "Content-Length: 0\r\n" - "\r\n" - ).encode("utf-8") + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 0\r\n" + b"\r\n" ) # Wait for the socket to close. @@ -860,7 +892,7 @@ def socket_handler(listener): successful = complete.wait(timeout=LONG_TIMEOUT) assert successful, "Timed out waiting for connection close" - def test_release_conn_param_is_respected_after_timeout_retry(self): + def test_release_conn_param_is_respected_after_timeout_retry(self) -> None: """For successful ```urlopen(release_conn=False)```, the connection isn't released, even after a retry. @@ -873,7 +905,7 @@ def test_release_conn_param_is_respected_after_timeout_retry(self): [1] """ - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] consume_socket(sock) @@ -885,22 +917,20 @@ def socket_handler(listener): # Expect a new request. Because we don't want to hang this thread, # we actually use select.select to confirm that a new request is # coming in: this lets us time the thread out. - rlist, _, _ = select.select([listener], [], []) + rlist, _, _ = select.select([listener], [], [], 5) assert rlist sock = listener.accept()[0] consume_socket(sock) # Send complete chunked response. sock.send( - ( - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/plain\r\n" - "Transfer-Encoding: chunked\r\n" - "\r\n" - "8\r\n" - "12345678\r\n" - "0\r\n\r\n" - ).encode("utf-8") + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + b"8\r\n" + b"12345678\r\n" + b"0\r\n\r\n" ) sock.close() @@ -921,6 +951,7 @@ def socket_handler(listener): # The connection should still be on the response object, and none # should be in the pool. We opened two though. assert pool.num_connections == 2 + assert pool.pool is not None assert pool.pool.qsize() == 0 assert response.connection is not None @@ -929,10 +960,138 @@ def socket_handler(listener): assert pool.pool.qsize() == 1 assert response.connection is None + def test_socket_close_socket_then_file(self) -> None: + quit_event = threading.Event() + + def consume_ssl_socket( + listener: socket.socket, + ) -> None: + try: + with ( + listener.accept()[0] as sock, + original_ssl_wrap_socket( + sock, + server_side=True, + keyfile=DEFAULT_CERTS["keyfile"], + certfile=DEFAULT_CERTS["certfile"], + ca_certs=DEFAULT_CA, + ) as ssl_sock, + ): + consume_socket(ssl_sock, quit_event=quit_event) + except (ConnectionResetError, ConnectionAbortedError, OSError): + pass + + self._start_server(consume_ssl_socket, quit_event=quit_event) + with ( + socket.create_connection((self.host, self.port)) as sock, + contextlib.closing( + ssl_wrap_socket(sock, server_hostname=self.host, ca_certs=DEFAULT_CA) + ) as ssl_sock, + ssl_sock.makefile("rb") as f, + ): + ssl_sock.close() + f.close() + with pytest.raises(OSError): + ssl_sock.sendall(b"hello") + assert ssl_sock.fileno() == -1 + + def test_socket_close_stays_open_with_makefile_open(self) -> None: + quit_event = threading.Event() + + def consume_ssl_socket(listener: socket.socket) -> None: + try: + with ( + listener.accept()[0] as sock, + original_ssl_wrap_socket( + sock, + server_side=True, + keyfile=DEFAULT_CERTS["keyfile"], + certfile=DEFAULT_CERTS["certfile"], + ca_certs=DEFAULT_CA, + ) as ssl_sock, + ): + consume_socket(ssl_sock, quit_event=quit_event) + except (ConnectionResetError, ConnectionAbortedError, OSError): + pass + + self._start_server(consume_ssl_socket, quit_event=quit_event) + with ( + socket.create_connection((self.host, self.port)) as sock, + contextlib.closing( + ssl_wrap_socket(sock, server_hostname=self.host, ca_certs=DEFAULT_CA) + ) as ssl_sock, + ssl_sock.makefile("rb"), + ): + ssl_sock.close() + ssl_sock.close() + ssl_sock.sendall(b"hello") + assert ssl_sock.fileno() > 0 + + def test_socket_shutdown_stops_recv(self) -> None: + timed_out, starting_read = Event(), Event() + + def socket_handler(listener: socket.socket) -> None: + sock = listener.accept()[0] + + ssl_sock = original_ssl_wrap_socket( + sock, + server_side=True, + keyfile=DEFAULT_CERTS["keyfile"], + certfile=DEFAULT_CERTS["certfile"], + ca_certs=DEFAULT_CA, + ) + + # Consume request + buf = b"" + while not buf.endswith(b"\r\n\r\n"): + buf = ssl_sock.recv(65535) + + # Send incomplete message (note Content-Length) + ssl_sock.send( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 10\r\n" + b"\r\n" + b"Hi-" + ) + timed_out.wait(5) + ssl_sock.close() + + self._start_server(socket_handler) + + class TestClient(threading.Thread): + def __init__(self, host: str, port: int) -> None: + super().__init__() + self.host, self.port = host, port + self.response: BaseHTTPResponse | None = None + + def run(self) -> None: + with HTTPSConnectionPool( + self.host, self.port, ca_certs=DEFAULT_CA + ) as pool: + self.response = pool.urlopen( + "GET", "/", preload_content=False, retries=0 + ) + with pytest.raises(ProtocolError, match="Connection broken"): + starting_read.set() + self.response.read() + + test_client = TestClient(self.host, self.port) + test_client.start() + # First, wait to make sure the client is really stuck reading + starting_read.wait(5) + time.sleep(LONG_TIMEOUT) + # Calling shutdown here calls shutdown() on the underlying socket, + # so that the remaining read will fail instead of blocking + # indefinitely + assert test_client.response is not None + test_client.response.shutdown() + timed_out.set() + class TestProxyManager(SocketDummyServerTestCase): - def test_simple(self): - def echo_socket_handler(listener): + def test_simple(self) -> None: + def echo_socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -951,7 +1110,7 @@ def echo_socket_handler(listener): sock.close() self._start_server(echo_socket_handler) - base_url = "http://%s:%d" % (self.host, self.port) + base_url = f"http://{self.host}:{self.port}" with proxy_from_url(base_url) as proxy: r = proxy.request("GET", "http://google.com/") @@ -971,8 +1130,8 @@ def echo_socket_handler(listener): ] ) - def test_headers(self): - def echo_socket_handler(listener): + def test_headers(self) -> None: + def echo_socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -991,7 +1150,7 @@ def echo_socket_handler(listener): sock.close() self._start_server(echo_socket_handler) - base_url = "http://%s:%d" % (self.host, self.port) + base_url = f"http://{self.host}:{self.port}" # Define some proxy headers. proxy_headers = HTTPHeaderDict({"For The Proxy": "YEAH!"}) @@ -1006,10 +1165,10 @@ def echo_socket_handler(listener): # OrderedDict/MultiDict). assert b"For The Proxy: YEAH!\r\n" in r.data - def test_retries(self): + def test_retries(self) -> None: close_event = Event() - def echo_socket_handler(listener): + def echo_socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] # First request, which should fail sock.close() @@ -1034,7 +1193,7 @@ def echo_socket_handler(listener): close_event.set() self._start_server(echo_socket_handler) - base_url = "http://%s:%d" % (self.host, self.port) + base_url = f"http://{self.host}:{self.port}" with proxy_from_url(base_url) as proxy: conn = proxy.connection_from_url("http://www.google.com") @@ -1053,8 +1212,29 @@ def echo_socket_handler(listener): retries=False, ) - def test_connect_reconn(self): - def proxy_ssl_one(listener): + def test_tunnel_sets_http_11_alpn(self) -> None: + done_receiving = Event() + self.buf = b"" + + def socket_handler(listener: socket.socket) -> None: + sock = listener.accept()[0] + + self.buf = sock.recv(65536) # We only accept one packet + done_receiving.set() # let the test know it can proceed + sock.close() + + self._start_server(socket_handler) + base_url = f"https://{self.host}:{self.port}" + with proxy_from_url(base_url) as proxy: + with pytest.raises(MaxRetryError): + proxy.request("GET", "https://localhost/") + + done_receiving.wait() + assert b"http/1.1" in self.buf + assert b"h2" not in self.buf + + def test_connect_reconn(self) -> None: + def proxy_ssl_one(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -1062,21 +1242,17 @@ def proxy_ssl_one(listener): buf += sock.recv(65536) s = buf.decode("utf-8") if not s.startswith("CONNECT "): - sock.send( - ( - "HTTP/1.1 405 Method not allowed\r\nAllow: CONNECT\r\n\r\n" - ).encode("utf-8") - ) + sock.send(b"HTTP/1.1 405 Method not allowed\r\nAllow: CONNECT\r\n\r\n") sock.close() return - if not s.startswith("CONNECT %s:443" % (self.host,)): - sock.send(("HTTP/1.1 403 Forbidden\r\n\r\n").encode("utf-8")) + if not s.startswith(f"CONNECT {self.host}:443"): + sock.send(b"HTTP/1.1 403 Forbidden\r\n\r\n") sock.close() return - sock.send(("HTTP/1.1 200 Connection Established\r\n\r\n").encode("utf-8")) - ssl_sock = ssl.wrap_socket( + sock.send(b"HTTP/1.1 200 Connection Established\r\n\r\n") + ssl_sock = original_ssl_wrap_socket( sock, server_side=True, keyfile=DEFAULT_CERTS["keyfile"], @@ -1089,36 +1265,34 @@ def proxy_ssl_one(listener): buf += ssl_sock.recv(65536) ssl_sock.send( - ( - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/plain\r\n" - "Content-Length: 2\r\n" - "Connection: close\r\n" - "\r\n" - "Hi" - ).encode("utf-8") + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 2\r\n" + b"Connection: close\r\n" + b"\r\n" + b"Hi" ) ssl_sock.close() - def echo_socket_handler(listener): + def echo_socket_handler(listener: socket.socket) -> None: proxy_ssl_one(listener) proxy_ssl_one(listener) self._start_server(echo_socket_handler) - base_url = "http://%s:%d" % (self.host, self.port) + base_url = f"http://{self.host}:{self.port}" with proxy_from_url(base_url, ca_certs=DEFAULT_CA) as proxy: - url = "https://{0}".format(self.host) + url = f"https://{self.host}" conn = proxy.connection_from_url(url) r = conn.urlopen("GET", url, retries=0) assert r.status == 200 r = conn.urlopen("GET", url, retries=0) assert r.status == 200 - def test_connect_ipv6_addr(self): + def test_connect_ipv6_addr(self) -> None: ipv6_addr = "2001:4998:c:a06::2:4008" - def echo_socket_handler(listener): + def echo_socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -1126,9 +1300,9 @@ def echo_socket_handler(listener): buf += sock.recv(65536) s = buf.decode("utf-8") - if s.startswith("CONNECT [%s]:443" % (ipv6_addr,)): + if s.startswith(f"CONNECT [{ipv6_addr}]:443"): sock.send(b"HTTP/1.1 200 Connection Established\r\n\r\n") - ssl_sock = ssl.wrap_socket( + ssl_sock = original_ssl_wrap_socket( sock, server_side=True, keyfile=DEFAULT_CERTS["keyfile"], @@ -1151,89 +1325,165 @@ def echo_socket_handler(listener): sock.close() self._start_server(echo_socket_handler) - base_url = "http://%s:%d" % (self.host, self.port) + base_url = f"http://{self.host}:{self.port}" with proxy_from_url(base_url, cert_reqs="NONE") as proxy: - url = "https://[{0}]".format(ipv6_addr) + url = f"https://[{ipv6_addr}]" conn = proxy.connection_from_url(url) try: - r = conn.urlopen("GET", url, retries=0) + with pytest.warns(InsecureRequestWarning): + r = conn.urlopen("GET", url, retries=0) assert r.status == 200 except MaxRetryError: - self.fail("Invalid IPv6 format in HTTP CONNECT request") + pytest.fail("Invalid IPv6 format in HTTP CONNECT request") @pytest.mark.parametrize("target_scheme", ["http", "https"]) - def test_https_proxymanager_connected_to_http_proxy(self, target_scheme): - if target_scheme == "https" and sys.version_info[0] == 2: - pytest.skip("HTTPS-in-HTTPS isn't supported on Python 2") - + def test_https_proxymanager_connected_to_http_proxy( + self, target_scheme: str + ) -> None: errored = Event() - def http_socket_handler(listener): + def http_socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] sock.send(b"HTTP/1.0 501 Not Implemented\r\nConnection: close\r\n\r\n") errored.wait() sock.close() self._start_server(http_socket_handler) - base_url = "https://%s:%d" % (self.host, self.port) + base_url = f"https://{self.host}:{self.port}" with ProxyManager(base_url, cert_reqs="NONE") as proxy: with pytest.raises(MaxRetryError) as e: - proxy.request("GET", "%s://example.com" % target_scheme, retries=0) + proxy.request("GET", f"{target_scheme}://example.com", retries=0) errored.set() # Avoid a ConnectionAbortedError on Windows. - assert type(e.value.reason) == ProxyError + assert type(e.value.reason) is ProxyError assert "Your proxy appears to only use HTTP and not HTTPS" in str( e.value.reason ) + def test_proxy_status_not_ok(self) -> None: + def http_socket_handler(listener: socket.socket) -> None: + sock = listener.accept()[0] + consume_socket(sock) + sock.send(b"HTTP/1.0 501 Not Implemented\r\nConnection: close\r\n\r\n") + sock.close() -class TestSSL(SocketDummyServerTestCase): - def test_ssl_failure_midway_through_conn(self): - def socket_handler(listener): + self._start_server(http_socket_handler) + base_url = f"http://{self.host}:{self.port}" + + with ProxyManager(base_url) as proxy: + with pytest.raises(MaxRetryError) as e: + proxy.request("GET", "https://example.com", retries=0) + + assert type(e.value.reason) is ProxyError + assert e.value.reason.args[0] == "Unable to connect to proxy" + assert type(e.value.reason.args[1]) is OSError + assert ( + str(e.value.reason.args[1]) + == "Tunnel connection failed: 501 Not Implemented" + ) + + def test_early_eof_doesnt_cause_infinite_loop(self) -> None: + def http_socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] - sock2 = sock.dup() - ssl_sock = ssl.wrap_socket( - sock, - server_side=True, - keyfile=DEFAULT_CERTS["keyfile"], - certfile=DEFAULT_CERTS["certfile"], - ca_certs=DEFAULT_CA, + consume_socket(sock) + sock.send(b"HTTP/1.0 200 OK\r\n") + sock.close() + + self._start_server(http_socket_handler) + base_url = f"http://{self.host}:{self.port}" + + with ProxyManager(base_url) as proxy: + with pytest.raises(MaxRetryError): + proxy.request("GET", "https://example.com", retries=0) + + def test_header_longer_than_maxline(self) -> None: + def http_socket_handler(listener: socket.socket) -> None: + sock = listener.accept()[0] + consume_socket(sock) + sock.send( + b"HTTP/1.0 200 OK\r\nThis-Header-Is-Too-Long: Way-Too-Long\r\n\r\n" ) + sock.close() - buf = b"" - while not buf.endswith(b"\r\n\r\n"): - buf += ssl_sock.recv(65536) + self._start_server(http_socket_handler) + base_url = f"http://{self.host}:{self.port}" - # Deliberately send from the non-SSL socket. - sock2.send( - ( - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/plain\r\n" - "Content-Length: 2\r\n" - "\r\n" - "Hi" - ).encode("utf-8") + with mock.patch("http.client._MAXLINE", 17): + with ProxyManager(base_url) as proxy: + with pytest.raises(MaxRetryError) as e: + proxy.request("GET", "https://example.com", retries=0) + + assert type(e.value.reason) is ProtocolError + assert e.value.reason.args[0] == "Connection aborted." + assert type(e.value.reason.args[1]) is http.client.LineTooLong + assert ( + str(e.value.reason.args[1]) + == "got more than 17 bytes when reading header line" ) - sock2.close() - ssl_sock.close() + + def test_debuglevel(self, capsys: pytest.CaptureFixture[str]) -> None: + def http_socket_handler(listener: socket.socket) -> None: + sock = listener.accept()[0] + consume_socket(sock) + sock.send(b"HTTP/1.0 200 OK\r\nExample-Header: Example-Value\r\n\r\n") + sock.close() + + self._start_server(http_socket_handler) + base_url = f"http://{self.host}:{self.port}" + + with mock.patch("http.client.HTTPConnection.debuglevel", 1): + with ProxyManager(base_url) as proxy: + with pytest.raises(MaxRetryError): + proxy.request("GET", "https://example.com", retries=0) + + assert "header: Example-Header: Example-Value\r\n\n" in capsys.readouterr().out + + +class TestSSL(SocketDummyServerTestCase): + def test_ssl_failure_midway_through_conn(self) -> None: + def socket_handler(listener: socket.socket) -> None: + with listener.accept()[0] as sock, sock.dup() as sock2: + ssl_sock = original_ssl_wrap_socket( + sock, + server_side=True, + keyfile=DEFAULT_CERTS["keyfile"], + certfile=DEFAULT_CERTS["certfile"], + ca_certs=DEFAULT_CA, + ) + + buf = b"" + while not buf.endswith(b"\r\n\r\n"): + buf += ssl_sock.recv(65536) + + # Deliberately send from the non-SSL socket. + sock2.send( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 2\r\n" + b"\r\n" + b"Hi" + ) + ssl_sock.close() self._start_server(socket_handler) with HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA) as pool: with pytest.raises( - SSLError, match=r"(wrong version number|record overflow)" + SSLError, + match=r"(wrong version number|record overflow|record layer failure)", ): pool.request("GET", "/", retries=False) - @notSecureTransport - def test_ssl_read_timeout(self): + def test_ssl_read_timeout(self) -> None: timed_out = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] - ssl_sock = ssl.wrap_socket( + # disable Nagle's algorithm so there's no delay in sending a partial body + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True) + ssl_sock = original_ssl_wrap_socket( sock, server_side=True, keyfile=DEFAULT_CERTS["keyfile"], @@ -1246,13 +1496,11 @@ def socket_handler(listener): # Send incomplete message (note Content-Length) ssl_sock.send( - ( - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/plain\r\n" - "Content-Length: 10\r\n" - "\r\n" - "Hi-" - ).encode("utf-8") + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 10\r\n" + b"\r\n" + b"Hi-" ) timed_out.wait() @@ -1270,35 +1518,45 @@ def socket_handler(listener): finally: timed_out.set() - def test_ssl_failed_fingerprint_verification(self): - def socket_handler(listener): + def test_ssl_failed_fingerprint_verification(self) -> None: + def socket_handler(listener: socket.socket) -> None: for i in range(2): sock = listener.accept()[0] - ssl_sock = ssl.wrap_socket( - sock, - server_side=True, - keyfile=DEFAULT_CERTS["keyfile"], - certfile=DEFAULT_CERTS["certfile"], - ca_certs=DEFAULT_CA, - ) + try: + ssl_sock = original_ssl_wrap_socket( + sock, + server_side=True, + keyfile=DEFAULT_CERTS["keyfile"], + certfile=DEFAULT_CERTS["certfile"], + ca_certs=DEFAULT_CA, + ) + except (ssl.SSLError, ConnectionResetError, ConnectionAbortedError): + pass - ssl_sock.send( - b"HTTP/1.1 200 OK\r\n" - b"Content-Type: text/plain\r\n" - b"Content-Length: 5\r\n\r\n" - b"Hello" - ) + else: + with ssl_sock: + try: + ssl_sock.send( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 5\r\n\r\n" + b"Hello" + ) + except (ssl.SSLEOFError, ConnectionResetError, BrokenPipeError): + pass - ssl_sock.close() sock.close() self._start_server(socket_handler) # GitHub's fingerprint. Valid, but not matching. fingerprint = "A0:C4:A7:46:00:ED:A7:2D:C0:BE:CB:9A:8C:B6:07:CA:58:EE:74:5E" - def request(): + def request() -> None: pool = HTTPSConnectionPool( - self.host, self.port, assert_fingerprint=fingerprint + self.host, + self.port, + assert_fingerprint=fingerprint, + cert_reqs="CERT_NONE", ) try: timeout = Timeout(connect=LONG_TIMEOUT, read=SHORT_TIMEOUT) @@ -1311,17 +1569,28 @@ def request(): with pytest.raises(MaxRetryError) as cm: request() - assert isinstance(cm.value.reason, SSLError) + assert type(cm.value.reason) is SSLError + assert str(cm.value.reason) == ( + "Fingerprints did not match. Expected " + '"a0c4a74600eda72dc0becb9a8cb607ca58ee745e", got ' + '"728b554c9afc1e88a11cad1bb2e7cc3edbc8f98a"' + ) # Should not hang, see https://github.com/urllib3/urllib3/issues/529 - with pytest.raises(MaxRetryError): + with pytest.raises(MaxRetryError) as cm2: request() + assert type(cm2.value.reason) is SSLError + assert str(cm2.value.reason) == ( + "Fingerprints did not match. Expected " + '"a0c4a74600eda72dc0becb9a8cb607ca58ee745e", got ' + '"728b554c9afc1e88a11cad1bb2e7cc3edbc8f98a"' + ) - def test_retry_ssl_error(self): - def socket_handler(listener): + def test_retry_ssl_error(self) -> None: + def socket_handler(listener: socket.socket) -> None: # first request, trigger an SSLError sock = listener.accept()[0] sock2 = sock.dup() - ssl_sock = ssl.wrap_socket( + ssl_sock = original_ssl_wrap_socket( sock, server_side=True, keyfile=DEFAULT_CERTS["keyfile"], @@ -1333,20 +1602,18 @@ def socket_handler(listener): # Deliberately send from the non-SSL socket to trigger an SSLError sock2.send( - ( - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/plain\r\n" - "Content-Length: 4\r\n" - "\r\n" - "Fail" - ).encode("utf-8") + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 4\r\n" + b"\r\n" + b"Fail" ) sock2.close() ssl_sock.close() # retried request sock = listener.accept()[0] - ssl_sock = ssl.wrap_socket( + ssl_sock = original_ssl_wrap_socket( sock, server_side=True, keyfile=DEFAULT_CERTS["keyfile"], @@ -1369,16 +1636,19 @@ def socket_handler(listener): response = pool.urlopen("GET", "/", retries=1) assert response.data == b"Success" - def test_ssl_load_default_certs_when_empty(self): - def socket_handler(listener): + def test_ssl_load_default_certs_when_empty(self) -> None: + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] - ssl_sock = ssl.wrap_socket( - sock, - server_side=True, - keyfile=DEFAULT_CERTS["keyfile"], - certfile=DEFAULT_CERTS["certfile"], - ca_certs=DEFAULT_CA, - ) + try: + ssl_sock = original_ssl_wrap_socket( + sock, + server_side=True, + keyfile=DEFAULT_CERTS["keyfile"], + certfile=DEFAULT_CERTS["certfile"], + ca_certs=DEFAULT_CA, + ) + except (ssl.SSLError, OSError): + return buf = b"" while not buf.endswith(b"\r\n\r\n"): @@ -1398,24 +1668,40 @@ def socket_handler(listener): context.load_default_certs = mock.Mock() context.options = 0 + class MockSSLSocket: + def __init__( + self, sock: socket.socket, *args: object, **kwargs: object + ) -> None: + self._sock = sock + + def close(self) -> None: + self._sock.close() + + context.wrap_socket = MockSSLSocket + with mock.patch("urllib3.util.ssl_.SSLContext", lambda *_, **__: context): self._start_server(socket_handler) with HTTPSConnectionPool(self.host, self.port) as pool: - with pytest.raises(MaxRetryError): + # Without a proper `SSLContext`, this request will fail in some + # arbitrary way, but we only want to know if load_default_certs() was + # called, which is why we accept any `Exception` here. + with pytest.raises(Exception): pool.request("GET", "/", timeout=SHORT_TIMEOUT) context.load_default_certs.assert_called_with() - @notPyPy2 - def test_ssl_dont_load_default_certs_when_given(self): - def socket_handler(listener): + def test_ssl_dont_load_default_certs_when_given(self) -> None: + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] - ssl_sock = ssl.wrap_socket( - sock, - server_side=True, - keyfile=DEFAULT_CERTS["keyfile"], - certfile=DEFAULT_CERTS["certfile"], - ca_certs=DEFAULT_CA, - ) + try: + ssl_sock = original_ssl_wrap_socket( + sock, + server_side=True, + keyfile=DEFAULT_CERTS["keyfile"], + certfile=DEFAULT_CERTS["certfile"], + ca_certs=DEFAULT_CA, + ) + except (ssl.SSLError, OSError): + return buf = b"" while not buf.endswith(b"\r\n\r\n"): @@ -1435,6 +1721,17 @@ def socket_handler(listener): context.load_default_certs = mock.Mock() context.options = 0 + class MockSSLSocket: + def __init__( + self, sock: socket.socket, *args: object, **kwargs: object + ) -> None: + self._sock = sock + + def close(self) -> None: + self._sock.close() + + context.wrap_socket = MockSSLSocket + with mock.patch("urllib3.util.ssl_.SSLContext", lambda *_, **__: context): for kwargs in [ {"ca_certs": "/a"}, @@ -1442,45 +1739,45 @@ def socket_handler(listener): {"ca_certs": "a", "ca_cert_dir": "a"}, {"ssl_context": context}, ]: - self._start_server(socket_handler) with HTTPSConnectionPool(self.host, self.port, **kwargs) as pool: - with pytest.raises(MaxRetryError): + with pytest.raises(Exception): pool.request("GET", "/", timeout=SHORT_TIMEOUT) context.load_default_certs.assert_not_called() - def test_load_verify_locations_exception(self): + def test_load_verify_locations_exception(self) -> None: """ Ensure that load_verify_locations raises SSLError for all backends """ with pytest.raises(SSLError): - ssl_wrap_socket(None, ca_certs="/tmp/fake-file") + ssl_wrap_socket(None, ca_certs="/tmp/fake-file") # type: ignore[call-overload] - def test_ssl_custom_validation_failure_terminates(self, tmpdir): + def test_ssl_custom_validation_failure_terminates(self, tmpdir: Path) -> None: """ Ensure that the underlying socket is terminated if custom validation fails. """ server_closed = Event() - def is_closed_socket(sock): + def is_closed_socket(sock: socket.socket) -> bool: try: - sock.settimeout(SHORT_TIMEOUT) # Python 3 - sock.recv(1) # Python 2 - except (OSError, socket.error): + sock.settimeout(SHORT_TIMEOUT) + except OSError: return True return False - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] try: - _ = ssl.wrap_socket( + _ = original_ssl_wrap_socket( sock, server_side=True, keyfile=DEFAULT_CERTS["keyfile"], certfile=DEFAULT_CERTS["certfile"], ca_certs=DEFAULT_CA, ) + except ConnectionResetError: + return except ssl.SSLError as e: assert "alert unknown ca" in str(e) if is_closed_socket(sock): @@ -1500,28 +1797,59 @@ def socket_handler(listener): pool.request("GET", "/", retries=False, timeout=LONG_TIMEOUT) assert server_closed.wait(LONG_TIMEOUT), "The socket was not terminated" - # SecureTransport can read only small pieces of data at the moment. - # https://github.com/urllib3/urllib3/pull/2674 - @notSecureTransport - @pytest.mark.skipif( - os.environ.get("CI") == "true" and platform.python_implementation() == "PyPy", - reason="too slow to run in CI", - ) + def _run_preload(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/") + assert len(response.data) == content_length + + def _run_read_None(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/", preload_content=False) + assert len(response.read(None)) == content_length + assert response.read(None) == b"" + + def _run_read_amt(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/", preload_content=False) + assert len(response.read(content_length)) == content_length + assert response.read(5) == b"" + + def _run_read1_None(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/", preload_content=False) + remaining = content_length + while True: + chunk = response.read1(None) + if not chunk: + break + remaining -= len(chunk) + assert remaining == 0 + + def _run_read1_amt(self, pool: HTTPSConnectionPool, content_length: int) -> None: + response = pool.request("GET", "/", preload_content=False) + remaining = content_length + while True: + chunk = response.read1(content_length) + if not chunk: + break + remaining -= len(chunk) + assert remaining == 0 + + @pytest.mark.integration @pytest.mark.parametrize( - "preload_content,read_amt", [(True, None), (False, None), (False, 2 ** 31)] + "method", + [_run_preload, _run_read_None, _run_read_amt, _run_read1_None, _run_read1_amt], ) - def test_requesting_large_resources_via_ssl(self, preload_content, read_amt): + def test_requesting_large_resources_via_ssl( + self, method: typing.Callable[[typing.Any, HTTPSConnectionPool, int], None] + ) -> None: """ Ensure that it is possible to read 2 GiB or more via an SSL socket. https://github.com/urllib3/urllib3/issues/2513 """ - content_length = 2 ** 31 # (`int` max value in C) + 1. + content_length = 2**31 # (`int` max value in C) + 1. ssl_ready = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] - ssl_sock = ssl.wrap_socket( + ssl_sock = original_ssl_wrap_socket( sock, server_side=True, keyfile=DEFAULT_CERTS["keyfile"], @@ -1541,7 +1869,7 @@ def socket_handler(listener): chunks = 2 for i in range(chunks): - ssl_sock.sendall(b"0" * (content_length // chunks)) + ssl_sock.sendall(bytes(content_length // chunks)) ssl_sock.close() sock.close() @@ -1551,13 +1879,11 @@ def socket_handler(listener): with HTTPSConnectionPool( self.host, self.port, ca_certs=DEFAULT_CA, retries=False ) as pool: - response = pool.request("GET", "/", preload_content=preload_content) - data = response.data if preload_content else response.read(read_amt) - assert len(data) == content_length + method(self, pool, content_length) class TestErrorWrapping(SocketDummyServerTestCase): - def test_bad_statusline(self): + def test_bad_statusline(self) -> None: self.start_response_handler( b"HTTP/1.1 Omg What Is This?\r\n" b"Content-Length: 0\r\n" b"\r\n" ) @@ -1565,7 +1891,7 @@ def test_bad_statusline(self): with pytest.raises(ProtocolError): pool.request("GET", "/") - def test_unknown_protocol(self): + def test_unknown_protocol(self) -> None: self.start_response_handler( b"HTTP/1000 200 OK\r\n" b"Content-Length: 0\r\n" b"\r\n" ) @@ -1575,8 +1901,7 @@ def test_unknown_protocol(self): class TestHeaders(SocketDummyServerTestCase): - @onlyPy3 - def test_httplib_headers_case_insensitive(self): + def test_httplib_headers_case_insensitive(self) -> None: self.start_response_handler( b"HTTP/1.1 200 OK\r\n" b"Content-Length: 0\r\n" @@ -1588,11 +1913,11 @@ def test_httplib_headers_case_insensitive(self): r = pool.request("GET", "/") assert HEADERS == dict(r.headers.items()) # to preserve case sensitivity - def start_parsing_handler(self): - self.parsed_headers = OrderedDict() - self.received_headers = [] + def start_parsing_handler(self) -> None: + self.parsed_headers: typing.OrderedDict[str, str] = OrderedDict() + self.received_headers: list[bytes] = [] - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -1607,21 +1932,19 @@ def socket_handler(listener): (key, value) = header.split(b": ") self.parsed_headers[key.decode("ascii")] = value.decode("ascii") - sock.send( - ("HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n").encode("utf-8") - ) + sock.send(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n") sock.close() self._start_server(socket_handler) - def test_headers_are_sent_with_the_original_case(self): + def test_headers_are_sent_with_the_original_case(self) -> None: headers = {"foo": "bar", "bAz": "quux"} self.start_parsing_handler() expected_headers = { "Accept-Encoding": "identity", - "Host": "{0}:{1}".format(self.host, self.port), + "Host": f"{self.host}:{self.port}", "User-Agent": _get_default_user_agent(), } expected_headers.update(headers) @@ -1630,13 +1953,13 @@ def test_headers_are_sent_with_the_original_case(self): pool.request("GET", "/", headers=HTTPHeaderDict(headers)) assert expected_headers == self.parsed_headers - def test_ua_header_can_be_overridden(self): + def test_ua_header_can_be_overridden(self) -> None: headers = {"uSeR-AgENt": "Definitely not urllib3!"} self.start_parsing_handler() expected_headers = { "Accept-Encoding": "identity", - "Host": "{0}:{1}".format(self.host, self.port), + "Host": f"{self.host}:{self.port}", } expected_headers.update(headers) @@ -1644,49 +1967,48 @@ def test_ua_header_can_be_overridden(self): pool.request("GET", "/", headers=HTTPHeaderDict(headers)) assert expected_headers == self.parsed_headers - def test_request_headers_are_sent_in_the_original_order(self): + def test_request_headers_are_sent_in_the_original_order(self) -> None: # NOTE: Probability this test gives a false negative is 1/(K!) K = 16 # NOTE: Provide headers in non-sorted order (i.e. reversed) # so that if the internal implementation tries to sort them, # a change will be detected. expected_request_headers = [ - (u"X-Header-%d" % i, str(i)) for i in reversed(range(K)) + (f"X-Header-{int(i)}", str(i)) for i in reversed(range(K)) ] - def filter_non_x_headers(d): + def filter_non_x_headers( + d: typing.OrderedDict[str, str] + ) -> list[tuple[str, str]]: return [(k, v) for (k, v) in d.items() if k.startswith("X-Header-")] - request_headers = OrderedDict() - self.start_parsing_handler() with HTTPConnectionPool(self.host, self.port, retries=False) as pool: pool.request("GET", "/", headers=OrderedDict(expected_request_headers)) - request_headers = filter_non_x_headers(self.parsed_headers) - assert expected_request_headers == request_headers + assert expected_request_headers == filter_non_x_headers(self.parsed_headers) - @resolvesLocalhostFQDN - def test_request_host_header_ignores_fqdn_dot(self): + @resolvesLocalhostFQDN() + def test_request_host_header_ignores_fqdn_dot(self) -> None: self.start_parsing_handler() with HTTPConnectionPool(self.host + ".", self.port, retries=False) as pool: pool.request("GET", "/") self.assert_header_received( - self.received_headers, "Host", "%s:%s" % (self.host, self.port) + self.received_headers, "Host", f"{self.host}:{self.port}" ) - def test_response_headers_are_returned_in_the_original_order(self): + def test_response_headers_are_returned_in_the_original_order(self) -> None: # NOTE: Probability this test gives a false negative is 1/(K!) K = 16 # NOTE: Provide headers in non-sorted order (i.e. reversed) # so that if the internal implementation tries to sort them, # a change will be detected. expected_response_headers = [ - ("X-Header-%d" % i, str(i)) for i in reversed(range(K)) + (f"X-Header-{int(i)}", str(i)) for i in reversed(range(K)) ] - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -1713,13 +2035,80 @@ def socket_handler(listener): ] assert expected_response_headers == actual_response_headers + @pytest.mark.parametrize( + "method_type, body_type", + [ + ("GET", None), + ("POST", None), + ("POST", "bytes"), + ("POST", "bytes-io"), + ], + ) + def test_headers_sent_with_add( + self, method_type: str, body_type: str | None + ) -> None: + """ + Confirm that when adding headers with combine=True that we simply append to the + most recent value, rather than create a new header line. + """ + body: None | bytes | io.BytesIO + if body_type is None: + body = None + expected = b"\r\n\r\n" + elif body_type == "bytes": + body = b"my-body" + expected = b"\r\n\r\nmy-body" + elif body_type == "bytes-io": + body = io.BytesIO(b"bytes-io-body") + body.seek(0, 0) + expected = b"bytes-io-body\r\n0\r\n\r\n" + else: + raise ValueError("Unknown body type") + + buffer: bytes = b"" + + def socket_handler(listener: socket.socket) -> None: + nonlocal buffer + sock = listener.accept()[0] + sock.settimeout(0) + + while expected not in buffer: + with contextlib.suppress(BlockingIOError): + buffer += sock.recv(65536) + + sock.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Server: example.com\r\n" + b"Content-Length: 0\r\n\r\n" + ) + sock.close() + + self._start_server(socket_handler) + + headers = HTTPHeaderDict() + headers.add("A", "1") + headers.add("C", "3") + headers.add("B", "2") + headers.add("B", "3") + headers.add("A", "4", combine=False) + headers.add("C", "5", combine=True) + headers.add("C", "6") + + with HTTPConnectionPool(self.host, self.port, retries=False) as pool: + r = pool.request( + method_type, + "/", + body=body, + headers=headers, + ) + assert r.status == 200 + assert b"A: 1\r\nA: 4\r\nC: 3, 5\r\nC: 6\r\nB: 2\r\nB: 3" in buffer + -@pytest.mark.skipif( - issubclass(httplib.HTTPMessage, MimeToolMessage), - reason="Header parsing errors not available", -) class TestBrokenHeaders(SocketDummyServerTestCase): - def _test_broken_header_parsing(self, headers, unparsed_data_check=None): + def _test_broken_header_parsing( + self, headers: list[bytes], unparsed_data_check: str | None = None + ) -> None: self.start_response_handler( ( b"HTTP/1.1 200 OK\r\n" @@ -1737,29 +2126,30 @@ def _test_broken_header_parsing(self, headers, unparsed_data_check=None): for record in logs: if ( "Failed to parse headers" in record.msg - and pool._absolute_url("/") == record.args[0] + and type(record.args) is tuple + and _url_from_pool(pool, "/") == record.args[0] ): if ( unparsed_data_check is None or unparsed_data_check in record.getMessage() ): return - self.fail("Missing log about unparsed headers") + pytest.fail("Missing log about unparsed headers") - def test_header_without_name(self): + def test_header_without_name(self) -> None: self._test_broken_header_parsing([b": Value", b"Another: Header"]) - def test_header_without_name_or_value(self): + def test_header_without_name_or_value(self) -> None: self._test_broken_header_parsing([b":", b"Another: Header"]) - def test_header_without_colon_or_value(self): + def test_header_without_colon_or_value(self) -> None: self._test_broken_header_parsing( [b"Broken Header", b"Another: Header"], "Broken Header" ) class TestHeaderParsingContentType(SocketDummyServerTestCase): - def _test_okay_header_parsing(self, header): + def _test_okay_header_parsing(self, header: bytes) -> None: self.start_response_handler( (b"HTTP/1.1 200 OK\r\n" b"Content-Length: 0\r\n") + header + b"\r\n\r\n" ) @@ -1771,15 +2161,15 @@ def _test_okay_header_parsing(self, header): for record in logs: assert "Failed to parse headers" not in record.msg - def test_header_text_plain(self): + def test_header_text_plain(self) -> None: self._test_okay_header_parsing(b"Content-type: text/plain") - def test_header_message_rfc822(self): + def test_header_message_rfc822(self) -> None: self._test_okay_header_parsing(b"Content-type: message/rfc822") class TestHEAD(SocketDummyServerTestCase): - def test_chunked_head_response_does_not_hang(self): + def test_chunked_head_response_does_not_hang(self) -> None: self.start_response_handler( b"HTTP/1.1 200 OK\r\n" b"Transfer-Encoding: chunked\r\n" @@ -1792,7 +2182,7 @@ def test_chunked_head_response_does_not_hang(self): # stream will use the read_chunked method here. assert [] == list(r.stream()) - def test_empty_head_response_does_not_hang(self): + def test_empty_head_response_does_not_hang(self) -> None: self.start_response_handler( b"HTTP/1.1 200 OK\r\n" b"Content-Length: 256\r\n" @@ -1807,10 +2197,10 @@ def test_empty_head_response_does_not_hang(self): class TestStream(SocketDummyServerTestCase): - def test_stream_none_unchunked_response_does_not_hang(self): + def test_stream_none_unchunked_response_does_not_hang(self) -> None: done_event = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -1836,12 +2226,60 @@ def socket_handler(listener): done_event.set() + def test_large_compressed_stream(self) -> None: + done_event = Event() + expected_total_length = 296085 + + def socket_handler(listener: socket.socket) -> None: + compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) + data = compress.compress(b"x" * expected_total_length) + data += compress.flush() + + sock = listener.accept()[0] + + buf = b"" + while not buf.endswith(b"\r\n\r\n"): + buf += sock.recv(65536) + + sock.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: %d\r\n" + b"Content-Encoding: gzip\r\n" + b"\r\n" % (len(data),) + data + ) + + done_event.wait(5) + sock.close() + + self._start_server(socket_handler) + + with HTTPConnectionPool(self.host, self.port, retries=False) as pool: + r = pool.request("GET", "/", timeout=LONG_TIMEOUT, preload_content=False) + + # Chunks must all be equal or less than 10240 + # and only the last chunk is allowed to be smaller + # than 10240. + total_length = 0 + chunks_smaller_than_10240 = 0 + for chunk in r.stream(10240, decode_content=True): + assert 0 < len(chunk) <= 10240 + if len(chunk) < 10240: + chunks_smaller_than_10240 += 1 + else: + assert chunks_smaller_than_10240 == 0 + total_length += len(chunk) + + assert chunks_smaller_than_10240 == 1 + assert expected_total_length == total_length + + done_event.set() + class TestBadContentLength(SocketDummyServerTestCase): - def test_enforce_content_length_get(self): + def test_enforce_content_length_get(self) -> None: done_event = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -1865,21 +2303,14 @@ def socket_handler(listener): "GET", url="/", preload_content=False, enforce_content_length=True ) data = get_response.stream(100) - # Read "good" data before we try to read again. - # This won't trigger till generator is exhausted. - next(data) - try: + with pytest.raises(ProtocolError, match="12 bytes read, 10 more expected"): next(data) - assert False - except ProtocolError as e: - assert "12 bytes read, 10 more expected" in str(e) - done_event.set() - def test_enforce_content_length_no_body(self): + def test_enforce_content_length_no_body(self) -> None: done_event = Event() - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: sock = listener.accept()[0] buf = b"" @@ -1908,8 +2339,8 @@ def socket_handler(listener): class TestRetryPoolSizeDrainFail(SocketDummyServerTestCase): - def test_pool_size_retry_drain_fail(self): - def socket_handler(listener): + def test_pool_size_retry_drain_fail(self) -> None: + def socket_handler(listener: socket.socket) -> None: for _ in range(2): sock = listener.accept()[0] while not sock.recv(65536).endswith(b"\r\n\r\n"): @@ -1935,8 +2366,8 @@ def socket_handler(listener): class TestBrokenPipe(SocketDummyServerTestCase): - @notWindows - def test_ignore_broken_pipe_errors(self, monkeypatch): + @notWindows() + def test_ignore_broken_pipe_errors(self, monkeypatch: pytest.MonkeyPatch) -> None: # On Windows an aborted connection raises an error on # attempts to read data out of a socket that's been closed. sock_shut = Event() @@ -1944,12 +2375,12 @@ def test_ignore_broken_pipe_errors(self, monkeypatch): # a buffer that will cause two sendall calls buf = "a" * 1024 * 1024 * 4 - def connect_and_wait(*args, **kw): + def connect_and_wait(*args: typing.Any, **kw: typing.Any) -> None: ret = orig_connect(*args, **kw) assert sock_shut.wait(5) return ret - def socket_handler(listener): + def socket_handler(listener: socket.socket) -> None: for i in range(2): sock = listener.accept()[0] sock.send( @@ -1978,12 +2409,29 @@ def socket_handler(listener): class TestMultipartResponse(SocketDummyServerTestCase): - def test_multipart_assert_header_parsing_no_defects(self): - def socket_handler(listener): + def test_multipart_assert_header_parsing_no_defects(self) -> None: + quit_event = threading.Event() + + def socket_handler(listener: socket.socket) -> None: for _ in range(2): - sock = listener.accept()[0] - while not sock.recv(65536).endswith(b"\r\n\r\n"): - pass + listener.settimeout(LONG_TIMEOUT) + + while True: + if quit_event and quit_event.is_set(): + return + try: + sock = listener.accept()[0] + break + except (TimeoutError, socket.timeout): + continue + + sock.settimeout(LONG_TIMEOUT) + while True: + if quit_event and quit_event.is_set(): + sock.close() + return + if sock.recv(65536).endswith(b"\r\n\r\n"): + break sock.sendall( b"HTTP/1.1 404 Not Found\r\n" @@ -1999,11 +2447,11 @@ def socket_handler(listener): ) sock.close() - self._start_server(socket_handler) + self._start_server(socket_handler, quit_event=quit_event) from urllib3.connectionpool import log with mock.patch.object(log, "warning") as log_warning: - with HTTPConnectionPool(self.host, self.port, timeout=3) as pool: + with HTTPConnectionPool(self.host, self.port, timeout=LONG_TIMEOUT) as pool: resp = pool.urlopen("GET", "/") assert resp.status == 404 assert ( @@ -2012,3 +2460,239 @@ def socket_handler(listener): ) assert len(resp.data) == 73 log_warning.assert_not_called() + + +class TestContentFraming(SocketDummyServerTestCase): + @pytest.mark.parametrize("content_length", [None, 0]) + @pytest.mark.parametrize("method", ["POST", "PUT", "PATCH"]) + def test_content_length_0_by_default( + self, method: str, content_length: int | None + ) -> None: + buffer = bytearray() + + def socket_handler(listener: socket.socket) -> None: + nonlocal buffer + sock = listener.accept()[0] + while not buffer.endswith(b"\r\n\r\n"): + buffer += sock.recv(65536) + sock.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Server: example.com\r\n" + b"Content-Length: 0\r\n\r\n" + ) + sock.close() + + self._start_server(socket_handler) + + headers = {} + if content_length is not None: + headers["Content-Length"] = str(content_length) + + with HTTPConnectionPool(self.host, self.port, timeout=3) as pool: + resp = pool.request(method, "/") + assert resp.status == 200 + + sent_bytes = bytes(buffer) + assert b"Accept-Encoding: identity\r\n" in sent_bytes + assert b"Content-Length: 0\r\n" in sent_bytes + assert b"transfer-encoding" not in sent_bytes.lower() + + @pytest.mark.parametrize("chunked", [True, False]) + @pytest.mark.parametrize("method", ["POST", "PUT", "PATCH"]) + @pytest.mark.parametrize("body_type", ["file", "generator", "bytes"]) + def test_chunked_specified( + self, method: str, chunked: bool, body_type: str + ) -> None: + quit_event = threading.Event() + buffer = bytearray() + expected_bytes = b"\r\n\r\na\r\nxxxxxxxxxx\r\n0\r\n\r\n" + + def socket_handler(listener: socket.socket) -> None: + nonlocal buffer + listener.settimeout(LONG_TIMEOUT) + while True: + if quit_event.is_set(): + return + try: + sock = listener.accept()[0] + break + except (TimeoutError, socket.timeout): + continue + sock.settimeout(LONG_TIMEOUT) + + while expected_bytes not in buffer: + if quit_event.is_set(): + return + with contextlib.suppress(BlockingIOError): + buffer += sock.recv(65536) + + sock.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Server: example.com\r\n" + b"Content-Length: 0\r\n\r\n" + ) + sock.close() + + self._start_server(socket_handler, quit_event=quit_event) + + body: typing.Any + if body_type == "generator": + + def body_generator() -> typing.Generator[bytes]: + yield b"x" * 10 + + body = body_generator() + elif body_type == "file": + body = io.BytesIO(b"x" * 10) + body.seek(0, 0) + else: + if chunked is False: + pytest.skip("urllib3 uses Content-Length in this case") + body = b"x" * 10 + + with HTTPConnectionPool( + self.host, self.port, timeout=LONG_TIMEOUT, retries=False + ) as pool: + resp = pool.request(method, "/", chunked=chunked, body=body) + assert resp.status == 200 + + sent_bytes = bytes(buffer) + assert sent_bytes.count(b":") == 5 + assert b"Host: localhost:" in sent_bytes + assert b"Accept-Encoding: identity\r\n" in sent_bytes + assert b"Transfer-Encoding: chunked\r\n" in sent_bytes + assert b"User-Agent: python-urllib3/" in sent_bytes + assert b"content-length" not in sent_bytes.lower() + assert expected_bytes in sent_bytes + + @pytest.mark.parametrize("method", ["POST", "PUT", "PATCH"]) + @pytest.mark.parametrize( + "body_type", ["file", "generator", "bytes", "bytearray", "file_text"] + ) + def test_chunked_not_specified(self, method: str, body_type: str) -> None: + buffer = bytearray() + expected_bytes: bytes + body: typing.Any + + if body_type == "generator": + + def body_generator() -> typing.Generator[bytes]: + yield b"x" * 10 + + body = body_generator() + should_be_chunked = True + elif body_type == "file": + body = io.BytesIO(b"x" * 10) + body.seek(0, 0) + should_be_chunked = True + elif body_type == "file_text": + body = io.StringIO("x" * 10) + body.seek(0, 0) + should_be_chunked = True + elif body_type == "bytearray": + body = bytearray(b"x" * 10) + should_be_chunked = False + else: + body = b"x" * 10 + should_be_chunked = False + + if should_be_chunked: + expected_bytes = b"\r\n\r\na\r\nxxxxxxxxxx\r\n0\r\n\r\n" + else: + expected_bytes = b"\r\n\r\nxxxxxxxxxx" + + def socket_handler(listener: socket.socket) -> None: + nonlocal buffer + sock = listener.accept()[0] + sock.settimeout(0) + + while expected_bytes not in buffer: + with contextlib.suppress(BlockingIOError): + buffer += sock.recv(65536) + + sock.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Server: example.com\r\n" + b"Content-Length: 0\r\n\r\n" + ) + sock.close() + + self._start_server(socket_handler) + + with HTTPConnectionPool( + self.host, self.port, timeout=LONG_TIMEOUT, retries=False + ) as pool: + resp = pool.request(method, "/", body=body) + assert resp.status == 200 + + sent_bytes = bytes(buffer) + assert sent_bytes.count(b":") == 5 + assert b"Host: localhost:" in sent_bytes + assert b"Accept-Encoding: identity\r\n" in sent_bytes + assert b"User-Agent: python-urllib3/" in sent_bytes + + if should_be_chunked: + assert b"content-length" not in sent_bytes.lower() + assert b"Transfer-Encoding: chunked\r\n" in sent_bytes + assert expected_bytes in sent_bytes + + else: + assert b"Content-Length: 10\r\n" in sent_bytes + assert b"transfer-encoding" not in sent_bytes.lower() + assert sent_bytes.endswith(expected_bytes) + + @pytest.mark.parametrize( + "header_transform", + [str.lower, str.title, str.upper], + ) + @pytest.mark.parametrize( + ["header", "header_value", "expected"], + [ + ("content-length", "10", b": 10\r\n\r\nxxxxxxxx"), + ( + "transfer-encoding", + "chunked", + b": chunked\r\n\r\n8\r\nxxxxxxxx\r\n0\r\n\r\n", + ), + ], + ) + def test_framing_set_via_headers( + self, + header_transform: typing.Callable[[str], str], + header: str, + header_value: str, + expected: bytes, + ) -> None: + buffer = bytearray() + + def socket_handler(listener: socket.socket) -> None: + nonlocal buffer + sock = listener.accept()[0] + sock.settimeout(0) + + while expected not in buffer: + with contextlib.suppress(BlockingIOError): + buffer += sock.recv(65536) + + sock.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Server: example.com\r\n" + b"Content-Length: 0\r\n\r\n" + ) + sock.close() + + self._start_server(socket_handler) + + with HTTPConnectionPool( + self.host, self.port, timeout=LONG_TIMEOUT, retries=False + ) as pool: + resp = pool.request( + "POST", + "/", + body=b"xxxxxxxx", + headers={header_transform(header): header_value}, + ) + assert resp.status == 200 + + sent_bytes = bytes(buffer) + assert sent_bytes.endswith(expected)