From cdf7baed55f626b4fd7dc5692b13162363c35749 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 27 May 2025 13:54:50 -0700 Subject: [PATCH 01/40] WIP --- awscrt/__init__.py | 1 + awscrt/http_asyncio.py | 335 +++++++++++++++++++++++++++++++++ examples/http2_asyncio_demo.py | 200 ++++++++++++++++++++ examples/http_asyncio_demo.py | 95 ++++++++++ test/test_http_asyncio.py | 195 +++++++++++++++++++ 5 files changed, 826 insertions(+) create mode 100644 awscrt/http_asyncio.py create mode 100644 examples/http2_asyncio_demo.py create mode 100644 examples/http_asyncio_demo.py create mode 100644 test/test_http_asyncio.py diff --git a/awscrt/__init__.py b/awscrt/__init__.py index b1a575120..1726b5e16 100644 --- a/awscrt/__init__.py +++ b/awscrt/__init__.py @@ -7,6 +7,7 @@ 'auth', 'crypto', 'http', + 'http_asyncio', 'io', 'mqtt', 'mqtt5', diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py new file mode 100644 index 000000000..3aa404bd5 --- /dev/null +++ b/awscrt/http_asyncio.py @@ -0,0 +1,335 @@ +""" +HTTP AsyncIO support + +This module provides asyncio wrappers around the awscrt.http module. +All network operations in `awscrt.http_asyncio` are asynchronous and use Python's asyncio framework. +""" + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +import _awscrt +import asyncio +from concurrent.futures import Future +from awscrt import NativeResource +import awscrt.exceptions +from awscrt.http import ( + HttpVersion, HttpClientConnection, HttpRequest, HttpClientStream, HttpProxyOptions, + Http2Setting, Http2ClientConnection, HttpConnectionBase, HttpStreamBase, Http2ClientStream +) +from awscrt.io import ( + ClientBootstrap, SocketOptions, TlsConnectionOptions, InputStream +) +from typing import List, Tuple, Dict, Optional, Union, Iterator, Callable, Any + + +def _future_to_async(future: Future) -> asyncio.Future: + """Convert a concurrent.futures.Future to asyncio.Future""" + loop = asyncio.get_event_loop() + async_future = loop.create_future() + + def _on_done(fut): + try: + result = fut.result() + loop.call_soon_threadsafe(async_future.set_result, result) + except Exception as e: + loop.call_soon_threadsafe(async_future.set_exception, e) + + future.add_done_callback(_on_done) + return async_future + + +class HttpClientConnectionAsync(HttpConnectionBase): + """ + An async HTTP client connection. + + Use `HttpClientConnectionAsync.new()` to establish a new connection. + """ + __slots__ = ('_host_name', '_port') + + @classmethod + async def new(cls, + host_name: str, + port: int, + bootstrap: Optional[ClientBootstrap] = None, + socket_options: Optional[SocketOptions] = None, + tls_connection_options: Optional[TlsConnectionOptions] = None, + proxy_options: Optional['HttpProxyOptions'] = None) -> "HttpClientConnectionAsync": + """ + Asynchronously establish a new HttpClientConnectionAsync. + + Args: + host_name (str): Connect to host. + + port (int): Connect to port. + + bootstrap (Optional [ClientBootstrap]): Client bootstrap to use when initiating socket connection. + If None is provided, the default singleton is used. + + socket_options (Optional[SocketOptions]): Optional socket options. + If None is provided, then default options are used. + + tls_connection_options (Optional[TlsConnectionOptions]): Optional TLS + connection options. If None is provided, then the connection will + be attempted over plain-text. + + proxy_options (Optional[HttpProxyOptions]): Optional proxy options. + If None is provided then a proxy is not used. + + Returns: + HttpClientConnectionAsync: A new HTTP client connection. + """ + future = HttpClientConnection.new( + host_name, + port, + bootstrap, + socket_options, + tls_connection_options, + proxy_options) + + connection = await _future_to_async(future) + return HttpClientConnectionAsync._from_connection(connection) + + @classmethod + def _from_connection(cls, connection): + """Create an HttpClientConnectionAsync from an HttpClientConnection""" + new_conn = cls.__new__(cls) + # Copy the binding and properties from the original connection + new_conn._binding = connection._binding + new_conn._shutdown_future = connection._shutdown_future + new_conn._version = connection._version + new_conn._host_name = connection._host_name + new_conn._port = connection._port + return new_conn + + @property + def host_name(self) -> str: + """Remote hostname""" + return self._host_name + + @property + def port(self) -> int: + """Remote port""" + return self._port + + async def close(self) -> None: + """Close the connection asynchronously. + + Shutdown is asynchronous. This call has no effect if the connection is already + closing. + + Returns: + None: When shutdown is complete. + """ + close_future = super().close() + await _future_to_async(close_future) + + def request(self, + request: 'HttpRequest', + on_response: Optional[Callable[..., None]] = None, + on_body: Optional[Callable[..., None]] = None) -> 'HttpClientStreamAsync': + """Create `HttpClientStreamAsync` to carry out the request/response exchange. + + NOTE: The HTTP stream sends no data until `HttpClientStreamAsync.activate()` + is called. Call activate() when you're ready for callbacks and events to fire. + + Args: + request (HttpRequest): Definition for outgoing request. + + on_response: Optional callback invoked once main response headers are received. + The function should take the following arguments and return nothing: + + * `http_stream` (`HttpClientStreamAsync`): HTTP stream carrying + out this request/response exchange. + + * `status_code` (int): Response status code. + + * `headers` (List[Tuple[str, str]]): Response headers as a + list of (name,value) pairs. + + * `**kwargs` (dict): Forward compatibility kwargs. + + An exception raise by this function will cause the HTTP stream to end in error. + This callback is always invoked on the connection's event-loop thread. + + on_body: Optional callback invoked 0+ times as response body data is received. + The function should take the following arguments and return nothing: + + * `http_stream` (`HttpClientStreamAsync`): HTTP stream carrying + out this request/response exchange. + + * `chunk` (buffer): Response body data (not necessarily + a whole "chunk" of chunked encoding). + + * `**kwargs` (dict): Forward-compatibility kwargs. + + An exception raise by this function will cause the HTTP stream to end in error. + This callback is always invoked on the connection's event-loop thread. + + Returns: + HttpClientStreamAsync: + """ + stream = HttpClientStream(self, request, on_response, on_body) + return HttpClientStreamAsync._from_stream(stream) + + +class Http2ClientConnectionAsync(HttpClientConnectionAsync): + """ + An async HTTP/2 client connection. + + Use `Http2ClientConnectionAsync.new()` to establish a new connection. + """ + + @classmethod + async def new(cls, + host_name: str, + port: int, + bootstrap: Optional[ClientBootstrap] = None, + socket_options: Optional[SocketOptions] = None, + tls_connection_options: Optional[TlsConnectionOptions] = None, + proxy_options: Optional['HttpProxyOptions'] = None, + initial_settings: Optional[List[Http2Setting]] = None, + on_remote_settings_changed: Optional[Callable[[List[Http2Setting]], + None]] = None) -> "Http2ClientConnectionAsync": + """ + Asynchronously establish an HTTP/2 client connection. + Notes: to set up the connection, the server must support HTTP/2 and TlsConnectionOptions + + This class extends HttpClientConnectionAsync with HTTP/2 specific functionality. + + HTTP/2 specific args: + initial_settings (List[Http2Setting]): The initial settings to change for the connection. + + on_remote_settings_changed: Optional callback invoked once the remote peer changes its settings. + And the settings are acknowledged by the local connection. + The function should take the following arguments and return nothing: + + * `settings` (List[Http2Setting]): List of settings that were changed. + """ + future = Http2ClientConnection.new( + host_name, + port, + bootstrap, + socket_options, + tls_connection_options, + proxy_options, + initial_settings, + on_remote_settings_changed) + + connection = await _future_to_async(future) + return Http2ClientConnectionAsync._from_connection(connection) + + def request(self, + request: 'HttpRequest', + on_response: Optional[Callable[..., None]] = None, + on_body: Optional[Callable[..., None]] = None, + manual_write: bool = False) -> 'Http2ClientStreamAsync': + """Create `Http2ClientStreamAsync` to carry out the request/response exchange. + + Args: + request (HttpRequest): Definition for outgoing request. + on_response: Optional callback invoked once main response headers are received. + on_body: Optional callback invoked 0+ times as response body data is received. + manual_write (bool): If True, enables manual data writing on the stream. + + Returns: + Http2ClientStreamAsync: + """ + stream = Http2ClientStream(self, request, on_response, on_body, manual_write) + return Http2ClientStreamAsync._from_stream(stream) + + +class HttpClientStreamAsync(HttpStreamBase): + """Async HTTP stream that sends a request and receives a response. + + Create an HttpClientStreamAsync with `HttpClientConnectionAsync.request()`. + + NOTE: The HTTP stream sends no data until `HttpClientStreamAsync.activate()` + is called. Call activate() when you're ready for callbacks and events to fire. + + Attributes: + connection (HttpClientConnectionAsync): This stream's connection. + + completion_future (asyncio.Future): Future that will contain + the response status code (int) when the request/response exchange + completes. If the exchange fails to complete, the Future will + contain an exception indicating why it failed. + """ + __slots__ = ('_response_status_code', '_on_response_cb', '_on_body_cb', '_request', '_version') + + @classmethod + def _from_stream(cls, stream: HttpClientStream) -> 'HttpClientStreamAsync': + """Create an HttpClientStreamAsync from an HttpClientStream""" + new_stream = cls.__new__(cls) + # Copy the binding and properties from the original stream + new_stream._binding = stream._binding + new_stream._connection = stream._connection + new_stream._completion_future = asyncio.get_event_loop().create_future() + + # Add a callback to bridge the original future to the asyncio future + def _on_done(fut): + try: + result = fut.result() + asyncio.get_event_loop().call_soon_threadsafe(new_stream._completion_future.set_result, result) + except Exception as e: + asyncio.get_event_loop().call_soon_threadsafe(new_stream._completion_future.set_exception, e) + + stream._completion_future.add_done_callback(_on_done) + + new_stream._on_body_cb = stream._on_body_cb + new_stream._response_status_code = stream._response_status_code + new_stream._on_response_cb = stream._on_response_cb + new_stream._request = stream._request + new_stream._version = stream._version + + return new_stream + + @property + def version(self) -> HttpVersion: + """HttpVersion: Protocol used by this stream""" + return self._version + + @property + def response_status_code(self) -> Optional[int]: + """int: The response status code. + + This is None until a response arrives.""" + return self._response_status_code + + def activate(self) -> None: + """Begin sending the request. + + The HTTP stream does nothing until this is called. Call activate() when you + are ready for its callbacks and events to fire. + """ + _awscrt.http_client_stream_activate(self) + + async def wait_for_completion(self) -> int: + """Wait asynchronously for the stream to complete. + + Returns: + int: The response status code. + """ + return await self._completion_future + + +class Http2ClientStreamAsync(HttpClientStreamAsync): + """HTTP/2 stream that sends a request and receives a response. + + Create an Http2ClientStreamAsync with `Http2ClientConnectionAsync.request()`. + """ + + async def write_data(self, + data_stream: Union[InputStream, Any], + end_stream: bool = False) -> None: + """Write data to the stream asynchronously. + + Args: + data_stream (Union[InputStream, Any]): Data to write. + end_stream (bool): Whether this is the last data to write. + + Returns: + None: When the write completes. + """ + future = Http2ClientStream.write_data(self, data_stream, end_stream) + await _future_to_async(future) diff --git a/examples/http2_asyncio_demo.py b/examples/http2_asyncio_demo.py new file mode 100644 index 000000000..74e73aea4 --- /dev/null +++ b/examples/http2_asyncio_demo.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +""" +This example demonstrates how to use the asyncio HTTP/2 client in awscrt. +It performs multiple concurrent requests to httpbin.org and shows HTTP/2 features. +""" + +import asyncio +import sys +from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions +from awscrt.http import HttpHeaders, HttpRequest, Http2Setting, Http2SettingID +from awscrt.http_asyncio import Http2ClientConnectionAsync + + +class Response: + """Holds contents of incoming response""" + + def __init__(self, request_name): + self.request_name = request_name + self.status_code = None + self.headers = None + self.body = bytearray() + + def on_response(self, http_stream, status_code, headers, **kwargs): + print(f"[{self.request_name}] Received response status: {status_code}") + self.status_code = status_code + self.headers = HttpHeaders(headers) + for name, value in headers: + print(f"[{self.request_name}] Header: {name}: {value}") + + def on_body(self, http_stream, chunk, **kwargs): + print(f"[{self.request_name}] Received body chunk of size: {len(chunk)} bytes") + self.body.extend(chunk) + + +def on_remote_settings_changed(settings): + """Handler for when the server updates HTTP/2 settings""" + print("Remote HTTP/2 settings changed:") + for setting in settings: + print(f" - {setting.id.name} = {setting.value}") + + +async def make_concurrent_requests(): + """Perform multiple concurrent HTTP/2 requests asynchronously.""" + # Create an event loop group and default host resolver + event_loop_group = EventLoopGroup() + host_resolver = DefaultHostResolver(event_loop_group) + bootstrap = ClientBootstrap(event_loop_group, host_resolver) + + # Connect to httpbin.org + host_name = "httpbin.org" + port = 443 + + # TLS options for HTTP/2 + tls_ctx_opt = TlsContextOptions() + tls_ctx_opt.verify_peer = True + tls_ctx = ClientTlsContext(tls_ctx_opt) + tls_conn_opt = tls_ctx.new_connection_options() + tls_conn_opt.set_server_name(host_name) + tls_conn_opt.set_alpn_list(["h2"]) # Set ALPN to HTTP/2 + + # Initial HTTP/2 settings + initial_settings = [ + Http2Setting(Http2SettingID.ENABLE_PUSH, 0), + Http2Setting(Http2SettingID.MAX_CONCURRENT_STREAMS, 100), + Http2Setting(Http2SettingID.INITIAL_WINDOW_SIZE, 65535), + ] + + print(f"Connecting to {host_name}:{port} using HTTP/2...") + connection = await Http2ClientConnectionAsync.new( + host_name=host_name, + port=port, + bootstrap=bootstrap, + tls_connection_options=tls_conn_opt, + initial_settings=initial_settings, + on_remote_settings_changed=on_remote_settings_changed + ) + print("HTTP/2 Connection established!") + + try: + # Create several requests to be executed concurrently + tasks = [] + + # Request 1: Simple GET + tasks.append(send_get_request(connection, host_name)) + + # Request 2: POST with JSON body + tasks.append(send_post_request(connection, host_name)) + + # Request 3: Stream data using manual write mode + tasks.append(send_stream_request(connection, host_name)) + + # Wait for all requests to complete + await asyncio.gather(*tasks) + + finally: + # Close the connection + print("Closing connection...") + await connection.close() + print("Connection closed!") + + +async def send_get_request(connection, host_name): + """Send a GET request using the HTTP/2 connection.""" + print("Sending GET request...") + request = HttpRequest("GET", "/get?param1=value1¶m2=value2") + request.headers.add("host", host_name) + + # Set up response handler + response = Response("GET") + stream = connection.request(request, response.on_response, response.on_body) + stream.activate() + + # Wait for completion + status_code = await stream.wait_for_completion() + print(f"GET request completed with status code: {status_code}") + print("\nGET Response body:") + print(response.body.decode("utf-8")) + return status_code + + +async def send_post_request(connection, host_name): + """Send a POST request with JSON body using the HTTP/2 connection.""" + print("Sending POST request with JSON body...") + + # Prepare JSON payload + json_payload = '{"name": "Example User", "id": 12345}' + + # Create request with headers + request = HttpRequest("POST", "/post") + request.headers.add("host", host_name) + request.headers.add("content-type", "application/json") + request.headers.add("content-length", str(len(json_payload))) + + # Set the body + request.body_stream = json_payload.encode("utf-8") + + # Set up response handler + response = Response("POST") + stream = connection.request(request, response.on_response, response.on_body) + stream.activate() + + # Wait for completion + status_code = await stream.wait_for_completion() + print(f"POST request completed with status code: {status_code}") + print("\nPOST Response body:") + print(response.body.decode("utf-8")) + return status_code + + +async def send_stream_request(connection, host_name): + """Send a request with streamed data using manual write mode.""" + print("Sending request with manual data streaming...") + + # Create request + request = HttpRequest("PUT", "/put") + request.headers.add("host", host_name) + request.headers.add("content-type", "text/plain") + # Note: We don't set content-length as we're streaming the data + + # Set up response handler + response = Response("STREAM") + stream = connection.request(request, response.on_response, response.on_body, manual_write=True) + stream.activate() + + # Stream data in chunks + data_chunks = [ + b"This is the first chunk of data.\n", + b"This is the second chunk of data.\n", + b"This is the final chunk of data." + ] + + for i, chunk in enumerate(data_chunks): + print(f"Sending chunk {i + 1}/{len(data_chunks)}, size: {len(chunk)} bytes") + await stream.write_data(chunk, end_stream=(i == len(data_chunks) - 1)) + # Simulate processing time between chunks + await asyncio.sleep(0.5) + + # Wait for completion + status_code = await stream.wait_for_completion() + print(f"Stream request completed with status code: {status_code}") + print("\nStream Response body:") + print(response.body.decode("utf-8")) + return status_code + + +def main(): + """Entry point for the example.""" + try: + asyncio.run(make_concurrent_requests()) + return 0 + except Exception as e: + print(f"Exception: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/http_asyncio_demo.py b/examples/http_asyncio_demo.py new file mode 100644 index 000000000..98f76d314 --- /dev/null +++ b/examples/http_asyncio_demo.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +""" +This example demonstrates how to use the asyncio HTTP client in awscrt. +It performs a simple GET request to httpbin.org and prints the response. +""" + +import asyncio +import sys +from awscrt.io import ClientBootstrap, DefaultHostResolver, EventLoopGroup +from awscrt.http import HttpHeaders, HttpRequest +from awscrt.http_asyncio import HttpClientConnectionAsync + + +class Response: + """Holds contents of incoming response""" + + def __init__(self): + self.status_code = None + self.headers = None + self.body = bytearray() + + def on_response(self, http_stream, status_code, headers, **kwargs): + print(f"Received response status: {status_code}") + self.status_code = status_code + self.headers = HttpHeaders(headers) + for name, value in headers: + print(f"Header: {name}: {value}") + + def on_body(self, http_stream, chunk, **kwargs): + print(f"Received body chunk of size: {len(chunk)} bytes") + self.body.extend(chunk) + + +async def make_request(): + """Perform an HTTP GET request asynchronously using the AWS CRT HTTP client.""" + # Create an event loop group and default host resolver + event_loop_group = EventLoopGroup() + host_resolver = DefaultHostResolver(event_loop_group) + bootstrap = ClientBootstrap(event_loop_group, host_resolver) + + # Connect to httpbin.org + host_name = "httpbin.org" + port = 443 + + print(f"Connecting to {host_name}:{port}...") + connection = await HttpClientConnectionAsync.new( + host_name=host_name, + port=port, + bootstrap=bootstrap, + tls_connection_options=None # For HTTPS, you would provide TLS options here + ) + print("Connection established!") + + # Create and send a simple GET request + print("Sending request...") + request = HttpRequest("GET", "/get") + request.headers.add("host", host_name) + + # Set up response handlers + response = Response() + print("before request") + stream = connection.request(request, response.on_response, response.on_body) + print("after request") + stream.activate() + print("activated") + + # Wait for the response to complete + status_code = await stream.wait_for_completion() + print(f"Request completed with status code: {status_code}") + + # Print the response body + print("\nResponse body:") + print(response.body.decode('utf-8')) + + # Close the connection + print("Closing connection...") + await connection.close() + print("Connection closed!") + + +def main(): + """Entry point for the example.""" + try: + asyncio.run(make_request()) + return 0 + except Exception as e: + print(f"Exception: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/test_http_asyncio.py b/test/test_http_asyncio.py new file mode 100644 index 000000000..692ef5ed3 --- /dev/null +++ b/test/test_http_asyncio.py @@ -0,0 +1,195 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +import time +import socket +import sys +import asyncio +import unittest +import threading +from test import NativeResourceTest +import ssl +import os +from io import BytesIO +from http.server import HTTPServer, SimpleHTTPRequestHandler +from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions +from awscrt.http import HttpHeaders, HttpProxyOptions, HttpRequest, HttpVersion +from awscrt.http_asyncio import HttpClientConnectionAsync, Http2ClientConnectionAsync +import awscrt.exceptions + + +class Response: + """Holds contents of incoming response""" + + def __init__(self): + self.status_code = None + self.headers = None + self.body = bytearray() + + def on_response(self, http_stream, status_code, headers, **kwargs): + self.status_code = status_code + self.headers = HttpHeaders(headers) + + def on_body(self, http_stream, chunk, **kwargs): + self.body.extend(chunk) + + +class TestRequestHandler(SimpleHTTPRequestHandler): + """Request handler for test server""" + + def do_PUT(self): + content_length = int(self.headers['Content-Length']) + # store put request on the server object + incoming_body_bytes = self.rfile.read(content_length) + self.server.put_requests[self.path] = incoming_body_bytes + self.send_response(200, 'OK') + self.end_headers() + + +class TestAsyncClient(NativeResourceTest): + hostname = 'localhost' + timeout = 5 # seconds + + def _start_server(self, secure, http_1_0=False): + # HTTP/1.0 closes the connection at the end of each request + # HTTP/1.1 will keep the connection alive + if http_1_0: + TestRequestHandler.protocol_version = "HTTP/1.0" + else: + TestRequestHandler.protocol_version = "HTTP/1.1" + + self.server = HTTPServer((self.hostname, 0), TestRequestHandler) + if secure: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certfile='test/resources/unittest.crt', keyfile="test/resources/unittest.key") + self.server.socket = context.wrap_socket(self.server.socket, server_side=True) + self.port = self.server.server_address[1] + + # put requests are stored in this dict + self.server.put_requests = {} + + self.server_thread = threading.Thread(target=self.server.serve_forever, name='test_server') + self.server_thread.start() + + def _stop_server(self): + self.server.shutdown() + self.server.server_close() + self.server_thread.join() + + async def _new_client_connection(self, secure, proxy_options=None): + if secure: + tls_ctx_opt = TlsContextOptions() + tls_ctx_opt.verify_peer = False + tls_ctx = ClientTlsContext(tls_ctx_opt) + tls_conn_opt = tls_ctx.new_connection_options() + tls_conn_opt.set_server_name(self.hostname) + else: + tls_conn_opt = None + + event_loop_group = EventLoopGroup() + host_resolver = DefaultHostResolver(event_loop_group) + bootstrap = ClientBootstrap(event_loop_group, host_resolver) + connection = await HttpClientConnectionAsync.new( + host_name=self.hostname, + port=self.port, + bootstrap=bootstrap, + tls_connection_options=tls_conn_opt, + proxy_options=proxy_options) + + return connection + + async def _test_connect(self, secure): + self._start_server(secure) + try: + connection = await self._new_client_connection(secure) + + # close connection + await connection.close() + self.assertFalse(connection.is_open()) + + finally: + self._stop_server() + + async def _test_get(self, secure): + # GET request receives this very file from the server + self._start_server(secure) + try: + connection = await self._new_client_connection(secure) + + test_asset_path = 'test/test_http_asyncio.py' + + request = HttpRequest('GET', '/' + test_asset_path) + response = Response() + stream = connection.request(request, response.on_response, response.on_body) + stream.activate() + + # wait for stream to complete + status_code = await stream.wait_for_completion() + self.assertEqual(200, status_code) + self.assertEqual(200, response.status_code) + + with open(test_asset_path, 'rb') as test_asset: + test_asset_bytes = test_asset.read() + self.assertEqual(test_asset_bytes, response.body) + + await connection.close() + + finally: + self._stop_server() + + async def _test_put(self, secure): + # PUT request sends this very file to the server + self._start_server(secure) + try: + connection = await self._new_client_connection(secure) + test_asset_path = 'test/test_http_asyncio.py' + with open(test_asset_path, 'rb') as outgoing_body_stream: + outgoing_body_bytes = outgoing_body_stream.read() + headers = HttpHeaders([ + ('Content-Length', str(len(outgoing_body_bytes))), + ]) + + # seek back to start of stream before trying to send it + outgoing_body_stream.seek(0) + + request = HttpRequest('PUT', '/' + test_asset_path, headers, outgoing_body_stream) + response = Response() + http_stream = connection.request(request, response.on_response, response.on_body) + http_stream.activate() + + # wait for stream to complete + status_code = await http_stream.wait_for_completion() + self.assertEqual(200, status_code) + self.assertEqual(200, response.status_code) + + # compare what we sent against what the server received + server_received = self.server.put_requests.get('/' + test_asset_path) + self.assertIsNotNone(server_received) + self.assertEqual(server_received, outgoing_body_bytes) + + await connection.close() + + finally: + self._stop_server() + + def test_connect_http(self): + asyncio.run(self._test_connect(secure=False)) + + def test_connect_https(self): + asyncio.run(self._test_connect(secure=True)) + + def test_get_http(self): + asyncio.run(self._test_get(secure=False)) + + def test_get_https(self): + asyncio.run(self._test_get(secure=True)) + + def test_put_http(self): + asyncio.run(self._test_put(secure=False)) + + def test_put_https(self): + asyncio.run(self._test_put(secure=True)) + + +if __name__ == '__main__': + unittest.main() From c433ede266af93d3e6d3ec1adf38a2d0e66366c9 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 10 Jun 2025 13:45:23 -0700 Subject: [PATCH 02/40] next --- awscrt/http.py | 17 +++-- awscrt/http_asyncio.py | 134 ++++++++++++++++++++++++--------- examples/http2_asyncio_demo.py | 82 ++++++++++++++++---- 3 files changed, 175 insertions(+), 58 deletions(-) diff --git a/awscrt/http.py b/awscrt/http.py index 85244b5a2..b4e2ae6cc 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -359,6 +359,7 @@ def completion_future(self) -> "concurrent.futures.Future": return self._completion_future def _on_body(self, chunk: bytes) -> None: + # print("########### _on_body, chunk size:", len(chunk)) if self._on_body_cb: self._on_body_cb(http_stream=self, chunk=chunk) @@ -397,7 +398,7 @@ def _init_common(self, assert isinstance(connection, HttpClientConnection) assert isinstance(request, HttpRequest) assert callable(on_response) or on_response is None - assert callable(on_body) or on_body is None + # assert callable(on_body) or on_body is None super().__init__(connection, on_body) @@ -407,7 +408,7 @@ def _init_common(self, # keep HttpRequest alive until stream completes self._request: 'HttpRequest' = request self._version: HttpVersion = connection.version - + print("########### http2_manual_write: ", http2_manual_write) self._binding = _awscrt.http_client_stream_new(self, connection, request, http2_manual_write) @property @@ -428,9 +429,11 @@ def activate(self) -> None: The HTTP stream does nothing until this is called. Call activate() when you are ready for its callbacks and events to fire. """ + print("########### activate") _awscrt.http_client_stream_activate(self) def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: + print("########### _on_response, status_code:", status_code) self._response_status_code = status_code if self._on_response_cb: @@ -532,6 +535,7 @@ def __init__(self, binding = _awscrt.http_message_new_request(headers) super().__init__(binding, headers, body_stream) + print("########### body_stream: ", body_stream) self.method = method self.path = path @@ -584,6 +588,8 @@ def __init__(self, name_value_pairs: Optional[List[Tuple[str, str]]] = None) -> super().__init__() self._binding = _awscrt.http_headers_new() if name_value_pairs: + for i in name_value_pairs: + print("########### headers: ", i) self.add_pairs(name_value_pairs) @classmethod @@ -822,15 +828,14 @@ def __init__( def _on_connection_setup(self, binding: Any, error_code: int, http_version: HttpVersion) -> None: if self._connect_future is None: return - + if error_code != 0: + self._connect_future.set_exception(awscrt.exceptions.from_code(error_code)) + return if self._expected_version and self._expected_version != http_version: # unexpected protocol version # AWS_ERROR_HTTP_UNSUPPORTED_PROTOCOL self._connect_future.set_exception(awscrt.exceptions.from_code(2060)) return - if error_code != 0: - self._connect_future.set_exception(awscrt.exceptions.from_code(error_code)) - return if http_version == HttpVersion.Http2: connection = Http2ClientConnection() else: diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py index 3aa404bd5..1ebda10a8 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/http_asyncio.py @@ -13,14 +13,15 @@ from concurrent.futures import Future from awscrt import NativeResource import awscrt.exceptions +from typing import List, Tuple, Dict, Optional, Union, Iterator, Callable, Any from awscrt.http import ( HttpVersion, HttpClientConnection, HttpRequest, HttpClientStream, HttpProxyOptions, - Http2Setting, Http2ClientConnection, HttpConnectionBase, HttpStreamBase, Http2ClientStream + Http2Setting, Http2ClientConnection, HttpConnectionBase, HttpHeaders, Http2ClientStream ) from awscrt.io import ( ClientBootstrap, SocketOptions, TlsConnectionOptions, InputStream ) -from typing import List, Tuple, Dict, Optional, Union, Iterator, Callable, Any +from collections import deque def _future_to_async(future: Future) -> asyncio.Future: @@ -39,7 +40,7 @@ def _on_done(fut): return async_future -class HttpClientConnectionAsync(HttpConnectionBase): +class HttpClientConnectionAsync(HttpClientConnection): """ An async HTTP client connection. @@ -86,8 +87,7 @@ async def new(cls, socket_options, tls_connection_options, proxy_options) - - connection = await _future_to_async(future) + connection = await asyncio.wrap_future(future) return HttpClientConnectionAsync._from_connection(connection) @classmethod @@ -100,6 +100,9 @@ def _from_connection(cls, connection): new_conn._version = connection._version new_conn._host_name = connection._host_name new_conn._port = connection._port + # Initialize the parent class without calling __init__ + HttpConnectionBase.__init__(new_conn) + new_conn._shutdown_future = connection._shutdown_future return new_conn @property @@ -122,12 +125,14 @@ async def close(self) -> None: None: When shutdown is complete. """ close_future = super().close() - await _future_to_async(close_future) + await asyncio.wrap_future(close_future) def request(self, request: 'HttpRequest', - on_response: Optional[Callable[..., None]] = None, - on_body: Optional[Callable[..., None]] = None) -> 'HttpClientStreamAsync': + on_response: Optional[Callable[..., + None]] = None, + on_body: Optional[Callable[..., + None]] = None) -> 'Http2ClientStreamAsync': """Create `HttpClientStreamAsync` to carry out the request/response exchange. NOTE: The HTTP stream sends no data until `HttpClientStreamAsync.activate()` @@ -167,10 +172,16 @@ def request(self, This callback is always invoked on the connection's event-loop thread. Returns: - HttpClientStreamAsync: + If use_response_wrapper is False: + HttpClientStreamAsync: Stream for the HTTP request/response exchange. + If use_response_wrapper is True: + Tuple[HttpClientStreamAsync, HttpResponseAsync]: A tuple containing the stream + and a response wrapper that provides async methods to access the response. """ stream = HttpClientStream(self, request, on_response, on_body) - return HttpClientStreamAsync._from_stream(stream) + async_stream = HttpClientStreamAsync._from_stream(stream) + + return async_stream class Http2ClientConnectionAsync(HttpClientConnectionAsync): @@ -216,13 +227,15 @@ async def new(cls, initial_settings, on_remote_settings_changed) - connection = await _future_to_async(future) + connection = await asyncio.wrap_future(future) return Http2ClientConnectionAsync._from_connection(connection) def request(self, request: 'HttpRequest', - on_response: Optional[Callable[..., None]] = None, - on_body: Optional[Callable[..., None]] = None, + on_response: Optional[Callable[..., + None]] = None, + on_body: Optional[Callable[..., + None]] = None, manual_write: bool = False) -> 'Http2ClientStreamAsync': """Create `Http2ClientStreamAsync` to carry out the request/response exchange. @@ -233,13 +246,16 @@ def request(self, manual_write (bool): If True, enables manual data writing on the stream. Returns: - Http2ClientStreamAsync: + Http2ClientStreamAsync: Stream for the HTTP/2 request/response exchange. """ + print("######################### async request called #########################") stream = Http2ClientStream(self, request, on_response, on_body, manual_write) - return Http2ClientStreamAsync._from_stream(stream) + async_stream = Http2ClientStreamAsync._from_stream(stream) + return async_stream -class HttpClientStreamAsync(HttpStreamBase): + +class HttpClientStreamAsync(HttpClientStream): """Async HTTP stream that sends a request and receives a response. Create an HttpClientStreamAsync with `HttpClientConnectionAsync.request()`. @@ -255,28 +271,66 @@ class HttpClientStreamAsync(HttpStreamBase): completes. If the exchange fails to complete, the Future will contain an exception indicating why it failed. """ - __slots__ = ('_response_status_code', '_on_response_cb', '_on_body_cb', '_request', '_version') + __slots__ = ( + '_response_status_code', + '_on_response_cb', + '_on_body_cb', + '_request', + '_version', + '_original_stream', + '_completion_future', + '_chunk_queue', + '_stream_completed', + '_original_on_body_cb') @classmethod def _from_stream(cls, stream: HttpClientStream) -> 'HttpClientStreamAsync': """Create an HttpClientStreamAsync from an HttpClientStream""" new_stream = cls.__new__(cls) + # Keep a reference to the original stream to prevent garbage collection + new_stream._original_stream = stream # Copy the binding and properties from the original stream new_stream._binding = stream._binding new_stream._connection = stream._connection - new_stream._completion_future = asyncio.get_event_loop().create_future() + + # Initialize chunk queue and stream completion flag + new_stream._chunk_futures = deque() + new_stream._stream_completed = False + new_stream._received_chunks = deque() + + # Capture the current event loop to use in callbacks + loop = asyncio.get_event_loop() + new_stream._completion_future = loop.create_future() # Add a callback to bridge the original future to the asyncio future def _on_done(fut): try: result = fut.result() - asyncio.get_event_loop().call_soon_threadsafe(new_stream._completion_future.set_result, result) + loop.call_soon_threadsafe(new_stream._completion_future.set_result, result) + # Mark the stream as completed when the completion_future resolves + loop.call_soon_threadsafe(lambda: setattr(new_stream, '_stream_completed', True)) except Exception as e: - asyncio.get_event_loop().call_soon_threadsafe(new_stream._completion_future.set_exception, e) + loop.call_soon_threadsafe(new_stream._completion_future.set_exception, e) + # Mark the stream as completed on error as well + loop.call_soon_threadsafe(lambda: setattr(new_stream, '_stream_completed', True)) stream._completion_future.add_done_callback(_on_done) - new_stream._on_body_cb = stream._on_body_cb + # Create a new on_body_cb that puts chunks in the queue + def wrapped_on_body_cb(http_stream, chunk, **kwargs): + # print("################# chunk is", chunk) + if new_stream._chunk_futures: + # print("################# chunk in future is", chunk) + future = new_stream._chunk_futures.popleft() + future.set_result(chunk) + else: + # print("################# chunk in recoved is", chunk) + new_stream._received_chunks.append(chunk) + + # Replace the on_body_cb with our wrapper + stream._on_body_cb = wrapped_on_body_cb + new_stream._on_body_cb = wrapped_on_body_cb + new_stream._response_status_code = stream._response_status_code new_stream._on_response_cb = stream._on_response_cb new_stream._request = stream._request @@ -284,25 +338,32 @@ def _on_done(fut): return new_stream - @property - def version(self) -> HttpVersion: - """HttpVersion: Protocol used by this stream""" - return self._version - - @property - def response_status_code(self) -> Optional[int]: - """int: The response status code. - - This is None until a response arrives.""" - return self._response_status_code - def activate(self) -> None: + print("######################### async activate called #########################") """Begin sending the request. The HTTP stream does nothing until this is called. Call activate() when you are ready for its callbacks and events to fire. """ - _awscrt.http_client_stream_activate(self) + # Use the original stream's binding + self._original_stream.activate() + + async def next(self) -> bytes: + # print("######################### async next called #########################") + """Get the next chunk from the response body. + + Returns: + bytes: The next chunk of data from the response body. + Returns empty bytes when the stream is completed and no more chunks are left. + """ + if self._received_chunks: + return self._received_chunks.popleft() + elif self._completion_future.done(): + return b"" + else: + future = Future[bytes]() + self._chunk_futures.append(future) + return await asyncio.wrap_future(future) async def wait_for_completion(self) -> int: """Wait asynchronously for the stream to complete. @@ -331,5 +392,6 @@ async def write_data(self, Returns: None: When the write completes. """ - future = Http2ClientStream.write_data(self, data_stream, end_stream) - await _future_to_async(future) + # print("######################### async write_data called #########################") + future = self._original_stream.write_data(data_stream, end_stream) + await asyncio.wrap_future(future) diff --git a/examples/http2_asyncio_demo.py b/examples/http2_asyncio_demo.py index 74e73aea4..f99c2831a 100644 --- a/examples/http2_asyncio_demo.py +++ b/examples/http2_asyncio_demo.py @@ -9,9 +9,13 @@ import asyncio import sys +import io from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions from awscrt.http import HttpHeaders, HttpRequest, Http2Setting, Http2SettingID from awscrt.http_asyncio import Http2ClientConnectionAsync +from awscrt_python_logging_example import PythonLoggingRedirector +import awscrt.io +import logging class Response: @@ -35,27 +39,42 @@ def on_body(self, http_stream, chunk, **kwargs): self.body.extend(chunk) +# Create an event for synchronizing remote settings +remote_settings_event = None +event_loop = None + + def on_remote_settings_changed(settings): """Handler for when the server updates HTTP/2 settings""" print("Remote HTTP/2 settings changed:") for setting in settings: print(f" - {setting.id.name} = {setting.value}") + # Signal that remote settings have been received + # This callback is called from a different thread, so we need to use call_soon_threadsafe + if event_loop and remote_settings_event: + event_loop.call_soon_threadsafe(remote_settings_event.set) async def make_concurrent_requests(): """Perform multiple concurrent HTTP/2 requests asynchronously.""" + global remote_settings_event, event_loop + + # Get the current event loop and create the event + event_loop = asyncio.get_running_loop() + remote_settings_event = asyncio.Event() + # Create an event loop group and default host resolver event_loop_group = EventLoopGroup() host_resolver = DefaultHostResolver(event_loop_group) bootstrap = ClientBootstrap(event_loop_group, host_resolver) # Connect to httpbin.org - host_name = "httpbin.org" - port = 443 + host_name = "localhost" # Change to "httpbin.org" for real requests + port = 3443 # TLS options for HTTP/2 tls_ctx_opt = TlsContextOptions() - tls_ctx_opt.verify_peer = True + tls_ctx_opt.verify_peer = False tls_ctx = ClientTlsContext(tls_ctx_opt) tls_conn_opt = tls_ctx.new_connection_options() tls_conn_opt.set_server_name(host_name) @@ -79,23 +98,36 @@ async def make_concurrent_requests(): ) print("HTTP/2 Connection established!") + # Wait for remote settings to be received + print("Waiting for remote settings...") + await remote_settings_event.wait() + print("Remote settings received, proceeding with requests...") + try: # Create several requests to be executed concurrently tasks = [] - # Request 1: Simple GET - tasks.append(send_get_request(connection, host_name)) + # # Request 1: Simple GET + # tasks.append(send_get_request(connection, host_name)) - # Request 2: POST with JSON body - tasks.append(send_post_request(connection, host_name)) + # # Request 2: POST with JSON body + # tasks.append(send_post_request(connection, host_name)) # Request 3: Stream data using manual write mode tasks.append(send_stream_request(connection, host_name)) # Wait for all requests to complete - await asyncio.gather(*tasks) + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Check for any exceptions + for i, result in enumerate(results): + if isinstance(result, Exception): + print(f"Task {i} failed with exception: {result}") finally: + # Add a small delay to ensure all responses are received + await asyncio.sleep(1) + # Close the connection print("Closing connection...") await connection.close() @@ -134,8 +166,8 @@ async def send_post_request(connection, host_name): request.headers.add("content-type", "application/json") request.headers.add("content-length", str(len(json_payload))) - # Set the body - request.body_stream = json_payload.encode("utf-8") + # Set the body using BytesIO stream + request.body_stream = io.BytesIO(json_payload.encode("utf-8")) # Set up response handler response = Response("POST") @@ -172,16 +204,22 @@ async def send_stream_request(connection, host_name): b"This is the final chunk of data." ] - for i, chunk in enumerate(data_chunks): - print(f"Sending chunk {i + 1}/{len(data_chunks)}, size: {len(chunk)} bytes") - await stream.write_data(chunk, end_stream=(i == len(data_chunks) - 1)) - # Simulate processing time between chunks - await asyncio.sleep(0.5) + # for i, chunk in enumerate(data_chunks): + # print(f"Sending chunk {i + 1}/{len(data_chunks)}, size: {len(chunk)} bytes") + # # Use BytesIO for each chunk + # chunk_stream = io.BytesIO(chunk) + # await stream.write_data(chunk_stream, end_stream=(i == len(data_chunks) - 1)) + # # Simulate processing time between chunks + # await asyncio.sleep(0.1) + await stream.write_data(io.BytesIO(data_chunks[0]), end_stream=False) + await stream.write_data(io.BytesIO(data_chunks[1]), end_stream=True) + # await stream.write_data(io.BytesIO(data_chunks[1]), end_stream=True) + result = await stream.next() # Wait for completion status_code = await stream.wait_for_completion() print(f"Stream request completed with status code: {status_code}") - print("\nStream Response body:") + print("\nStream Response body:", result) print(response.body.decode("utf-8")) return status_code @@ -189,7 +227,19 @@ async def send_stream_request(connection, host_name): def main(): """Entry point for the example.""" try: + + # Set up Python logging + logging.basicConfig(level=logging.DEBUG) + + # Create and activate redirector + # redirector = PythonLoggingRedirector(base_logger_name="myapp.awscrt") + # redirector.activate(aws_log_level=awscrt.io.LogLevel.Trace) + asyncio.run(make_concurrent_requests()) + # Your AWS CRT operations here... + # Logs will now appear in Python's logging system + + # redirector.deactivate() return 0 except Exception as e: print(f"Exception: {e}", file=sys.stderr) From 143b9e9637ac97495ead764ca6421f3a28d265a3 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 10 Jun 2025 14:39:38 -0700 Subject: [PATCH 03/40] WIP --- awscrt/http.py | 6 +- awscrt/http_asyncio.py | 149 ++++++++++++++++++----------------------- 2 files changed, 68 insertions(+), 87 deletions(-) diff --git a/awscrt/http.py b/awscrt/http.py index b4e2ae6cc..0781786f6 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -398,7 +398,7 @@ def _init_common(self, assert isinstance(connection, HttpClientConnection) assert isinstance(request, HttpRequest) assert callable(on_response) or on_response is None - # assert callable(on_body) or on_body is None + assert callable(on_body) or on_body is None super().__init__(connection, on_body) @@ -588,8 +588,8 @@ def __init__(self, name_value_pairs: Optional[List[Tuple[str, str]]] = None) -> super().__init__() self._binding = _awscrt.http_headers_new() if name_value_pairs: - for i in name_value_pairs: - print("########### headers: ", i) + # for i in name_value_pairs: + # print("########### headers: ", i) self.add_pairs(name_value_pairs) @classmethod diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py index 1ebda10a8..88d649e0f 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/http_asyncio.py @@ -128,11 +128,7 @@ async def close(self) -> None: await asyncio.wrap_future(close_future) def request(self, - request: 'HttpRequest', - on_response: Optional[Callable[..., - None]] = None, - on_body: Optional[Callable[..., - None]] = None) -> 'Http2ClientStreamAsync': + request: 'HttpRequest') -> 'Http2ClientStreamAsync': """Create `HttpClientStreamAsync` to carry out the request/response exchange. NOTE: The HTTP stream sends no data until `HttpClientStreamAsync.activate()` @@ -178,8 +174,7 @@ def request(self, Tuple[HttpClientStreamAsync, HttpResponseAsync]: A tuple containing the stream and a response wrapper that provides async methods to access the response. """ - stream = HttpClientStream(self, request, on_response, on_body) - async_stream = HttpClientStreamAsync._from_stream(stream) + async_stream = HttpClientStreamAsync(self, request) return async_stream @@ -232,10 +227,7 @@ async def new(cls, def request(self, request: 'HttpRequest', - on_response: Optional[Callable[..., - None]] = None, - on_body: Optional[Callable[..., - None]] = None, + on_response: Optional[Callable[..., None]] = None, manual_write: bool = False) -> 'Http2ClientStreamAsync': """Create `Http2ClientStreamAsync` to carry out the request/response exchange. @@ -249,8 +241,7 @@ def request(self, Http2ClientStreamAsync: Stream for the HTTP/2 request/response exchange. """ print("######################### async request called #########################") - stream = Http2ClientStream(self, request, on_response, on_body, manual_write) - async_stream = Http2ClientStreamAsync._from_stream(stream) + async_stream = Http2ClientStreamAsync(self, request, on_response, manual_write) return async_stream @@ -272,8 +263,7 @@ class HttpClientStreamAsync(HttpClientStream): contain an exception indicating why it failed. """ __slots__ = ( - '_response_status_code', - '_on_response_cb', + '_response_status_future', '_on_body_cb', '_request', '_version', @@ -283,70 +273,42 @@ class HttpClientStreamAsync(HttpClientStream): '_stream_completed', '_original_on_body_cb') - @classmethod - def _from_stream(cls, stream: HttpClientStream) -> 'HttpClientStreamAsync': - """Create an HttpClientStreamAsync from an HttpClientStream""" - new_stream = cls.__new__(cls) - # Keep a reference to the original stream to prevent garbage collection - new_stream._original_stream = stream - # Copy the binding and properties from the original stream - new_stream._binding = stream._binding - new_stream._connection = stream._connection - - # Initialize chunk queue and stream completion flag - new_stream._chunk_futures = deque() - new_stream._stream_completed = False - new_stream._received_chunks = deque() - - # Capture the current event loop to use in callbacks + def __init__(self, connection, request): + super()._init_common(connection, request) + + def _init_common(self, connection, request, on_response, http2_manual_write: bool = False) -> None: + super()._init_common(connection, request, on_response, http2_manual_write=http2_manual_write) + self._chunk_futures = deque() + self._stream_completed = False + self._received_chunks = deque() loop = asyncio.get_event_loop() - new_stream._completion_future = loop.create_future() - - # Add a callback to bridge the original future to the asyncio future - def _on_done(fut): - try: - result = fut.result() - loop.call_soon_threadsafe(new_stream._completion_future.set_result, result) - # Mark the stream as completed when the completion_future resolves - loop.call_soon_threadsafe(lambda: setattr(new_stream, '_stream_completed', True)) - except Exception as e: - loop.call_soon_threadsafe(new_stream._completion_future.set_exception, e) - # Mark the stream as completed on error as well - loop.call_soon_threadsafe(lambda: setattr(new_stream, '_stream_completed', True)) - - stream._completion_future.add_done_callback(_on_done) - - # Create a new on_body_cb that puts chunks in the queue - def wrapped_on_body_cb(http_stream, chunk, **kwargs): - # print("################# chunk is", chunk) - if new_stream._chunk_futures: - # print("################# chunk in future is", chunk) - future = new_stream._chunk_futures.popleft() - future.set_result(chunk) - else: - # print("################# chunk in recoved is", chunk) - new_stream._received_chunks.append(chunk) - - # Replace the on_body_cb with our wrapper - stream._on_body_cb = wrapped_on_body_cb - new_stream._on_body_cb = wrapped_on_body_cb - - new_stream._response_status_code = stream._response_status_code - new_stream._on_response_cb = stream._on_response_cb - new_stream._request = stream._request - new_stream._version = stream._version - - return new_stream - - def activate(self) -> None: - print("######################### async activate called #########################") - """Begin sending the request. - - The HTTP stream does nothing until this is called. Call activate() when you - are ready for its callbacks and events to fire. - """ - # Use the original stream's binding - self._original_stream.activate() + self._completion_future = loop.create_future() + self._response_status_future = loop.create_future() + self._response_headers_future = loop.create_future() + + def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: + self._response_status_future.set_result(status_code) + self._response_headers_future.set_result(name_value_pairs) + + # Create a new on_body_cb that puts chunks in the queue + def _on_body(self, chunk: bytes) -> None: + print("################# chunk is", chunk) + if self._chunk_futures: + # print("################# chunk in future is", chunk) + future = self._chunk_futures.popleft() + future.set_result(chunk) + else: + # print("################# chunk in recoved is", chunk) + self._received_chunks.append(chunk) + + def _on_complete(self, error_code: int) -> None: + # done with HttpRequest, drop reference + self._request = None # type: ignore + + if error_code == 0: + self._completion_future.set_result(self._response_status_future.result()) + else: + self._completion_future.set_exception(awscrt.exceptions.from_code(error_code)) async def next(self) -> bytes: # print("######################### async next called #########################") @@ -373,16 +335,35 @@ async def wait_for_completion(self) -> int: """ return await self._completion_future + async def response_status_code(self) -> int: + """Get the response status code asynchronously. + + Returns: + int: The response status code. + """ + return await self._response_status_future -class Http2ClientStreamAsync(HttpClientStreamAsync): + async def response_headers(self) -> List[Tuple[str, str]]: + """Get the response headers asynchronously. + + Returns: + List[Tuple[str, str]]: The response headers as a list of (name, value) tuples. + """ + return await self._response_headers_future + + +class Http2ClientStreamAsync(HttpClientStreamAsync, Http2ClientStream): """HTTP/2 stream that sends a request and receives a response. Create an Http2ClientStreamAsync with `Http2ClientConnectionAsync.request()`. """ - async def write_data(self, - data_stream: Union[InputStream, Any], - end_stream: bool = False) -> None: + def __init__(self, connection, request, on_response, manual_write): + super()._init_common(connection, request, on_response, http2_manual_write=manual_write) + + async def write_data_async(self, + data_stream: Union[InputStream, Any], + end_stream: bool = False) -> None: """Write data to the stream asynchronously. Args: @@ -392,6 +373,6 @@ async def write_data(self, Returns: None: When the write completes. """ - # print("######################### async write_data called #########################") - future = self._original_stream.write_data(data_stream, end_stream) + print("######################### async write_data called #########################") + future = self.write_data(data_stream, end_stream) await asyncio.wrap_future(future) From e87e26eab76cce69fd323666252a49e238260034 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 10 Jun 2025 15:08:07 -0700 Subject: [PATCH 04/40] it works with smithy take2 --- awscrt/http_asyncio.py | 3 +-- examples/http2_asyncio_demo.py | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py index 88d649e0f..c0ac990f1 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/http_asyncio.py @@ -292,7 +292,6 @@ def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]] # Create a new on_body_cb that puts chunks in the queue def _on_body(self, chunk: bytes) -> None: - print("################# chunk is", chunk) if self._chunk_futures: # print("################# chunk in future is", chunk) future = self._chunk_futures.popleft() @@ -373,6 +372,6 @@ async def write_data_async(self, Returns: None: When the write completes. """ - print("######################### async write_data called #########################") + # print("######################### async write_data called #########################") future = self.write_data(data_stream, end_stream) await asyncio.wrap_future(future) diff --git a/examples/http2_asyncio_demo.py b/examples/http2_asyncio_demo.py index f99c2831a..67178c6a1 100644 --- a/examples/http2_asyncio_demo.py +++ b/examples/http2_asyncio_demo.py @@ -142,7 +142,7 @@ async def send_get_request(connection, host_name): # Set up response handler response = Response("GET") - stream = connection.request(request, response.on_response, response.on_body) + stream = connection.request(request) stream.activate() # Wait for completion @@ -171,7 +171,7 @@ async def send_post_request(connection, host_name): # Set up response handler response = Response("POST") - stream = connection.request(request, response.on_response, response.on_body) + stream = connection.request(request) stream.activate() # Wait for completion @@ -194,7 +194,7 @@ async def send_stream_request(connection, host_name): # Set up response handler response = Response("STREAM") - stream = connection.request(request, response.on_response, response.on_body, manual_write=True) + stream = connection.request(request, manual_write=True) stream.activate() # Stream data in chunks @@ -208,14 +208,21 @@ async def send_stream_request(connection, host_name): # print(f"Sending chunk {i + 1}/{len(data_chunks)}, size: {len(chunk)} bytes") # # Use BytesIO for each chunk # chunk_stream = io.BytesIO(chunk) - # await stream.write_data(chunk_stream, end_stream=(i == len(data_chunks) - 1)) + # await stream.write_data_async(chunk_stream, end_stream=(i == len(data_chunks) - 1)) # # Simulate processing time between chunks # await asyncio.sleep(0.1) - await stream.write_data(io.BytesIO(data_chunks[0]), end_stream=False) - await stream.write_data(io.BytesIO(data_chunks[1]), end_stream=True) - # await stream.write_data(io.BytesIO(data_chunks[1]), end_stream=True) + await stream.write_data_async(io.BytesIO(data_chunks[0]), end_stream=False) + await stream.write_data_async(io.BytesIO(data_chunks[1]), end_stream=True) + # await stream.write_data_async(io.BytesIO(data_chunks[1]), end_stream=True) result = await stream.next() + status_code = await stream.response_status_code() + print(f"Stream request completed with status code: {status_code}") + headers = await stream.response_headers() + print("\nStream Response headers:") + for name, value in headers: + print(f"{name}: {value}") + # Wait for completion status_code = await stream.wait_for_completion() print(f"Stream request completed with status code: {status_code}") From e46471ccd0c855ddf734bd4117d08486d6da922d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 10 Jun 2025 15:55:14 -0700 Subject: [PATCH 05/40] clean up --- awscrt/http.py | 3 - awscrt/http_asyncio.py | 126 ++++++++++------------------------------- 2 files changed, 29 insertions(+), 100 deletions(-) diff --git a/awscrt/http.py b/awscrt/http.py index 0781786f6..c24dc1a9b 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -429,11 +429,9 @@ def activate(self) -> None: The HTTP stream does nothing until this is called. Call activate() when you are ready for its callbacks and events to fire. """ - print("########### activate") _awscrt.http_client_stream_activate(self) def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: - print("########### _on_response, status_code:", status_code) self._response_status_code = status_code if self._on_response_cb: @@ -535,7 +533,6 @@ def __init__(self, binding = _awscrt.http_message_new_request(headers) super().__init__(binding, headers, body_stream) - print("########### body_stream: ", body_stream) self.method = method self.path = path diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py index c0ac990f1..09b6677d0 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/http_asyncio.py @@ -8,15 +8,13 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0. -import _awscrt import asyncio from concurrent.futures import Future -from awscrt import NativeResource import awscrt.exceptions -from typing import List, Tuple, Dict, Optional, Union, Iterator, Callable, Any +from typing import List, Tuple, Optional, Union, Callable, Any from awscrt.http import ( - HttpVersion, HttpClientConnection, HttpRequest, HttpClientStream, HttpProxyOptions, - Http2Setting, Http2ClientConnection, HttpConnectionBase, HttpHeaders, Http2ClientStream + HttpClientConnection, HttpRequest, HttpClientStream, HttpProxyOptions, + Http2Setting, Http2ClientConnection, HttpConnectionBase, Http2ClientStream ) from awscrt.io import ( ClientBootstrap, SocketOptions, TlsConnectionOptions, InputStream @@ -24,22 +22,6 @@ from collections import deque -def _future_to_async(future: Future) -> asyncio.Future: - """Convert a concurrent.futures.Future to asyncio.Future""" - loop = asyncio.get_event_loop() - async_future = loop.create_future() - - def _on_done(fut): - try: - result = fut.result() - loop.call_soon_threadsafe(async_future.set_result, result) - except Exception as e: - loop.call_soon_threadsafe(async_future.set_exception, e) - - future.add_done_callback(_on_done) - return async_future - - class HttpClientConnectionAsync(HttpClientConnection): """ An async HTTP client connection. @@ -91,12 +73,11 @@ async def new(cls, return HttpClientConnectionAsync._from_connection(connection) @classmethod - def _from_connection(cls, connection): + def _from_connection(cls, connection: HttpClientConnection) -> "HttpClientConnectionAsync": """Create an HttpClientConnectionAsync from an HttpClientConnection""" new_conn = cls.__new__(cls) # Copy the binding and properties from the original connection new_conn._binding = connection._binding - new_conn._shutdown_future = connection._shutdown_future new_conn._version = connection._version new_conn._host_name = connection._host_name new_conn._port = connection._port @@ -128,55 +109,16 @@ async def close(self) -> None: await asyncio.wrap_future(close_future) def request(self, - request: 'HttpRequest') -> 'Http2ClientStreamAsync': + request: 'HttpRequest') -> 'HttpClientStreamAsync': """Create `HttpClientStreamAsync` to carry out the request/response exchange. - NOTE: The HTTP stream sends no data until `HttpClientStreamAsync.activate()` - is called. Call activate() when you're ready for callbacks and events to fire. - Args: request (HttpRequest): Definition for outgoing request. - on_response: Optional callback invoked once main response headers are received. - The function should take the following arguments and return nothing: - - * `http_stream` (`HttpClientStreamAsync`): HTTP stream carrying - out this request/response exchange. - - * `status_code` (int): Response status code. - - * `headers` (List[Tuple[str, str]]): Response headers as a - list of (name,value) pairs. - - * `**kwargs` (dict): Forward compatibility kwargs. - - An exception raise by this function will cause the HTTP stream to end in error. - This callback is always invoked on the connection's event-loop thread. - - on_body: Optional callback invoked 0+ times as response body data is received. - The function should take the following arguments and return nothing: - - * `http_stream` (`HttpClientStreamAsync`): HTTP stream carrying - out this request/response exchange. - - * `chunk` (buffer): Response body data (not necessarily - a whole "chunk" of chunked encoding). - - * `**kwargs` (dict): Forward-compatibility kwargs. - - An exception raise by this function will cause the HTTP stream to end in error. - This callback is always invoked on the connection's event-loop thread. - Returns: - If use_response_wrapper is False: - HttpClientStreamAsync: Stream for the HTTP request/response exchange. - If use_response_wrapper is True: - Tuple[HttpClientStreamAsync, HttpResponseAsync]: A tuple containing the stream - and a response wrapper that provides async methods to access the response. + HttpClientStreamAsync: Stream for the HTTP request/response exchange. """ - async_stream = HttpClientStreamAsync(self, request) - - return async_stream + return HttpClientStreamAsync(self, request) class Http2ClientConnectionAsync(HttpClientConnectionAsync): @@ -227,23 +169,17 @@ async def new(cls, def request(self, request: 'HttpRequest', - on_response: Optional[Callable[..., None]] = None, manual_write: bool = False) -> 'Http2ClientStreamAsync': """Create `Http2ClientStreamAsync` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. - on_response: Optional callback invoked once main response headers are received. - on_body: Optional callback invoked 0+ times as response body data is received. manual_write (bool): If True, enables manual data writing on the stream. Returns: Http2ClientStreamAsync: Stream for the HTTP/2 request/response exchange. """ - print("######################### async request called #########################") - async_stream = Http2ClientStreamAsync(self, request, on_response, manual_write) - - return async_stream + return Http2ClientStreamAsync(self, request, manual_write) class HttpClientStreamAsync(HttpClientStream): @@ -251,9 +187,6 @@ class HttpClientStreamAsync(HttpClientStream): Create an HttpClientStreamAsync with `HttpClientConnectionAsync.request()`. - NOTE: The HTTP stream sends no data until `HttpClientStreamAsync.activate()` - is called. Call activate() when you're ready for callbacks and events to fire. - Attributes: connection (HttpClientConnectionAsync): This stream's connection. @@ -264,53 +197,53 @@ class HttpClientStreamAsync(HttpClientStream): """ __slots__ = ( '_response_status_future', - '_on_body_cb', - '_request', - '_version', - '_original_stream', + '_response_headers_future', + '_chunk_futures', + '_received_chunks', '_completion_future', - '_chunk_queue', - '_stream_completed', - '_original_on_body_cb') + '_stream_completed') - def __init__(self, connection, request): + def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest) -> None: super()._init_common(connection, request) - def _init_common(self, connection, request, on_response, http2_manual_write: bool = False) -> None: - super()._init_common(connection, request, on_response, http2_manual_write=http2_manual_write) + def _init_common(self, connection: HttpClientConnectionAsync, + request: HttpRequest, + http2_manual_write: bool = False) -> None: + # Initialize the parent class + super()._init_common(connection, request, http2_manual_write=http2_manual_write) + + # Set up async state tracking + loop = asyncio.get_event_loop() self._chunk_futures = deque() - self._stream_completed = False self._received_chunks = deque() - loop = asyncio.get_event_loop() + self._stream_completed = False + + # Create futures for async operations self._completion_future = loop.create_future() self._response_status_future = loop.create_future() self._response_headers_future = loop.create_future() + # Activate the stream immediately + self.activate() + def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: self._response_status_future.set_result(status_code) self._response_headers_future.set_result(name_value_pairs) - # Create a new on_body_cb that puts chunks in the queue def _on_body(self, chunk: bytes) -> None: if self._chunk_futures: - # print("################# chunk in future is", chunk) future = self._chunk_futures.popleft() future.set_result(chunk) else: - # print("################# chunk in recoved is", chunk) self._received_chunks.append(chunk) def _on_complete(self, error_code: int) -> None: - # done with HttpRequest, drop reference - self._request = None # type: ignore - if error_code == 0: self._completion_future.set_result(self._response_status_future.result()) else: self._completion_future.set_exception(awscrt.exceptions.from_code(error_code)) async def next(self) -> bytes: - # print("######################### async next called #########################") """Get the next chunk from the response body. Returns: @@ -357,8 +290,8 @@ class Http2ClientStreamAsync(HttpClientStreamAsync, Http2ClientStream): Create an Http2ClientStreamAsync with `Http2ClientConnectionAsync.request()`. """ - def __init__(self, connection, request, on_response, manual_write): - super()._init_common(connection, request, on_response, http2_manual_write=manual_write) + def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, manual_write: bool) -> None: + super()._init_common(connection, request, http2_manual_write=manual_write) async def write_data_async(self, data_stream: Union[InputStream, Any], @@ -372,6 +305,5 @@ async def write_data_async(self, Returns: None: When the write completes. """ - # print("######################### async write_data called #########################") future = self.write_data(data_stream, end_stream) await asyncio.wrap_future(future) From 6c802b15ec3c2a6a68224936456d0edfdcab2c3f Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 10 Jun 2025 16:43:37 -0700 Subject: [PATCH 06/40] fix test --- awscrt/http.py | 1 - awscrt/http_asyncio.py | 28 +++++++++++++++++++------- test/test_http_asyncio.py | 41 ++++++++++++++++++++++++++------------- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/awscrt/http.py b/awscrt/http.py index c24dc1a9b..bf3513e64 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -408,7 +408,6 @@ def _init_common(self, # keep HttpRequest alive until stream completes self._request: 'HttpRequest' = request self._version: HttpVersion = connection.version - print("########### http2_manual_write: ", http2_manual_write) self._binding = _awscrt.http_client_stream_new(self, connection, request, http2_manual_write) @property diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py index 09b6677d0..9e5759966 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/http_asyncio.py @@ -201,10 +201,12 @@ class HttpClientStreamAsync(HttpClientStream): '_chunk_futures', '_received_chunks', '_completion_future', - '_stream_completed') + '_stream_completed', + '_status_code', + '_loop') def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest) -> None: - super()._init_common(connection, request) + self._init_common(connection, request) def _init_common(self, connection: HttpClientConnectionAsync, request: HttpRequest, @@ -213,20 +215,27 @@ def _init_common(self, connection: HttpClientConnectionAsync, super()._init_common(connection, request, http2_manual_write=http2_manual_write) # Set up async state tracking - loop = asyncio.get_event_loop() + self._loop = asyncio.get_event_loop() self._chunk_futures = deque() self._received_chunks = deque() self._stream_completed = False # Create futures for async operations - self._completion_future = loop.create_future() - self._response_status_future = loop.create_future() - self._response_headers_future = loop.create_future() + self._completion_future = self._loop.create_future() + self._response_status_future = self._loop.create_future() + self._response_headers_future = self._loop.create_future() + self._status_code = None # Activate the stream immediately self.activate() def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: + self._status_code = status_code + # invoked from the C thread, so we need to schedule the result setting on the event loop + self._loop.call_soon_threadsafe(self._set_response, status_code, name_value_pairs) + + def _set_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: + """Set the response status and headers in the futures.""" self._response_status_future.set_result(status_code) self._response_headers_future.set_result(name_value_pairs) @@ -238,8 +247,13 @@ def _on_body(self, chunk: bytes) -> None: self._received_chunks.append(chunk) def _on_complete(self, error_code: int) -> None: + # invoked from the C thread, so we need to schedule the result setting on the event loop + self._loop.call_soon_threadsafe(self._set_completion, error_code) + + def _set_completion(self, error_code: int) -> None: + """Set the completion status of the stream.""" if error_code == 0: - self._completion_future.set_result(self._response_status_future.result()) + self._completion_future.set_result(self._status_code) else: self._completion_future.set_exception(awscrt.exceptions.from_code(error_code)) diff --git a/test/test_http_asyncio.py b/test/test_http_asyncio.py index 692ef5ed3..1acf3a9c5 100644 --- a/test/test_http_asyncio.py +++ b/test/test_http_asyncio.py @@ -26,12 +26,22 @@ def __init__(self): self.headers = None self.body = bytearray() - def on_response(self, http_stream, status_code, headers, **kwargs): - self.status_code = status_code - self.headers = HttpHeaders(headers) + async def collect_response(self, stream): + """Collects complete response from a stream""" + # Get status code and headers + self.status_code = await stream.response_status_code() + headers_list = await stream.response_headers() + self.headers = HttpHeaders(headers_list) - def on_body(self, http_stream, chunk, **kwargs): - self.body.extend(chunk) + # Collect body chunks + while True: + chunk = await stream.next() + if not chunk: + break + self.body.extend(chunk) + + # Return status code for convenience + return self.status_code class TestRequestHandler(SimpleHTTPRequestHandler): @@ -115,16 +125,19 @@ async def _test_get(self, secure): self._start_server(secure) try: connection = await self._new_client_connection(secure) + self.assertTrue(connection.is_open()) test_asset_path = 'test/test_http_asyncio.py' + # Create request and get stream - stream is already activated request = HttpRequest('GET', '/' + test_asset_path) + stream = connection.request(request) + + # Collect and process response response = Response() - stream = connection.request(request, response.on_response, response.on_body) - stream.activate() + status_code = await response.collect_response(stream) - # wait for stream to complete - status_code = await stream.wait_for_completion() + # Verify results self.assertEqual(200, status_code) self.assertEqual(200, response.status_code) @@ -152,13 +165,15 @@ async def _test_put(self, secure): # seek back to start of stream before trying to send it outgoing_body_stream.seek(0) + # Create request and get stream - stream is already activated request = HttpRequest('PUT', '/' + test_asset_path, headers, outgoing_body_stream) + stream = connection.request(request) + + # Collect and process response response = Response() - http_stream = connection.request(request, response.on_response, response.on_body) - http_stream.activate() + status_code = await response.collect_response(stream) - # wait for stream to complete - status_code = await http_stream.wait_for_completion() + # Verify results self.assertEqual(200, status_code) self.assertEqual(200, response.status_code) From b8af49913ee9164531ae91bd06bb2d64a8c7a414 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 10 Jun 2025 16:51:00 -0700 Subject: [PATCH 07/40] clean up --- awscrt/http.py | 3 - crt/aws-c-cal | 2 +- crt/aws-c-http | 2 +- crt/aws-c-io | 2 +- crt/aws-c-mqtt | 2 +- crt/aws-c-s3 | 2 +- crt/aws-c-sdkutils | 2 +- crt/aws-lc | 2 +- crt/s2n | 2 +- examples/http2_asyncio_demo.py | 257 --------------------------------- examples/http_asyncio_demo.py | 95 ------------ 11 files changed, 8 insertions(+), 363 deletions(-) delete mode 100644 examples/http2_asyncio_demo.py delete mode 100644 examples/http_asyncio_demo.py diff --git a/awscrt/http.py b/awscrt/http.py index bf3513e64..becf6c8c4 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -359,7 +359,6 @@ def completion_future(self) -> "concurrent.futures.Future": return self._completion_future def _on_body(self, chunk: bytes) -> None: - # print("########### _on_body, chunk size:", len(chunk)) if self._on_body_cb: self._on_body_cb(http_stream=self, chunk=chunk) @@ -584,8 +583,6 @@ def __init__(self, name_value_pairs: Optional[List[Tuple[str, str]]] = None) -> super().__init__() self._binding = _awscrt.http_headers_new() if name_value_pairs: - # for i in name_value_pairs: - # print("########### headers: ", i) self.add_pairs(name_value_pairs) @classmethod diff --git a/crt/aws-c-cal b/crt/aws-c-cal index fa108de52..8703b3e59 160000 --- a/crt/aws-c-cal +++ b/crt/aws-c-cal @@ -1 +1 @@ -Subproject commit fa108de5280afd71018e0a0534edb36b33f030f6 +Subproject commit 8703b3e5930c9fd508025b268ab837fc9df3c4fd diff --git a/crt/aws-c-http b/crt/aws-c-http index 3eedf1ef8..10961a708 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 3eedf1ef8c6874cd941dbde794a6ab3bd979e181 +Subproject commit 10961a708a4148c57db139232277573da2f6e99c diff --git a/crt/aws-c-io b/crt/aws-c-io index 8286c781b..689dee3cb 160000 --- a/crt/aws-c-io +++ b/crt/aws-c-io @@ -1 +1 @@ -Subproject commit 8286c781b95b426ca2f0783b6c1fe49ff519c4e7 +Subproject commit 689dee3cb8dbd8a6906431d154a3695f7688c056 diff --git a/crt/aws-c-mqtt b/crt/aws-c-mqtt index 9fc2f573c..3ac506507 160000 --- a/crt/aws-c-mqtt +++ b/crt/aws-c-mqtt @@ -1 +1 @@ -Subproject commit 9fc2f573c0fb608c052230d4f2495725d7252285 +Subproject commit 3ac506507679a86677f8875dcc07589b63907863 diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 7d2d4b307..233c587f2 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 7d2d4b3070109c882ff78e8719f60597c7ba0472 +Subproject commit 233c587f29fab457c1874988c19e04a2a8c8c00b diff --git a/crt/aws-c-sdkutils b/crt/aws-c-sdkutils index ba6a28fab..f678bda9e 160000 --- a/crt/aws-c-sdkutils +++ b/crt/aws-c-sdkutils @@ -1 +1 @@ -Subproject commit ba6a28fab7ed5d7f1b3b1d12eb672088be093824 +Subproject commit f678bda9e21f7217e4bbf35e0d1ea59540687933 diff --git a/crt/aws-lc b/crt/aws-lc index a614f9752..d6ade6ae1 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit a614f97527d16461d5c904ef90d3bb647e35265f +Subproject commit d6ade6ae1537adfff53c0f0489b99ba1a111f0cc diff --git a/crt/s2n b/crt/s2n index 1c5798b82..a772605d2 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 1c5798b82442067bace943f748f4f24ae1770bed +Subproject commit a772605d27afcb62c0e0d1ee92f9003cb11ca8ef diff --git a/examples/http2_asyncio_demo.py b/examples/http2_asyncio_demo.py deleted file mode 100644 index 67178c6a1..000000000 --- a/examples/http2_asyncio_demo.py +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env python3 -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0. - -""" -This example demonstrates how to use the asyncio HTTP/2 client in awscrt. -It performs multiple concurrent requests to httpbin.org and shows HTTP/2 features. -""" - -import asyncio -import sys -import io -from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions -from awscrt.http import HttpHeaders, HttpRequest, Http2Setting, Http2SettingID -from awscrt.http_asyncio import Http2ClientConnectionAsync -from awscrt_python_logging_example import PythonLoggingRedirector -import awscrt.io -import logging - - -class Response: - """Holds contents of incoming response""" - - def __init__(self, request_name): - self.request_name = request_name - self.status_code = None - self.headers = None - self.body = bytearray() - - def on_response(self, http_stream, status_code, headers, **kwargs): - print(f"[{self.request_name}] Received response status: {status_code}") - self.status_code = status_code - self.headers = HttpHeaders(headers) - for name, value in headers: - print(f"[{self.request_name}] Header: {name}: {value}") - - def on_body(self, http_stream, chunk, **kwargs): - print(f"[{self.request_name}] Received body chunk of size: {len(chunk)} bytes") - self.body.extend(chunk) - - -# Create an event for synchronizing remote settings -remote_settings_event = None -event_loop = None - - -def on_remote_settings_changed(settings): - """Handler for when the server updates HTTP/2 settings""" - print("Remote HTTP/2 settings changed:") - for setting in settings: - print(f" - {setting.id.name} = {setting.value}") - # Signal that remote settings have been received - # This callback is called from a different thread, so we need to use call_soon_threadsafe - if event_loop and remote_settings_event: - event_loop.call_soon_threadsafe(remote_settings_event.set) - - -async def make_concurrent_requests(): - """Perform multiple concurrent HTTP/2 requests asynchronously.""" - global remote_settings_event, event_loop - - # Get the current event loop and create the event - event_loop = asyncio.get_running_loop() - remote_settings_event = asyncio.Event() - - # Create an event loop group and default host resolver - event_loop_group = EventLoopGroup() - host_resolver = DefaultHostResolver(event_loop_group) - bootstrap = ClientBootstrap(event_loop_group, host_resolver) - - # Connect to httpbin.org - host_name = "localhost" # Change to "httpbin.org" for real requests - port = 3443 - - # TLS options for HTTP/2 - tls_ctx_opt = TlsContextOptions() - tls_ctx_opt.verify_peer = False - tls_ctx = ClientTlsContext(tls_ctx_opt) - tls_conn_opt = tls_ctx.new_connection_options() - tls_conn_opt.set_server_name(host_name) - tls_conn_opt.set_alpn_list(["h2"]) # Set ALPN to HTTP/2 - - # Initial HTTP/2 settings - initial_settings = [ - Http2Setting(Http2SettingID.ENABLE_PUSH, 0), - Http2Setting(Http2SettingID.MAX_CONCURRENT_STREAMS, 100), - Http2Setting(Http2SettingID.INITIAL_WINDOW_SIZE, 65535), - ] - - print(f"Connecting to {host_name}:{port} using HTTP/2...") - connection = await Http2ClientConnectionAsync.new( - host_name=host_name, - port=port, - bootstrap=bootstrap, - tls_connection_options=tls_conn_opt, - initial_settings=initial_settings, - on_remote_settings_changed=on_remote_settings_changed - ) - print("HTTP/2 Connection established!") - - # Wait for remote settings to be received - print("Waiting for remote settings...") - await remote_settings_event.wait() - print("Remote settings received, proceeding with requests...") - - try: - # Create several requests to be executed concurrently - tasks = [] - - # # Request 1: Simple GET - # tasks.append(send_get_request(connection, host_name)) - - # # Request 2: POST with JSON body - # tasks.append(send_post_request(connection, host_name)) - - # Request 3: Stream data using manual write mode - tasks.append(send_stream_request(connection, host_name)) - - # Wait for all requests to complete - results = await asyncio.gather(*tasks, return_exceptions=True) - - # Check for any exceptions - for i, result in enumerate(results): - if isinstance(result, Exception): - print(f"Task {i} failed with exception: {result}") - - finally: - # Add a small delay to ensure all responses are received - await asyncio.sleep(1) - - # Close the connection - print("Closing connection...") - await connection.close() - print("Connection closed!") - - -async def send_get_request(connection, host_name): - """Send a GET request using the HTTP/2 connection.""" - print("Sending GET request...") - request = HttpRequest("GET", "/get?param1=value1¶m2=value2") - request.headers.add("host", host_name) - - # Set up response handler - response = Response("GET") - stream = connection.request(request) - stream.activate() - - # Wait for completion - status_code = await stream.wait_for_completion() - print(f"GET request completed with status code: {status_code}") - print("\nGET Response body:") - print(response.body.decode("utf-8")) - return status_code - - -async def send_post_request(connection, host_name): - """Send a POST request with JSON body using the HTTP/2 connection.""" - print("Sending POST request with JSON body...") - - # Prepare JSON payload - json_payload = '{"name": "Example User", "id": 12345}' - - # Create request with headers - request = HttpRequest("POST", "/post") - request.headers.add("host", host_name) - request.headers.add("content-type", "application/json") - request.headers.add("content-length", str(len(json_payload))) - - # Set the body using BytesIO stream - request.body_stream = io.BytesIO(json_payload.encode("utf-8")) - - # Set up response handler - response = Response("POST") - stream = connection.request(request) - stream.activate() - - # Wait for completion - status_code = await stream.wait_for_completion() - print(f"POST request completed with status code: {status_code}") - print("\nPOST Response body:") - print(response.body.decode("utf-8")) - return status_code - - -async def send_stream_request(connection, host_name): - """Send a request with streamed data using manual write mode.""" - print("Sending request with manual data streaming...") - - # Create request - request = HttpRequest("PUT", "/put") - request.headers.add("host", host_name) - request.headers.add("content-type", "text/plain") - # Note: We don't set content-length as we're streaming the data - - # Set up response handler - response = Response("STREAM") - stream = connection.request(request, manual_write=True) - stream.activate() - - # Stream data in chunks - data_chunks = [ - b"This is the first chunk of data.\n", - b"This is the second chunk of data.\n", - b"This is the final chunk of data." - ] - - # for i, chunk in enumerate(data_chunks): - # print(f"Sending chunk {i + 1}/{len(data_chunks)}, size: {len(chunk)} bytes") - # # Use BytesIO for each chunk - # chunk_stream = io.BytesIO(chunk) - # await stream.write_data_async(chunk_stream, end_stream=(i == len(data_chunks) - 1)) - # # Simulate processing time between chunks - # await asyncio.sleep(0.1) - await stream.write_data_async(io.BytesIO(data_chunks[0]), end_stream=False) - await stream.write_data_async(io.BytesIO(data_chunks[1]), end_stream=True) - # await stream.write_data_async(io.BytesIO(data_chunks[1]), end_stream=True) - result = await stream.next() - - status_code = await stream.response_status_code() - print(f"Stream request completed with status code: {status_code}") - headers = await stream.response_headers() - print("\nStream Response headers:") - for name, value in headers: - print(f"{name}: {value}") - - # Wait for completion - status_code = await stream.wait_for_completion() - print(f"Stream request completed with status code: {status_code}") - print("\nStream Response body:", result) - print(response.body.decode("utf-8")) - return status_code - - -def main(): - """Entry point for the example.""" - try: - - # Set up Python logging - logging.basicConfig(level=logging.DEBUG) - - # Create and activate redirector - # redirector = PythonLoggingRedirector(base_logger_name="myapp.awscrt") - # redirector.activate(aws_log_level=awscrt.io.LogLevel.Trace) - - asyncio.run(make_concurrent_requests()) - # Your AWS CRT operations here... - # Logs will now appear in Python's logging system - - # redirector.deactivate() - return 0 - except Exception as e: - print(f"Exception: {e}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/http_asyncio_demo.py b/examples/http_asyncio_demo.py deleted file mode 100644 index 98f76d314..000000000 --- a/examples/http_asyncio_demo.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python3 -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0. - -""" -This example demonstrates how to use the asyncio HTTP client in awscrt. -It performs a simple GET request to httpbin.org and prints the response. -""" - -import asyncio -import sys -from awscrt.io import ClientBootstrap, DefaultHostResolver, EventLoopGroup -from awscrt.http import HttpHeaders, HttpRequest -from awscrt.http_asyncio import HttpClientConnectionAsync - - -class Response: - """Holds contents of incoming response""" - - def __init__(self): - self.status_code = None - self.headers = None - self.body = bytearray() - - def on_response(self, http_stream, status_code, headers, **kwargs): - print(f"Received response status: {status_code}") - self.status_code = status_code - self.headers = HttpHeaders(headers) - for name, value in headers: - print(f"Header: {name}: {value}") - - def on_body(self, http_stream, chunk, **kwargs): - print(f"Received body chunk of size: {len(chunk)} bytes") - self.body.extend(chunk) - - -async def make_request(): - """Perform an HTTP GET request asynchronously using the AWS CRT HTTP client.""" - # Create an event loop group and default host resolver - event_loop_group = EventLoopGroup() - host_resolver = DefaultHostResolver(event_loop_group) - bootstrap = ClientBootstrap(event_loop_group, host_resolver) - - # Connect to httpbin.org - host_name = "httpbin.org" - port = 443 - - print(f"Connecting to {host_name}:{port}...") - connection = await HttpClientConnectionAsync.new( - host_name=host_name, - port=port, - bootstrap=bootstrap, - tls_connection_options=None # For HTTPS, you would provide TLS options here - ) - print("Connection established!") - - # Create and send a simple GET request - print("Sending request...") - request = HttpRequest("GET", "/get") - request.headers.add("host", host_name) - - # Set up response handlers - response = Response() - print("before request") - stream = connection.request(request, response.on_response, response.on_body) - print("after request") - stream.activate() - print("activated") - - # Wait for the response to complete - status_code = await stream.wait_for_completion() - print(f"Request completed with status code: {status_code}") - - # Print the response body - print("\nResponse body:") - print(response.body.decode('utf-8')) - - # Close the connection - print("Closing connection...") - await connection.close() - print("Connection closed!") - - -def main(): - """Entry point for the example.""" - try: - asyncio.run(make_request()) - return 0 - except Exception as e: - print(f"Exception: {e}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) From 5e8eb43644ef31e67edaaf6fdfe3147529c883b9 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 11 Jun 2025 15:16:24 -0700 Subject: [PATCH 08/40] couple fixes --- awscrt/http.py | 24 +- awscrt/http_asyncio.py | 112 +++++---- test/test_http_asyncio.py | 501 +++++++++++++++++++++++++++++++++++++- 3 files changed, 578 insertions(+), 59 deletions(-) diff --git a/awscrt/http.py b/awscrt/http.py index becf6c8c4..2246dc203 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -190,7 +190,8 @@ def _generic_new( proxy_options: Optional['HttpProxyOptions'] = None, expected_version: Optional[HttpVersion] = None, initial_settings: Optional[List[Http2Setting]] = None, - on_remote_settings_changed: Optional[Callable[[List[Http2Setting]], None]] = None) -> "concurrent.futures.Future": + on_remote_settings_changed: Optional[Callable[[List[Http2Setting]], None]] = None, + asyncio_connection=False) -> "concurrent.futures.Future": """ Initialize the generic part of the HttpClientConnection class. """ @@ -217,7 +218,8 @@ def _generic_new( tls_connection_options=tls_connection_options, connect_future=future, expected_version=expected_version, - on_remote_settings_changed=on_remote_settings_changed) + on_remote_settings_changed=on_remote_settings_changed, + asyncio_connection=asyncio_connection) _awscrt.http_client_connection_new( bootstrap, @@ -808,7 +810,8 @@ def __init__( tls_connection_options: Optional[TlsConnectionOptions] = None, connect_future: Optional[Future] = None, expected_version: Optional[HttpVersion] = None, - on_remote_settings_changed: Optional[Callable[[List[Http2Setting]], None]] = None) -> None: + on_remote_settings_changed: Optional[Callable[[List[Http2Setting]], None]] = None, + asyncio_connection=False) -> None: self._shutdown_future = None self._host_name = host_name self._port = port @@ -817,6 +820,7 @@ def __init__( self._connect_future = connect_future self._expected_version = expected_version self._on_remote_settings_changed_from_user = on_remote_settings_changed + self._asyncio_connection = asyncio_connection def _on_connection_setup(self, binding: Any, error_code: int, http_version: HttpVersion) -> None: if self._connect_future is None: @@ -829,10 +833,18 @@ def _on_connection_setup(self, binding: Any, error_code: int, http_version: Http # AWS_ERROR_HTTP_UNSUPPORTED_PROTOCOL self._connect_future.set_exception(awscrt.exceptions.from_code(2060)) return - if http_version == HttpVersion.Http2: - connection = Http2ClientConnection() + if self._asyncio_connection: + # Import is done here to avoid circular import issues + from awscrt.http_asyncio import HttpClientConnectionAsync, Http2ClientConnectionAsync + if http_version == HttpVersion.Http2: + connection = Http2ClientConnectionAsync() + else: + connection = HttpClientConnectionAsync() else: - connection = HttpClientConnection() + if http_version == HttpVersion.Http2: + connection = Http2ClientConnection() + else: + connection = HttpClientConnection() connection._host_name = self._host_name connection._port = self._port diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py index 9e5759966..5ab51cc66 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/http_asyncio.py @@ -3,6 +3,16 @@ This module provides asyncio wrappers around the awscrt.http module. All network operations in `awscrt.http_asyncio` are asynchronous and use Python's asyncio framework. + +Threading Notes: + - Each asyncio event loop is typically associated with a specific thread + - When using HTTP streams across threads, you must specify the appropriate event loop + when creating the stream to avoid "Future attached to a different loop" errors + - For example, if you create a connection in Thread A but want to use it in Thread B, + pass Thread B's event loop when creating the stream: + `stream = connection.request(request, loop=thread_b_event_loop)` + - All async operations on a stream (await stream.next(), etc.) must be performed in the + thread that owns the event loop used to create the stream """ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. @@ -14,7 +24,7 @@ from typing import List, Tuple, Optional, Union, Callable, Any from awscrt.http import ( HttpClientConnection, HttpRequest, HttpClientStream, HttpProxyOptions, - Http2Setting, Http2ClientConnection, HttpConnectionBase, Http2ClientStream + Http2Setting, HttpVersion, Http2ClientStream ) from awscrt.io import ( ClientBootstrap, SocketOptions, TlsConnectionOptions, InputStream @@ -62,39 +72,15 @@ async def new(cls, Returns: HttpClientConnectionAsync: A new HTTP client connection. """ - future = HttpClientConnection.new( + future = HttpClientConnection._generic_new( host_name, port, bootstrap, socket_options, tls_connection_options, - proxy_options) - connection = await asyncio.wrap_future(future) - return HttpClientConnectionAsync._from_connection(connection) - - @classmethod - def _from_connection(cls, connection: HttpClientConnection) -> "HttpClientConnectionAsync": - """Create an HttpClientConnectionAsync from an HttpClientConnection""" - new_conn = cls.__new__(cls) - # Copy the binding and properties from the original connection - new_conn._binding = connection._binding - new_conn._version = connection._version - new_conn._host_name = connection._host_name - new_conn._port = connection._port - # Initialize the parent class without calling __init__ - HttpConnectionBase.__init__(new_conn) - new_conn._shutdown_future = connection._shutdown_future - return new_conn - - @property - def host_name(self) -> str: - """Remote hostname""" - return self._host_name - - @property - def port(self) -> int: - """Remote port""" - return self._port + proxy_options, + asyncio_connection=True) + return await asyncio.wrap_future(future) async def close(self) -> None: """Close the connection asynchronously. @@ -109,16 +95,19 @@ async def close(self) -> None: await asyncio.wrap_future(close_future) def request(self, - request: 'HttpRequest') -> 'HttpClientStreamAsync': + request: 'HttpRequest', + loop: Optional[asyncio.AbstractEventLoop] = None) -> 'HttpClientStreamAsync': """Create `HttpClientStreamAsync` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. + loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. + If None, the current event loop is used. Returns: HttpClientStreamAsync: Stream for the HTTP request/response exchange. """ - return HttpClientStreamAsync(self, request) + return HttpClientStreamAsync(self, request, loop) class Http2ClientConnectionAsync(HttpClientConnectionAsync): @@ -154,32 +143,35 @@ async def new(cls, * `settings` (List[Http2Setting]): List of settings that were changed. """ - future = Http2ClientConnection.new( + future = HttpClientConnection._generic_new( host_name, port, bootstrap, socket_options, tls_connection_options, proxy_options, + HttpVersion.Http2, initial_settings, - on_remote_settings_changed) - - connection = await asyncio.wrap_future(future) - return Http2ClientConnectionAsync._from_connection(connection) + on_remote_settings_changed, + asyncio_connection=True) + return await asyncio.wrap_future(future) def request(self, request: 'HttpRequest', - manual_write: bool = False) -> 'Http2ClientStreamAsync': + manual_write: bool = False, + loop: Optional[asyncio.AbstractEventLoop] = None) -> 'Http2ClientStreamAsync': """Create `Http2ClientStreamAsync` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. manual_write (bool): If True, enables manual data writing on the stream. + loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. + If None, the current event loop is used. Returns: Http2ClientStreamAsync: Stream for the HTTP/2 request/response exchange. """ - return Http2ClientStreamAsync(self, request, manual_write) + return Http2ClientStreamAsync(self, request, manual_write, loop) class HttpClientStreamAsync(HttpClientStream): @@ -205,17 +197,35 @@ class HttpClientStreamAsync(HttpClientStream): '_status_code', '_loop') - def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest) -> None: - self._init_common(connection, request) + def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, + loop: Optional[asyncio.AbstractEventLoop] = None) -> None: + """Initialize an HTTP client stream. + + Args: + connection (HttpClientConnectionAsync): The connection to send the request on. + request (HttpRequest): The HTTP request to send. + loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. + If None, the current event loop is used. + """ + self._init_common(connection, request, loop=loop) def _init_common(self, connection: HttpClientConnectionAsync, request: HttpRequest, - http2_manual_write: bool = False) -> None: + http2_manual_write: bool = False, + loop: Optional[asyncio.AbstractEventLoop] = None) -> None: # Initialize the parent class super()._init_common(connection, request, http2_manual_write=http2_manual_write) - # Set up async state tracking - self._loop = asyncio.get_event_loop() + # Attach the event loop for async operations + if loop is None: + # Use the current event loop if none is provided + loop = asyncio.get_event_loop() + elif not isinstance(loop, asyncio.AbstractEventLoop): + raise TypeError("loop must be an instance of asyncio.AbstractEventLoop") + self._loop = loop + + # deque is thread-safe for appending and popping, so that we don't need + # locks to handle the callbacks from the C thread self._chunk_futures = deque() self._received_chunks = deque() self._stream_completed = False @@ -240,6 +250,10 @@ def _set_response(self, status_code: int, name_value_pairs: List[Tuple[str, str] self._response_headers_future.set_result(name_value_pairs) def _on_body(self, chunk: bytes) -> None: + self._loop.call_soon_threadsafe(self._set_body_chunk, chunk) + + def _set_body_chunk(self, chunk: bytes) -> None: + """Process body chunk on the correct event loop thread.""" if self._chunk_futures: future = self._chunk_futures.popleft() future.set_result(chunk) @@ -257,6 +271,11 @@ def _set_completion(self, error_code: int) -> None: else: self._completion_future.set_exception(awscrt.exceptions.from_code(error_code)) + if self._chunk_futures: + # the stream is completed, so we need to set the futures + future = self._chunk_futures.popleft() + future.set_result("") + async def next(self) -> bytes: """Get the next chunk from the response body. @@ -271,7 +290,7 @@ async def next(self) -> bytes: else: future = Future[bytes]() self._chunk_futures.append(future) - return await asyncio.wrap_future(future) + return await asyncio.wrap_future(future, loop=self._loop) async def wait_for_completion(self) -> int: """Wait asynchronously for the stream to complete. @@ -304,8 +323,9 @@ class Http2ClientStreamAsync(HttpClientStreamAsync, Http2ClientStream): Create an Http2ClientStreamAsync with `Http2ClientConnectionAsync.request()`. """ - def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, manual_write: bool) -> None: - super()._init_common(connection, request, http2_manual_write=manual_write) + def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, manual_write: bool, + loop: Optional[asyncio.AbstractEventLoop] = None) -> None: + super()._init_common(connection, request, http2_manual_write=manual_write, loop=loop) async def write_data_async(self, data_stream: Union[InputStream, Any], diff --git a/test/test_http_asyncio.py b/test/test_http_asyncio.py index 1acf3a9c5..88cd03af0 100644 --- a/test/test_http_asyncio.py +++ b/test/test_http_asyncio.py @@ -7,15 +7,19 @@ import asyncio import unittest import threading +import subprocess +import concurrent.futures +from urllib.parse import urlparse from test import NativeResourceTest import ssl import os from io import BytesIO from http.server import HTTPServer, SimpleHTTPRequestHandler -from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions -from awscrt.http import HttpHeaders, HttpProxyOptions, HttpRequest, HttpVersion +from awscrt import io +from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsContextOptions, TlsCipherPref +from awscrt.http import HttpHeaders, HttpRequest, HttpVersion, Http2Setting, Http2SettingID from awscrt.http_asyncio import HttpClientConnectionAsync, Http2ClientConnectionAsync -import awscrt.exceptions +import threading class Response: @@ -32,7 +36,6 @@ async def collect_response(self, stream): self.status_code = await stream.response_status_code() headers_list = await stream.response_headers() self.headers = HttpHeaders(headers_list) - # Collect body chunks while True: chunk = await stream.next() @@ -99,15 +102,13 @@ async def _new_client_connection(self, secure, proxy_options=None): event_loop_group = EventLoopGroup() host_resolver = DefaultHostResolver(event_loop_group) bootstrap = ClientBootstrap(event_loop_group, host_resolver) - connection = await HttpClientConnectionAsync.new( + return await HttpClientConnectionAsync.new( host_name=self.hostname, port=self.port, bootstrap=bootstrap, tls_connection_options=tls_conn_opt, proxy_options=proxy_options) - return connection - async def _test_connect(self, secure): self._start_server(secure) try: @@ -187,6 +188,147 @@ async def _test_put(self, secure): finally: self._stop_server() + async def _test_shutdown_error(self, secure): + # Use HTTP/1.0 connection to force a connection close after request completes + self._start_server(secure, http_1_0=True) + try: + connection = await self._new_client_connection(secure) + + # Send request + request = HttpRequest('GET', '/') + stream = connection.request(request) + + # Collect response + response = Response() + await response.collect_response(stream) + + # With HTTP/1.0, the server should close the connection + # We'll wait a bit and verify the connection is closed + await asyncio.sleep(0.5) # Give time for the server to close connection + self.assertFalse(connection.is_open()) + + finally: + self._stop_server() + + async def _test_stream_lives_until_complete(self, secure): + # Ensure that stream and connection classes stay alive until work is complete + self._start_server(secure) + try: + connection = await self._new_client_connection(secure) + + request = HttpRequest('GET', '/test/test_http_asyncio.py') + stream = connection.request(request) + + # Store stream but delete all local references + response = Response() + + # Schedule task to collect response but don't await it yet + collect_task = asyncio.create_task(response.collect_response(stream)) + + # Delete references to stream and connection + del stream + del connection + + # Now await the collection task - stream should still complete successfully + status_code = await collect_task + self.assertEqual(200, status_code) + + finally: + self._stop_server() + + async def _test_request_lives_until_stream_complete(self, secure): + # Ensure HttpRequest and body InputStream stay alive until HttpClientStream completes + self._start_server(secure) + try: + connection = await self._new_client_connection(secure) + + request = HttpRequest( + method='PUT', + path='/test/test_request_refcounts.txt', + headers=HttpHeaders([('Host', self.hostname), ('Content-Length', '5')]), + body_stream=BytesIO(b'hello')) + + # Create stream but delete the request + stream = connection.request(request) + del request + + # Now collect the response - should still work since the stream keeps the request alive + response = Response() + status_code = await response.collect_response(stream) + self.assertEqual(200, status_code) + + await connection.close() + + finally: + self._stop_server() + + async def _new_h2_client_connection(self, url): + event_loop_group = EventLoopGroup() + host_resolver = DefaultHostResolver(event_loop_group) + bootstrap = ClientBootstrap(event_loop_group, host_resolver) + + port = url.port + if port is None: + port = 443 + + tls_ctx_options = TlsContextOptions() + tls_ctx_options.verify_peer = False # Allow localhost + tls_ctx = ClientTlsContext(tls_ctx_options) + tls_conn_opt = tls_ctx.new_connection_options() + tls_conn_opt.set_server_name(url.hostname) + tls_conn_opt.set_alpn_list(["h2"]) + + connection = await Http2ClientConnectionAsync.new( + host_name=url.hostname, + port=port, + bootstrap=bootstrap, + tls_connection_options=tls_conn_opt) + + return connection + + async def _test_h2_client(self): + url = urlparse("https://d1cz66xoahf9cl.cloudfront.net/http_test_doc.txt") + connection = await self._new_h2_client_connection(url) + + # Check we set an h2 connection + self.assertEqual(connection.version, HttpVersion.Http2) + + request = HttpRequest('GET', url.path) + request.headers.add('host', url.hostname) + stream = connection.request(request) + + response = Response() + status_code = await response.collect_response(stream) + + # Check result + self.assertEqual(200, status_code) + self.assertEqual(200, response.status_code) + self.assertEqual(14428801, len(response.body)) + + await connection.close() + + async def _test_h2_manual_write_exception(self): + url = urlparse("https://d1cz66xoahf9cl.cloudfront.net/http_test_doc.txt") + connection = await self._new_h2_client_connection(url) + + # Check we set an h2 connection + self.assertEqual(connection.version, HttpVersion.Http2) + + request = HttpRequest('GET', url.path) + request.headers.add('host', url.hostname) + stream = connection.request(request) + + exception = None + try: + # If the stream is not configured to allow manual writes, this should throw an exception + await stream.write_data_async(BytesIO(b'hello'), False) + except RuntimeError as e: + exception = e + + self.assertIsNotNone(exception) + + await connection.close() + def test_connect_http(self): asyncio.run(self._test_connect(secure=False)) @@ -205,6 +347,351 @@ def test_put_http(self): def test_put_https(self): asyncio.run(self._test_put(secure=True)) + def test_shutdown_error_http(self): + asyncio.run(self._test_shutdown_error(secure=False)) + + def test_shutdown_error_https(self): + asyncio.run(self._test_shutdown_error(secure=True)) + + def test_stream_lives_until_complete_http(self): + asyncio.run(self._test_stream_lives_until_complete(secure=False)) + + def test_stream_lives_until_complete_https(self): + asyncio.run(self._test_stream_lives_until_complete(secure=True)) + + def test_request_lives_until_stream_complete_http(self): + asyncio.run(self._test_request_lives_until_stream_complete(secure=False)) + + def test_request_lives_until_stream_complete_https(self): + asyncio.run(self._test_request_lives_until_stream_complete(secure=True)) + + def test_h2_client(self): + asyncio.run(self._test_h2_client()) + + def test_h2_manual_write_exception(self): + asyncio.run(self._test_h2_manual_write_exception()) + + @unittest.skipIf(not TlsCipherPref.PQ_DEFAULT.is_supported(), "Cipher pref not supported") + def test_connect_pq_default(self): + async def _test(): + await self._test_connect(secure=True) + asyncio.run(_test()) + + async def _test_cross_thread_http_client(self, secure): + """Test using an HTTP client from a different thread/event loop.""" + self._start_server(secure) + try: + # Create connection in the main thread + connection = await self._new_client_connection(secure) + self.assertTrue(connection.is_open()) + + # Function to run in a different thread with a different event loop + async def thread_func(conn): + # Create new event loop for this thread + test_asset_path = 'test/test_http_asyncio.py' + request = HttpRequest('GET', '/' + test_asset_path) + + # Use the connection but with the current thread's event loop + thread_loop = asyncio.get_event_loop() + stream = conn.request(request, loop=thread_loop) + + # Collect and process response + response = Response() + status_code = await response.collect_response(stream) + + # Verify results + assert status_code == 200 + + with open(test_asset_path, 'rb') as test_asset: + test_asset_bytes = test_asset.read() + assert test_asset_bytes == response.body + + return True + + # Run in executor to get a different thread + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit( + lambda: asyncio.run(thread_func(connection)) + ) + result = future.result() + self.assertTrue(result) + + await connection.close() + + finally: + self._stop_server() + + async def _test_cross_thread_http2_client(self): + """Test using an HTTP/2 client from a different thread/event loop.""" + url = urlparse("https://d1cz66xoahf9cl.cloudfront.net/http_test_doc.txt") + connection = await self._new_h2_client_connection(url) + + # Check we set an h2 connection + self.assertEqual(connection.version, HttpVersion.Http2) + + # Function to run in a different thread with a different event loop + async def thread_func(conn): + request = HttpRequest('GET', url.path) + request.headers.add('host', url.hostname) + + # Use the connection but with the current thread's event loop + thread_loop = asyncio.get_event_loop() + stream = conn.request(request, loop=thread_loop) + + response = Response() + status_code = await response.collect_response(stream) + # Check result + assert status_code == 200 + return len(response.body) + + # Run in executor to get a different thread + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit( + lambda: asyncio.run(thread_func(connection)) + ) + body_length = future.result() + self.assertEqual(14428801, body_length) + + await connection.close() + + def test_cross_thread_http_client(self): + asyncio.run(self._test_cross_thread_http_client(secure=False)) + + def test_cross_thread_https_client(self): + asyncio.run(self._test_cross_thread_http_client(secure=True)) + + def test_cross_thread_http2_client(self): + asyncio.run(self._test_cross_thread_http2_client()) + + +@unittest.skipUnless(os.environ.get('AWS_TEST_LOCALHOST'), 'set env var to run test: AWS_TEST_LOCALHOST') +class TestAsyncClientMockServer(NativeResourceTest): + timeout = 5 # seconds + p_server = None + mock_server_url = None + + def setUp(self): + super().setUp() + # Start the mock server from the aws-c-http + server_path = os.path.join( + os.path.dirname(__file__), + '..', + 'crt', + 'aws-c-http', + 'tests', + 'py_localhost', + 'server.py') + python_path = sys.executable + self.mock_server_url = urlparse("https://localhost:3443/upload_test") + self.p_server = subprocess.Popen([python_path, server_path]) + # Wait for server to be ready + self._wait_for_server_ready() + + def _wait_for_server_ready(self): + """Wait until server is accepting connections.""" + max_attempts = 20 + + for attempt in range(max_attempts): + try: + with socket.create_connection(("127.0.0.1", self.mock_server_url.port), timeout=1): + return # Server is ready + except (ConnectionRefusedError, socket.timeout): + time.sleep(0.5) + + # If we get here, server failed to start + stdout, stderr = self.p_server.communicate(timeout=0.5) + raise RuntimeError(f"Server failed to start after {max_attempts} attempts.\n" + f"STDOUT: {stdout.decode()}\nSTDERR: {stderr.decode()}") + + def tearDown(self): + self.p_server.terminate() + try: + self.p_server.wait(timeout=5) + except subprocess.TimeoutExpired: + self.p_server.kill() + super().tearDown() + + def _on_remote_settings_changed(self, settings): + # The mock server has the default settings with + # ENABLE_PUSH = 0 + # MAX_CONCURRENT_STREAMS = 100 + # MAX_HEADER_LIST_SIZE = 2**16 + # Check the settings here + self.assertEqual(len(settings), 3) + for i in settings: + if i.id == Http2SettingID.ENABLE_PUSH: + self.assertEqual(i.value, 0) + if i.id == Http2SettingID.MAX_CONCURRENT_STREAMS: + self.assertEqual(i.value, 100) + if i.id == Http2SettingID.MAX_HEADER_LIST_SIZE: + self.assertEqual(i.value, 2**16) + + async def _new_mock_connection(self, initial_settings=None): + event_loop_group = EventLoopGroup() + host_resolver = DefaultHostResolver(event_loop_group) + bootstrap = ClientBootstrap(event_loop_group, host_resolver) + + port = self.mock_server_url.port + # only test https + if port is None: + port = 443 + tls_ctx_options = TlsContextOptions() + tls_ctx_options.verify_peer = False # allow localhost + tls_ctx = ClientTlsContext(tls_ctx_options) + tls_conn_opt = tls_ctx.new_connection_options() + tls_conn_opt.set_server_name(self.mock_server_url.hostname) + tls_conn_opt.set_alpn_list(["h2"]) + + if initial_settings is None: + initial_settings = [Http2Setting(Http2SettingID.ENABLE_PUSH, 0)] + + connection = await Http2ClientConnectionAsync.new( + host_name=self.mock_server_url.hostname, + port=port, + bootstrap=bootstrap, + tls_connection_options=tls_conn_opt, + initial_settings=initial_settings, + on_remote_settings_changed=self._on_remote_settings_changed) + + return connection + + async def _test_h2_mock_server_manual_write(self): + connection = await self._new_mock_connection() + # check we set an h2 connection + self.assertEqual(connection.version, HttpVersion.Http2) + + request = HttpRequest('POST', self.mock_server_url.path) + request.headers.add('host', self.mock_server_url.hostname) + stream = connection.request(request, manual_write=True) + + # Write data in chunks + await stream.write_data_async(BytesIO(b'hello'), False) + await stream.write_data_async(BytesIO(b'he123123'), False) + await stream.write_data_async(None, False) + await stream.write_data_async(BytesIO(b'hello'), True) + + # Collect response + response = Response() + status_code = await response.collect_response(stream) + + # Check result + self.assertEqual(200, status_code) + self.assertEqual(200, response.status_code) + + await connection.close() + + class DelayStream: + def __init__(self, bad_read=False): + self._read = False + self.bad_read = bad_read + + def read(self, _len): + if self.bad_read: + # simulate a bad read that raises an exception + # this will cause the stream to fail + raise RuntimeError("bad read exception") + if self._read: + # return empty as EOS + return b'' + else: + self._read = True + return b'hello' + + async def _test_h2_mock_server_manual_write_read_exception(self): + connection = await self._new_mock_connection() + # check we set an h2 connection + self.assertEqual(connection.version, HttpVersion.Http2) + + request = HttpRequest('POST', self.mock_server_url.path) + request.headers.add('host', self.mock_server_url.hostname) + stream = connection.request(request, manual_write=True) + + # Try to write data with a bad stream that raises an exception + exception = None + data = self.DelayStream(bad_read=True) + try: + await stream.write_data_async(data, False) + except Exception as e: + exception = e + stream_completion_exception = None + try: + await stream.wait_for_completion() + except Exception as e: + stream_completion_exception = e + + self.assertIsNotNone(exception) + self.assertIsNotNone(stream_completion_exception) + # assert that the exception is the same as the one we got from write_data. + self.assertEqual(str(exception), str(stream_completion_exception)) + await connection.close() + + async def _test_h2_mock_server_manual_write_lifetime(self): + connection = await self._new_mock_connection() + # check we set an h2 connection + self.assertEqual(connection.version, HttpVersion.Http2) + + request = HttpRequest('POST', self.mock_server_url.path) + request.headers.add('host', self.mock_server_url.hostname) + stream = connection.request(request, manual_write=True) + + # Create data stream and immediately delete the reference after writing + data = self.DelayStream(bad_read=False) + await stream.write_data_async(data, False) + del data + + # Finish the request + await stream.write_data_async(None, True) + + # Collect response + response = Response() + status_code = await response.collect_response(stream) + + # Check result + self.assertEqual(200, status_code) + + await connection.close() + + async def _test_h2_mock_server_settings(self): + # Test with invalid settings - should throw an exception + exception = None + try: + # Invalid settings type + initial_settings = [100] + await self._new_mock_connection(initial_settings) + except Exception as e: + exception = e + self.assertIsNotNone(exception) + + # Test with valid settings + connection = await self._new_mock_connection() + self.assertEqual(connection.version, HttpVersion.Http2) + + request = HttpRequest('POST', self.mock_server_url.path) + request.headers.add('host', self.mock_server_url.hostname) + stream = connection.request(request, manual_write=True) + + await stream.write_data_async(BytesIO(b'hello'), True) + + response = Response() + status_code = await response.collect_response(stream) + + self.assertEqual(200, status_code) + self.assertEqual(200, response.status_code) + + await connection.close() + + def test_h2_mock_server_manual_write(self): + asyncio.run(self._test_h2_mock_server_manual_write()) + + def test_h2_mock_server_manual_write_read_exception(self): + asyncio.run(self._test_h2_mock_server_manual_write_read_exception()) + + def test_h2_mock_server_manual_write_lifetime(self): + asyncio.run(self._test_h2_mock_server_manual_write_lifetime()) + + def test_h2_mock_server_settings(self): + asyncio.run(self._test_h2_mock_server_settings()) + if __name__ == '__main__': unittest.main() From ad2520b5b46e4d7bfc561bff4b733b49564fd593 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 11 Jun 2025 15:22:28 -0700 Subject: [PATCH 09/40] rename and move the method to be more logically ordered --- awscrt/http_asyncio.py | 34 +++++++++++++++++----------------- test/test_http_asyncio.py | 6 +++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py index 5ab51cc66..e7e1b1c2a 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/http_asyncio.py @@ -276,7 +276,23 @@ def _set_completion(self, error_code: int) -> None: future = self._chunk_futures.popleft() future.set_result("") - async def next(self) -> bytes: + async def get_response_status_code(self) -> int: + """Get the response status code asynchronously. + + Returns: + int: The response status code. + """ + return await self._response_status_future + + async def get_response_headers(self) -> List[Tuple[str, str]]: + """Get the response headers asynchronously. + + Returns: + List[Tuple[str, str]]: The response headers as a list of (name, value) tuples. + """ + return await self._response_headers_future + + async def get_next_response_chunk(self) -> bytes: """Get the next chunk from the response body. Returns: @@ -300,22 +316,6 @@ async def wait_for_completion(self) -> int: """ return await self._completion_future - async def response_status_code(self) -> int: - """Get the response status code asynchronously. - - Returns: - int: The response status code. - """ - return await self._response_status_future - - async def response_headers(self) -> List[Tuple[str, str]]: - """Get the response headers asynchronously. - - Returns: - List[Tuple[str, str]]: The response headers as a list of (name, value) tuples. - """ - return await self._response_headers_future - class Http2ClientStreamAsync(HttpClientStreamAsync, Http2ClientStream): """HTTP/2 stream that sends a request and receives a response. diff --git a/test/test_http_asyncio.py b/test/test_http_asyncio.py index 88cd03af0..867f826aa 100644 --- a/test/test_http_asyncio.py +++ b/test/test_http_asyncio.py @@ -33,12 +33,12 @@ def __init__(self): async def collect_response(self, stream): """Collects complete response from a stream""" # Get status code and headers - self.status_code = await stream.response_status_code() - headers_list = await stream.response_headers() + self.status_code = await stream.get_response_status_code() + headers_list = await stream.get_response_headers() self.headers = HttpHeaders(headers_list) # Collect body chunks while True: - chunk = await stream.next() + chunk = await stream.get_next_response_chunk() if not chunk: break self.body.extend(chunk) From d9c90a5915b90603790334c009d400927ce9dbb7 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 11 Jun 2025 15:59:02 -0700 Subject: [PATCH 10/40] fix python38 --- awscrt/http_asyncio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py index e7e1b1c2a..5a2ab4a9f 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/http_asyncio.py @@ -304,7 +304,7 @@ async def get_next_response_chunk(self) -> bytes: elif self._completion_future.done(): return b"" else: - future = Future[bytes]() + future = Future() self._chunk_futures.append(future) return await asyncio.wrap_future(future, loop=self._loop) From 7e2dfeb98e524c35861b561f03991b43d1f068fa Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 11 Jun 2025 16:04:14 -0700 Subject: [PATCH 11/40] python 3.8 has been removed by github --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef879ace1..b949bf654 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,7 +250,8 @@ jobs: windows: - runs-on: windows-2022 # latest + # https://github.com/actions/runner-images/issues/12034 Python 3.8.10 is no longer available on windows-2022 + runs-on: windows-2022 strategy: matrix: arch: [x86, x64] @@ -266,7 +267,7 @@ jobs: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.8.10\\${{ matrix.arch }}\\python.exe" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.9.13\\${{ matrix.arch }}\\python.exe" macos: runs-on: macos-14 # latest From 584b2f9748c469b4cf73be776f8fc55c83fbeaad Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 11 Jun 2025 16:36:15 -0700 Subject: [PATCH 12/40] incase of the future cancelled, just return --- awscrt/http.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/awscrt/http.py b/awscrt/http.py index 2246dc203..62c563fa5 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -463,6 +463,9 @@ def write_data(self, body_stream: InputStream = InputStream.wrap(data_stream, allow_none=True) def on_write_complete(error_code: int) -> None: + if future.cancelled(): + # the future was cancelled, so we don't need to set the result or exception + return if error_code: future.set_exception(awscrt.exceptions.from_code(error_code)) else: From e2bbb98e66adea952e0c458a32d6644241c7ba31 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 12 Jun 2025 09:44:19 -0700 Subject: [PATCH 13/40] edit the comments --- awscrt/http_asyncio.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/awscrt/http_asyncio.py b/awscrt/http_asyncio.py index 5a2ab4a9f..1a5135aa7 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/http_asyncio.py @@ -3,16 +3,6 @@ This module provides asyncio wrappers around the awscrt.http module. All network operations in `awscrt.http_asyncio` are asynchronous and use Python's asyncio framework. - -Threading Notes: - - Each asyncio event loop is typically associated with a specific thread - - When using HTTP streams across threads, you must specify the appropriate event loop - when creating the stream to avoid "Future attached to a different loop" errors - - For example, if you create a connection in Thread A but want to use it in Thread B, - pass Thread B's event loop when creating the stream: - `stream = connection.request(request, loop=thread_b_event_loop)` - - All async operations on a stream (await stream.next(), etc.) must be performed in the - thread that owns the event loop used to create the stream """ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. @@ -186,6 +176,10 @@ class HttpClientStreamAsync(HttpClientStream): the response status code (int) when the request/response exchange completes. If the exchange fails to complete, the Future will contain an exception indicating why it failed. + + Notes: + All async method on a stream (await stream.next(), etc.) must be performed in the + thread that owns the event loop used to create the stream """ __slots__ = ( '_response_status_future', From 7d27da37be0cc0eddfe8c15ea437e3bbdb6fdec9 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 16 Jun 2025 11:32:59 -0700 Subject: [PATCH 14/40] show me the path --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b949bf654..2a6ee2964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -251,7 +251,7 @@ jobs: windows: # https://github.com/actions/runner-images/issues/12034 Python 3.8.10 is no longer available on windows-2022 - runs-on: windows-2022 + runs-on: windows-2025 strategy: matrix: arch: [x86, x64] @@ -266,6 +266,7 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | + where python python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.9.13\\${{ matrix.arch }}\\python.exe" From 16f67d6959e4cb1977dab3b807d5d0ba4f75bc93 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 17 Jun 2025 14:05:20 -0700 Subject: [PATCH 15/40] more python style ood --- awscrt/{ => aio}/http_asyncio.py | 120 +++++++++++------- awscrt/http.py | 202 +++++++++++++++++-------------- source/io.c | 2 + test/test_http_asyncio.py | 20 +-- 4 files changed, 198 insertions(+), 146 deletions(-) rename awscrt/{ => aio}/http_asyncio.py (85%) diff --git a/awscrt/http_asyncio.py b/awscrt/aio/http_asyncio.py similarity index 85% rename from awscrt/http_asyncio.py rename to awscrt/aio/http_asyncio.py index 1a5135aa7..d8d949b32 100644 --- a/awscrt/http_asyncio.py +++ b/awscrt/aio/http_asyncio.py @@ -9,11 +9,13 @@ # SPDX-License-Identifier: Apache-2.0. import asyncio +import io +import _awscrt from concurrent.futures import Future import awscrt.exceptions -from typing import List, Tuple, Optional, Union, Callable, Any +from typing import List, Tuple, Optional, Union, Callable, Any, AsyncIterator from awscrt.http import ( - HttpClientConnection, HttpRequest, HttpClientStream, HttpProxyOptions, + HttpClientConnectionBase, HttpRequest, HttClientStreamBase, HttpProxyOptions, Http2Setting, HttpVersion, Http2ClientStream ) from awscrt.io import ( @@ -22,13 +24,12 @@ from collections import deque -class HttpClientConnectionAsync(HttpClientConnection): +class HttpClientConnectionAsync(HttpClientConnectionBase): """ An async HTTP client connection. Use `HttpClientConnectionAsync.new()` to establish a new connection. """ - __slots__ = ('_host_name', '_port') @classmethod async def new(cls, @@ -62,7 +63,7 @@ async def new(cls, Returns: HttpClientConnectionAsync: A new HTTP client connection. """ - future = HttpClientConnection._generic_new( + future = cls._generic_new( host_name, port, bootstrap, @@ -81,8 +82,8 @@ async def close(self) -> None: Returns: None: When shutdown is complete. """ - close_future = super().close() - await asyncio.wrap_future(close_future) + _awscrt.http_connection_close(self._binding) + await asyncio.wrap_future(self.shutdown_future) def request(self, request: 'HttpRequest', @@ -100,7 +101,7 @@ def request(self, return HttpClientStreamAsync(self, request, loop) -class Http2ClientConnectionAsync(HttpClientConnectionAsync): +class Http2ClientConnectionAsync(HttpClientConnectionBase): """ An async HTTP/2 client connection. @@ -133,7 +134,7 @@ async def new(cls, * `settings` (List[Http2Setting]): List of settings that were changed. """ - future = HttpClientConnection._generic_new( + future = cls._generic_new( host_name, port, bootstrap, @@ -146,6 +147,18 @@ async def new(cls, asyncio_connection=True) return await asyncio.wrap_future(future) + async def close(self) -> None: + """Close the connection asynchronously. + + Shutdown is asynchronous. This call has no effect if the connection is already + closing. + + Returns: + None: When shutdown is complete. + """ + _awscrt.http_connection_close(self._binding) + await asyncio.wrap_future(self.shutdown_future) + def request(self, request: 'HttpRequest', manual_write: bool = False, @@ -164,23 +177,7 @@ def request(self, return Http2ClientStreamAsync(self, request, manual_write, loop) -class HttpClientStreamAsync(HttpClientStream): - """Async HTTP stream that sends a request and receives a response. - - Create an HttpClientStreamAsync with `HttpClientConnectionAsync.request()`. - - Attributes: - connection (HttpClientConnectionAsync): This stream's connection. - - completion_future (asyncio.Future): Future that will contain - the response status code (int) when the request/response exchange - completes. If the exchange fails to complete, the Future will - contain an exception indicating why it failed. - - Notes: - All async method on a stream (await stream.next(), etc.) must be performed in the - thread that owns the event loop used to create the stream - """ +class HttpClientStreamAsyncBase(HttClientStreamBase): __slots__ = ( '_response_status_future', '_response_headers_future', @@ -191,18 +188,6 @@ class HttpClientStreamAsync(HttpClientStream): '_status_code', '_loop') - def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, - loop: Optional[asyncio.AbstractEventLoop] = None) -> None: - """Initialize an HTTP client stream. - - Args: - connection (HttpClientConnectionAsync): The connection to send the request on. - request (HttpRequest): The HTTP request to send. - loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. - If None, the current event loop is used. - """ - self._init_common(connection, request, loop=loop) - def _init_common(self, connection: HttpClientConnectionAsync, request: HttpRequest, http2_manual_write: bool = False, @@ -231,7 +216,7 @@ def _init_common(self, connection: HttpClientConnectionAsync, self._status_code = None # Activate the stream immediately - self.activate() + _awscrt.http_client_stream_activate(self) def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: self._status_code = status_code @@ -311,7 +296,38 @@ async def wait_for_completion(self) -> int: return await self._completion_future -class Http2ClientStreamAsync(HttpClientStreamAsync, Http2ClientStream): +class HttpClientStreamAsync(HttpClientStreamAsyncBase): + """Async HTTP stream that sends a request and receives a response. + + Create an HttpClientStreamAsync with `HttpClientConnectionAsync.request()`. + + Attributes: + connection (HttpClientConnectionAsync): This stream's connection. + + completion_future (asyncio.Future): Future that will contain + the response status code (int) when the request/response exchange + completes. If the exchange fails to complete, the Future will + contain an exception indicating why it failed. + + Notes: + All async method on a stream (await stream.next(), etc.) must be performed in the + thread that owns the event loop used to create the stream + """ + + def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, + loop: Optional[asyncio.AbstractEventLoop] = None) -> None: + """Initialize an HTTP client stream. + + Args: + connection (HttpClientConnectionAsync): The connection to send the request on. + request (HttpRequest): The HTTP request to send. + loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. + If None, the current event loop is used. + """ + super()._init_common(connection, request, loop=loop) + + +class Http2ClientStreamAsync(HttpClientStreamAsyncBase): """HTTP/2 stream that sends a request and receives a response. Create an Http2ClientStreamAsync with `Http2ClientConnectionAsync.request()`. @@ -321,17 +337,31 @@ def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: super()._init_common(connection, request, http2_manual_write=manual_write, loop=loop) - async def write_data_async(self, - data_stream: Union[InputStream, Any], - end_stream: bool = False) -> None: + async def write_data(self, + data_stream: Union[InputStream, Any], + end_stream: bool = False) -> None: """Write data to the stream asynchronously. Args: - data_stream (Union[InputStream, Any]): Data to write. + data_stream (AsyncIterator[bytes]): Async iterator that yields bytes to write. + Can be None to write an empty body, which is useful to finalize a request + with end_stream=True. end_stream (bool): Whether this is the last data to write. Returns: None: When the write completes. """ - future = self.write_data(data_stream, end_stream) + future: Future = Future() + body_stream: InputStream = InputStream.wrap(data_stream, allow_none=True) + + def on_write_complete(error_code: int) -> None: + if future.cancelled(): + # the future was cancelled, so we don't need to set the result or exception + return + if error_code: + future.set_exception(awscrt.exceptions.from_code(error_code)) + else: + future.set_result(None) + + _awscrt.http2_client_stream_write_data(self, body_stream, end_stream, on_write_complete) await asyncio.wrap_future(future) diff --git a/awscrt/http.py b/awscrt/http.py index 62c563fa5..771a24d28 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -107,19 +107,6 @@ def version(self) -> HttpVersion: """HttpVersion: Protocol used by this connection""" return self._version - def close(self) -> "concurrent.futures.Future": - """Close the connection. - - Shutdown is asynchronous. This call has no effect if the connection is already - closing. - - Returns: - concurrent.futures.Future: This connection's :attr:`shutdown_future`, - which completes when shutdown has finished. - """ - _awscrt.http_connection_close(self._binding) - return self.shutdown_future - def is_open(self) -> bool: """ Returns: @@ -130,56 +117,9 @@ def is_open(self) -> bool: return _awscrt.http_connection_is_open(self._binding) -class HttpClientConnection(HttpConnectionBase): - """ - An HTTP client connection. - - Use :meth:`HttpClientConnection.new()` to establish a new connection. - """ +class HttpClientConnectionBase(HttpConnectionBase): __slots__ = ('_host_name', '_port') - @classmethod - def new(cls, - host_name: str, - port: int, - bootstrap: Optional[ClientBootstrap] = None, - socket_options: Optional[SocketOptions] = None, - tls_connection_options: Optional[TlsConnectionOptions] = None, - proxy_options: Optional['HttpProxyOptions'] = None) -> "concurrent.futures.Future": - """ - Asynchronously establish a new HttpClientConnection. - - Args: - host_name (str): Connect to host. - - port (int): Connect to port. - - bootstrap (Optional [ClientBootstrap]): Client bootstrap to use when initiating socket connection. - If None is provided, the default singleton is used. - - socket_options (Optional[SocketOptions]): Optional socket options. - If None is provided, then default options are used. - - tls_connection_options (Optional[TlsConnectionOptions]): Optional TLS - connection options. If None is provided, then the connection will - be attempted over plain-text. - - proxy_options (Optional[HttpProxyOptions]): Optional proxy options. - If None is provided then a proxy is not used. - - Returns: - concurrent.futures.Future: A Future which completes when connection succeeds or fails. - If successful, the Future will contain a new :class:`HttpClientConnection`. - Otherwise, it will contain an exception. - """ - return HttpClientConnection._generic_new( - host_name, - port, - bootstrap, - socket_options, - tls_connection_options, - proxy_options) - @staticmethod def _generic_new( host_name: str, @@ -247,6 +187,57 @@ def port(self) -> int: """Remote port""" return self._port + +class HttpClientConnection(HttpClientConnectionBase): + """ + An HTTP client connection. + + Use :meth:`HttpClientConnection.new()` to establish a new connection. + """ + __slots__ = ('_host_name', '_port') + + @classmethod + def new(cls, + host_name: str, + port: int, + bootstrap: Optional[ClientBootstrap] = None, + socket_options: Optional[SocketOptions] = None, + tls_connection_options: Optional[TlsConnectionOptions] = None, + proxy_options: Optional['HttpProxyOptions'] = None) -> "concurrent.futures.Future": + """ + Asynchronously establish a new HttpClientConnection. + + Args: + host_name (str): Connect to host. + + port (int): Connect to port. + + bootstrap (Optional [ClientBootstrap]): Client bootstrap to use when initiating socket connection. + If None is provided, the default singleton is used. + + socket_options (Optional[SocketOptions]): Optional socket options. + If None is provided, then default options are used. + + tls_connection_options (Optional[TlsConnectionOptions]): Optional TLS + connection options. If None is provided, then the connection will + be attempted over plain-text. + + proxy_options (Optional[HttpProxyOptions]): Optional proxy options. + If None is provided then a proxy is not used. + + Returns: + concurrent.futures.Future: A Future which completes when connection succeeds or fails. + If successful, the Future will contain a new :class:`HttpClientConnection`. + Otherwise, it will contain an exception. + """ + return cls._generic_new( + host_name, + port, + bootstrap, + socket_options, + tls_connection_options, + proxy_options) + def request(self, request: 'HttpRequest', on_response: Optional[Callable[..., None]] = None, @@ -294,8 +285,21 @@ def request(self, """ return HttpClientStream(self, request, on_response, on_body) + def close(self) -> "concurrent.futures.Future": + """Close the connection. -class Http2ClientConnection(HttpClientConnection): + Shutdown is asynchronous. This call has no effect if the connection is already + closing. + + Returns: + concurrent.futures.Future: This connection's :attr:`shutdown_future`, + which completes when shutdown has finished. + """ + _awscrt.http_connection_close(self._binding) + return self.shutdown_future + + +class Http2ClientConnection(HttpClientConnectionBase): @classmethod def new(cls, @@ -323,7 +327,7 @@ def new(cls, * `settings` (List[Http2Setting]): List of settings that were changed. """ - return HttpClientConnection._generic_new( + return cls._generic_new( host_name, port, bootstrap, @@ -341,6 +345,19 @@ def request(self, manual_write: bool = False) -> 'Http2ClientStream': return Http2ClientStream(self, request, on_response, on_body, manual_write) + def close(self) -> "concurrent.futures.Future": + """Close the connection. + + Shutdown is asynchronous. This call has no effect if the connection is already + closing. + + Returns: + concurrent.futures.Future: This connection's :attr:`shutdown_future`, + which completes when shutdown has finished. + """ + _awscrt.http_connection_close(self._binding) + return self.shutdown_future + class HttpStreamBase(NativeResource): """Base for HTTP stream classes""" @@ -365,38 +382,16 @@ def _on_body(self, chunk: bytes) -> None: self._on_body_cb(http_stream=self, chunk=chunk) -class HttpClientStream(HttpStreamBase): - """HTTP stream that sends a request and receives a response. - - Create an HttpClientStream with :meth:`HttpClientConnection.request()`. - - NOTE: The HTTP stream sends no data until :meth:`HttpClientStream.activate()` - is called. Call activate() when you're ready for callbacks and events to fire. - - Attributes: - connection (HttpClientConnection): This stream's connection. - - completion_future (concurrent.futures.Future): Future that will contain - the response status code (int) when the request/response exchange - completes. If the exchange fails to complete, the Future will - contain an exception indicating why it failed. - """ +class HttClientStreamBase(HttpStreamBase): __slots__ = ('_response_status_code', '_on_response_cb', '_on_body_cb', '_request', '_version') - def __init__(self, - connection: HttpClientConnection, - request: 'HttpRequest', - on_response: Optional[Callable[..., None]] = None, - on_body: Optional[Callable[..., None]] = None) -> None: - self._init_common(connection, request, on_response, on_body) - def _init_common(self, - connection: HttpClientConnection, + connection: HttpClientConnectionBase, request: 'HttpRequest', on_response: Optional[Callable[..., None]] = None, on_body: Optional[Callable[..., None]] = None, http2_manual_write: bool = False) -> None: - assert isinstance(connection, HttpClientConnection) + assert isinstance(connection, HttpClientConnectionBase) assert isinstance(request, HttpRequest) assert callable(on_response) or on_response is None assert callable(on_body) or on_body is None @@ -423,6 +418,31 @@ def response_status_code(self) -> Optional[int]: This is None until a response arrives.""" return self._response_status_code + +class HttpClientStream(HttClientStreamBase): + """HTTP stream that sends a request and receives a response. + + Create an HttpClientStream with :meth:`HttpClientConnection.request()`. + + NOTE: The HTTP stream sends no data until :meth:`HttpClientStream.activate()` + is called. Call activate() when you're ready for callbacks and events to fire. + + Attributes: + connection (HttpClientConnection): This stream's connection. + + completion_future (concurrent.futures.Future): Future that will contain + the response status code (int) when the request/response exchange + completes. If the exchange fails to complete, the Future will + contain an exception indicating why it failed. + """ + + def __init__(self, + connection: HttpClientConnection, + request: 'HttpRequest', + on_response: Optional[Callable[..., None]] = None, + on_body: Optional[Callable[..., None]] = None) -> None: + self._init_common(connection, request, on_response, on_body) + def activate(self) -> None: """Begin sending the request. @@ -447,14 +467,14 @@ def _on_complete(self, error_code: int) -> None: self._completion_future.set_exception(awscrt.exceptions.from_code(error_code)) -class Http2ClientStream(HttpClientStream): +class Http2ClientStream(HttClientStreamBase): def __init__(self, connection: HttpClientConnection, request: 'HttpRequest', on_response: Optional[Callable[..., None]] = None, on_body: Optional[Callable[..., None]] = None, manual_write: bool = False) -> None: - super()._init_common(connection, request, on_response, on_body, manual_write) + self._init_common(connection, request, on_response, on_body, manual_write) def write_data(self, data_stream: Union[InputStream, Any], @@ -838,7 +858,7 @@ def _on_connection_setup(self, binding: Any, error_code: int, http_version: Http return if self._asyncio_connection: # Import is done here to avoid circular import issues - from awscrt.http_asyncio import HttpClientConnectionAsync, Http2ClientConnectionAsync + from awscrt.aio.http_asyncio import HttpClientConnectionAsync, Http2ClientConnectionAsync if http_version == HttpVersion.Http2: connection = Http2ClientConnectionAsync() else: diff --git a/source/io.c b/source/io.c index de218299c..93e3746d1 100644 --- a/source/io.c +++ b/source/io.c @@ -303,7 +303,9 @@ static void s_client_bootstrap_on_shutdown_complete(void *user_data) { aws_mem_release(aws_py_get_allocator(), bootstrap); if (shutdown_complete) { + printf("Client bootstrap shutdown complete, invoking callback %p\n", shutdown_complete); PyObject *result = PyObject_CallFunction(shutdown_complete, "()"); + printf("Result is %p\n", result); if (result) { Py_DECREF(result); } else { diff --git a/test/test_http_asyncio.py b/test/test_http_asyncio.py index 867f826aa..2f95c01e7 100644 --- a/test/test_http_asyncio.py +++ b/test/test_http_asyncio.py @@ -18,7 +18,7 @@ from awscrt import io from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsContextOptions, TlsCipherPref from awscrt.http import HttpHeaders, HttpRequest, HttpVersion, Http2Setting, Http2SettingID -from awscrt.http_asyncio import HttpClientConnectionAsync, Http2ClientConnectionAsync +from awscrt.aio.http_asyncio import HttpClientConnectionAsync, Http2ClientConnectionAsync import threading @@ -321,7 +321,7 @@ async def _test_h2_manual_write_exception(self): exception = None try: # If the stream is not configured to allow manual writes, this should throw an exception - await stream.write_data_async(BytesIO(b'hello'), False) + await stream.write_data(BytesIO(b'hello'), False) except RuntimeError as e: exception = e @@ -565,10 +565,10 @@ async def _test_h2_mock_server_manual_write(self): stream = connection.request(request, manual_write=True) # Write data in chunks - await stream.write_data_async(BytesIO(b'hello'), False) - await stream.write_data_async(BytesIO(b'he123123'), False) - await stream.write_data_async(None, False) - await stream.write_data_async(BytesIO(b'hello'), True) + await stream.write_data(BytesIO(b'hello'), False) + await stream.write_data(BytesIO(b'he123123'), False) + await stream.write_data(None, False) + await stream.write_data(BytesIO(b'hello'), True) # Collect response response = Response() @@ -610,7 +610,7 @@ async def _test_h2_mock_server_manual_write_read_exception(self): exception = None data = self.DelayStream(bad_read=True) try: - await stream.write_data_async(data, False) + await stream.write_data(data, False) except Exception as e: exception = e stream_completion_exception = None @@ -636,11 +636,11 @@ async def _test_h2_mock_server_manual_write_lifetime(self): # Create data stream and immediately delete the reference after writing data = self.DelayStream(bad_read=False) - await stream.write_data_async(data, False) + await stream.write_data(data, False) del data # Finish the request - await stream.write_data_async(None, True) + await stream.write_data(None, True) # Collect response response = Response() @@ -670,7 +670,7 @@ async def _test_h2_mock_server_settings(self): request.headers.add('host', self.mock_server_url.hostname) stream = connection.request(request, manual_write=True) - await stream.write_data_async(BytesIO(b'hello'), True) + await stream.write_data(BytesIO(b'hello'), True) response = Response() status_code = await response.collect_response(stream) From 54f3e8e1f3c4f4e684aba3bab0800c1684b03f32 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 17 Jun 2025 14:08:13 -0700 Subject: [PATCH 16/40] Potential fix for code scanning alert no. 9: Use of insecure SSL/TLS version Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- test/test_http_asyncio.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_http_asyncio.py b/test/test_http_asyncio.py index 2f95c01e7..00d996e6b 100644 --- a/test/test_http_asyncio.py +++ b/test/test_http_asyncio.py @@ -74,6 +74,7 @@ def _start_server(self, secure, http_1_0=False): self.server = HTTPServer((self.hostname, 0), TestRequestHandler) if secure: context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 context.load_cert_chain(certfile='test/resources/unittest.crt', keyfile="test/resources/unittest.key") self.server.socket = context.wrap_socket(self.server.socket, server_side=True) self.port = self.server.server_address[1] From 3f48ddda22dd8f01b6b8385c75bfddde36355938 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 17 Jun 2025 14:11:31 -0700 Subject: [PATCH 17/40] couple more fix --- awscrt/http.py | 38 +++++++++++++++++++++++--------------- source/io.c | 2 -- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/awscrt/http.py b/awscrt/http.py index 771a24d28..244f8ea61 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -418,6 +418,21 @@ def response_status_code(self) -> Optional[int]: This is None until a response arrives.""" return self._response_status_code + def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: + self._response_status_code = status_code + + if self._on_response_cb: + self._on_response_cb(http_stream=self, status_code=status_code, headers=name_value_pairs) + + def _on_complete(self, error_code: int) -> None: + # done with HttpRequest, drop reference + self._request = None # type: ignore + + if error_code == 0: + self._completion_future.set_result(self._response_status_code) + else: + self._completion_future.set_exception(awscrt.exceptions.from_code(error_code)) + class HttpClientStream(HttClientStreamBase): """HTTP stream that sends a request and receives a response. @@ -451,21 +466,6 @@ def activate(self) -> None: """ _awscrt.http_client_stream_activate(self) - def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: - self._response_status_code = status_code - - if self._on_response_cb: - self._on_response_cb(http_stream=self, status_code=status_code, headers=name_value_pairs) - - def _on_complete(self, error_code: int) -> None: - # done with HttpRequest, drop reference - self._request = None # type: ignore - - if error_code == 0: - self._completion_future.set_result(self._response_status_code) - else: - self._completion_future.set_exception(awscrt.exceptions.from_code(error_code)) - class Http2ClientStream(HttClientStreamBase): def __init__(self, @@ -476,6 +476,14 @@ def __init__(self, manual_write: bool = False) -> None: self._init_common(connection, request, on_response, on_body, manual_write) + def activate(self) -> None: + """Begin sending the request. + + The HTTP stream does nothing until this is called. Call activate() when you + are ready for its callbacks and events to fire. + """ + _awscrt.http_client_stream_activate(self) + def write_data(self, data_stream: Union[InputStream, Any], end_stream: bool = False) -> "concurrent.futures.Future": diff --git a/source/io.c b/source/io.c index 93e3746d1..de218299c 100644 --- a/source/io.c +++ b/source/io.c @@ -303,9 +303,7 @@ static void s_client_bootstrap_on_shutdown_complete(void *user_data) { aws_mem_release(aws_py_get_allocator(), bootstrap); if (shutdown_complete) { - printf("Client bootstrap shutdown complete, invoking callback %p\n", shutdown_complete); PyObject *result = PyObject_CallFunction(shutdown_complete, "()"); - printf("Result is %p\n", result); if (result) { Py_DECREF(result); } else { From 75848947eb92ef160fee5314fabe444a6299d9d4 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 17 Jun 2025 14:55:15 -0700 Subject: [PATCH 18/40] remove unneeded --- .github/workflows/ci.yml | 1 - awscrt/aio/http_asyncio.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae8c91c52..cb7cc25e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,7 +272,6 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | - where python python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "${{ steps.python38.outputs.python-path }}" diff --git a/awscrt/aio/http_asyncio.py b/awscrt/aio/http_asyncio.py index d8d949b32..4f8edc038 100644 --- a/awscrt/aio/http_asyncio.py +++ b/awscrt/aio/http_asyncio.py @@ -16,7 +16,7 @@ from typing import List, Tuple, Optional, Union, Callable, Any, AsyncIterator from awscrt.http import ( HttpClientConnectionBase, HttpRequest, HttClientStreamBase, HttpProxyOptions, - Http2Setting, HttpVersion, Http2ClientStream + Http2Setting, HttpVersion ) from awscrt.io import ( ClientBootstrap, SocketOptions, TlsConnectionOptions, InputStream From 955d1765df21ab4ca6b35bac2df3e9465f7c3ef4 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 17 Jun 2025 16:29:22 -0700 Subject: [PATCH 19/40] I'll fix test later --- awscrt/aio/http_asyncio.py | 56 ++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/awscrt/aio/http_asyncio.py b/awscrt/aio/http_asyncio.py index 4f8edc038..a87a722c9 100644 --- a/awscrt/aio/http_asyncio.py +++ b/awscrt/aio/http_asyncio.py @@ -161,7 +161,7 @@ async def close(self) -> None: def request(self, request: 'HttpRequest', - manual_write: bool = False, + async_body: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> 'Http2ClientStreamAsync': """Create `Http2ClientStreamAsync` to carry out the request/response exchange. @@ -174,7 +174,7 @@ def request(self, Returns: Http2ClientStreamAsync: Stream for the HTTP/2 request/response exchange. """ - return Http2ClientStreamAsync(self, request, manual_write, loop) + return Http2ClientStreamAsync(self, request, async_body, loop) class HttpClientStreamAsyncBase(HttClientStreamBase): @@ -188,11 +188,12 @@ class HttpClientStreamAsyncBase(HttClientStreamBase): '_status_code', '_loop') - def _init_common(self, connection: HttpClientConnectionAsync, + def _init_common(self, connection: HttpClientConnectionBase, request: HttpRequest, - http2_manual_write: bool = False, + async_body: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: # Initialize the parent class + http2_manual_write = async_body is not None and connection.version is HttpVersion.Http2 super()._init_common(connection, request, http2_manual_write=http2_manual_write) # Attach the event loop for async operations @@ -215,6 +216,10 @@ def _init_common(self, connection: HttpClientConnectionAsync, self._response_headers_future = self._loop.create_future() self._status_code = None + self._async_body = async_body + if self._async_body is not None: + self._writer = asyncio.Task(self._set_async_body(self._async_body)) + # Activate the stream immediately _awscrt.http_client_stream_activate(self) @@ -333,26 +338,16 @@ class Http2ClientStreamAsync(HttpClientStreamAsyncBase): Create an Http2ClientStreamAsync with `Http2ClientConnectionAsync.request()`. """ - def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, manual_write: bool, + def __init__(self, + connection: HttpClientConnectionAsync, + request: HttpRequest, + async_body: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: - super()._init_common(connection, request, http2_manual_write=manual_write, loop=loop) - - async def write_data(self, - data_stream: Union[InputStream, Any], - end_stream: bool = False) -> None: - """Write data to the stream asynchronously. + super()._init_common(connection, request, async_body=async_body, loop=loop) - Args: - data_stream (AsyncIterator[bytes]): Async iterator that yields bytes to write. - Can be None to write an empty body, which is useful to finalize a request - with end_stream=True. - end_stream (bool): Whether this is the last data to write. - - Returns: - None: When the write completes. - """ + async def _write_data(self, body, end_stream): future: Future = Future() - body_stream: InputStream = InputStream.wrap(data_stream, allow_none=True) + body_stream: InputStream = InputStream.wrap(body, allow_none=True) def on_write_complete(error_code: int) -> None: if future.cancelled(): @@ -365,3 +360,22 @@ def on_write_complete(error_code: int) -> None: _awscrt.http2_client_stream_write_data(self, body_stream, end_stream, on_write_complete) await asyncio.wrap_future(future) + + async def _set_async_body(self, body_iterator: AsyncIterator[bytes]): + """Write data to the stream asynchronously. + + Args: + data_stream (AsyncIterator[bytes]): Async iterator that yields bytes to write. + Can be None to write an empty body, which is useful to finalize a request + with end_stream=True. + + Returns: + None: When the write completes. + """ + try: + async for chunk in body_iterator: + await self._write_data(io.BytesIO(chunk), False) + except Exception: + raise + finally: + await self._write_data(None, True) From c95841bd2dfdda231d3e1076cc6e57016a018c55 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 17 Jun 2025 16:39:48 -0700 Subject: [PATCH 20/40] use the loop --- awscrt/aio/http_asyncio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awscrt/aio/http_asyncio.py b/awscrt/aio/http_asyncio.py index a87a722c9..882335e46 100644 --- a/awscrt/aio/http_asyncio.py +++ b/awscrt/aio/http_asyncio.py @@ -218,7 +218,7 @@ def _init_common(self, connection: HttpClientConnectionBase, self._async_body = async_body if self._async_body is not None: - self._writer = asyncio.Task(self._set_async_body(self._async_body)) + self._writer = self._loop.create_task(self._set_async_body(self._async_body)) # Activate the stream immediately _awscrt.http_client_stream_activate(self) From 4886a3bee0ef1497ba6291b593365026fb57f1db Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 17 Jun 2025 16:45:36 -0700 Subject: [PATCH 21/40] demo for testing --- examples/http2_asyncio_demo.py | 291 +++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 examples/http2_asyncio_demo.py diff --git a/examples/http2_asyncio_demo.py b/examples/http2_asyncio_demo.py new file mode 100644 index 000000000..7880ccda3 --- /dev/null +++ b/examples/http2_asyncio_demo.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +""" +This example demonstrates how to use the asyncio HTTP/2 client in awscrt. +It performs multiple concurrent requests to httpbin.org and shows HTTP/2 features. +""" + +import asyncio +import sys +import io +from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions +from awscrt.http import HttpHeaders, HttpRequest, Http2Setting, Http2SettingID +from awscrt.aio.http_asyncio import Http2ClientConnectionAsync +# from awscrt_python_logging_example import PythonLoggingRedirector +import awscrt.io +import logging +from typing import List, Tuple, Optional, Union, Callable, Any, AsyncIterator + + +class Response: + """Holds contents of incoming response""" + + def __init__(self, request_name): + self.request_name = request_name + self.status_code = None + self.headers = None + self.body = bytearray() + + def on_response(self, http_stream, status_code, headers, **kwargs): + print(f"[{self.request_name}] Received response status: {status_code}") + self.status_code = status_code + self.headers = HttpHeaders(headers) + for name, value in headers: + print(f"[{self.request_name}] Header: {name}: {value}") + + def on_body(self, http_stream, chunk, **kwargs): + print(f"[{self.request_name}] Received body chunk of size: {len(chunk)} bytes") + self.body.extend(chunk) + + +# Create an event for synchronizing remote settings +remote_settings_event = None +event_loop = None + + +def on_remote_settings_changed(settings): + """Handler for when the server updates HTTP/2 settings""" + print("Remote HTTP/2 settings changed:") + for setting in settings: + print(f" - {setting.id.name} = {setting.value}") + # Signal that remote settings have been received + # This callback is called from a different thread, so we need to use call_soon_threadsafe + if event_loop and remote_settings_event: + event_loop.call_soon_threadsafe(remote_settings_event.set) + + +async def make_concurrent_requests(): + """Perform multiple concurrent HTTP/2 requests asynchronously.""" + global remote_settings_event, event_loop + + # Get the current event loop and create the event + event_loop = asyncio.get_running_loop() + remote_settings_event = asyncio.Event() + + # Create an event loop group and default host resolver + event_loop_group = EventLoopGroup() + host_resolver = DefaultHostResolver(event_loop_group) + bootstrap = ClientBootstrap(event_loop_group, host_resolver) + + # Connect to httpbin.org + # host_name = "postman-echo.com" # Change to "httpbin.org" for real requests + # port = 443 + host_name = "localhost" # Change to "httpbin.org" for real requests + port = 3443 + + # TLS options for HTTP/2 + tls_ctx_opt = TlsContextOptions() + tls_ctx_opt.verify_peer = False + tls_ctx = ClientTlsContext(tls_ctx_opt) + tls_conn_opt = tls_ctx.new_connection_options() + tls_conn_opt.set_server_name(host_name) + tls_conn_opt.set_alpn_list(["h2"]) # Set ALPN to HTTP/2 + + # Initial HTTP/2 settings + initial_settings = [ + Http2Setting(Http2SettingID.ENABLE_PUSH, 0), + Http2Setting(Http2SettingID.MAX_CONCURRENT_STREAMS, 100), + Http2Setting(Http2SettingID.INITIAL_WINDOW_SIZE, 65535), + ] + + print(f"Connecting to {host_name}:{port} using HTTP/2...") + connection = await Http2ClientConnectionAsync.new( + host_name=host_name, + port=port, + bootstrap=bootstrap, + tls_connection_options=tls_conn_opt, + initial_settings=initial_settings, + on_remote_settings_changed=on_remote_settings_changed + ) + print("HTTP/2 Connection established!") + + # Wait for remote settings to be received + print("Waiting for remote settings...") + await remote_settings_event.wait() + print("Remote settings received, proceeding with requests...") + + try: + # Create several requests to be executed concurrently + tasks = [] + + # Request 1: Simple GET + # tasks.append(send_get_request(connection, host_name)) + + # Request 2: POST with JSON body + # tasks.append(send_post_request(connection, host_name)) + + # Request 3: Stream data using manual write mode + tasks.append(send_stream_request(connection, host_name)) + + # Wait for all requests to complete + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Check for any exceptions + for i, result in enumerate(results): + if isinstance(result, Exception): + print(f"Task {i} failed with exception: {result}") + + finally: + # Add a small delay to ensure all responses are received + await asyncio.sleep(1) + + # Close the connection + print("Closing connection...") + await connection.close() + print("Connection closed!") + + +async def send_get_request(connection, host_name): + """Send a GET request using the HTTP/2 connection.""" + print("Sending GET request...") + request = HttpRequest("GET", "/delete") + request.headers.add("host", host_name) + + # Set up response handler + response = Response("GET") + stream = connection.request(request) + headers = await stream.get_response_headers() + print(headers) + + # Wait for completion + status_code = await stream.wait_for_completion() + print(f"GET request completed with status code: {status_code}") + print("\nGET Response body:") + print(response.body.decode("utf-8")) + return status_code + + +async def send_post_request(connection, host_name): + """Send a POST request with JSON body using the HTTP/2 connection.""" + print("Sending POST request with JSON body...") + + # Prepare JSON payload + json_payload = '{"name": "Example User", "id": 12345}' + + # Create request with headers + request = HttpRequest("POST", "/post") + request.headers.add("host", host_name) + request.headers.add("content-type", "application/json") + request.headers.add("content-length", str(len(json_payload))) + + # Set the body using BytesIO stream + request.body_stream = io.BytesIO(json_payload.encode("utf-8")) + + # Set up response handler + response = Response("POST") + stream = connection.request(request) + + # Wait for completion + status_code = await stream.wait_for_completion() + print(f"POST request completed with status code: {status_code}") + print("\nPOST Response body:") + print(response.body.decode("utf-8")) + return status_code + + +async def data_generator() -> AsyncIterator[bytes]: + for i in range(5): + yield f"chunk {i}".encode() + await asyncio.sleep(0.1) # Simulate delay between chunks + + +async def send_stream_request(connection, host_name): + """Send a request with streamed data using manual write mode.""" + # print("Sending request with manual data streaming...") + + # # Create request + # request = HttpRequest("PUT", "/put") + # request.headers.add("host", host_name) + # request.headers.add("content-type", "text/plain") + # # Note: We don't set content-length as we're streaming the data + + # # Set up response handler + # stream = connection.request(request, manual_write=True) + + # # Stream data in chunks using BytesIO (legacy method) + # print("\nMethod 1: Using BytesIO (legacy method)") + # data_chunks = [ + # b"This is the first chunk of data.\n", + # b"This is the second chunk of data.\n" + # ] + + # await stream.write_data(io.BytesIO(data_chunks[0]), end_stream=False) + # await stream.write_data(io.BytesIO(data_chunks[1]), end_stream=True) + + # # Wait for completion + # status_code = await stream.wait_for_completion() + # print(f"Stream request completed with status code: {status_code}") + + # Create a new stream for the AsyncIterator demo + print("\nMethod 2: Using AsyncIterator[bytes] (new method)") + request = HttpRequest("PUT", "/put") + request.headers.add("host", host_name) + request.headers.add("content-type", "text/plain") + + # Create an async generator function that yields chunks + + async def async_data_generator(): + chunks = [ + b"This is the first async chunk.\n", + b"This is the second async chunk.\n", + b"This is the final async chunk." + ] + for chunk in chunks: + print(f"Yielding chunk of size: {len(chunk)} bytes") + yield chunk + # Simulate some async processing between chunks + await asyncio.sleep(2) + + stream = connection.request(request, async_body=async_data_generator()) + + # Process the response + print("hey") + status_code = await stream.get_response_status_code() + print(f"Async iterator stream request completed with status code: {status_code}") + headers = await stream.get_response_headers() + print("\nStream Response headers:") + for name, value in headers: + print(f"{name}: {value}") + + # Get the response body + # body = bytearray() + # while True: + # chunk = await stream.get_next_response_chunk() + # if not chunk: + # break + # body.extend(chunk) + + # print("\nStream Response body:") + # print(body.decode("utf-8")) + await stream.wait_for_completion() + + # Return the status code from the async iterator example + return status_code + + +def main(): + """Entry point for the example.""" + try: + + # Set up Python logging + logging.basicConfig(level=logging.DEBUG) + # awscrt.io.init_logging(awscrt.io.LogLevel.Trace, "stdout") + # Create and activate redirector + # redirector = PythonLoggingRedirector(base_logger_name="myapp.awscrt") + # redirector.activate(aws_log_level=awscrt.io.LogLevel.Trace) + + asyncio.run(make_concurrent_requests()) + # Your AWS CRT operations here... + # Logs will now appear in Python's logging system + + # redirector.deactivate() + return 0 + except Exception as e: + print(f"Exception: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) From 76f0e33a6d07b7cfd2596c8992114e8e4422884c Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 Jun 2025 09:05:16 -0700 Subject: [PATCH 22/40] unified interface as the base --- awscrt/aio/http_asyncio.py | 127 ++++++++++++++++++++++++++----------- 1 file changed, 89 insertions(+), 38 deletions(-) diff --git a/awscrt/aio/http_asyncio.py b/awscrt/aio/http_asyncio.py index 882335e46..fd2ea9070 100644 --- a/awscrt/aio/http_asyncio.py +++ b/awscrt/aio/http_asyncio.py @@ -24,7 +24,7 @@ from collections import deque -class HttpClientConnectionAsync(HttpClientConnectionBase): +class HttpClientConnectionAsyncUnified(HttpClientConnectionBase): """ An async HTTP client connection. @@ -38,9 +38,9 @@ async def new(cls, bootstrap: Optional[ClientBootstrap] = None, socket_options: Optional[SocketOptions] = None, tls_connection_options: Optional[TlsConnectionOptions] = None, - proxy_options: Optional['HttpProxyOptions'] = None) -> "HttpClientConnectionAsync": + proxy_options: Optional['HttpProxyOptions'] = None) -> "HttpClientConnectionAsyncUnified": """ - Asynchronously establish a new HttpClientConnectionAsync. + Asynchronously establish a new HttpClientConnectionAsyncUnified. Args: host_name (str): Connect to host. @@ -87,6 +87,75 @@ async def close(self) -> None: def request(self, request: 'HttpRequest', + async_body: AsyncIterator[bytes] = None, + loop: Optional[asyncio.AbstractEventLoop] = None) -> 'HttpClientStreamAsyncUnified': + """Create `HttpClientStreamAsync` to carry out the request/response exchange. + + Args: + request (HttpRequest): Definition for outgoing request. + loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. + If None, the current event loop is used. + + Returns: + HttpClientStreamAsync: Stream for the HTTP request/response exchange. + """ + # return HttpClientStreamAsyncBase(self, request, loop) + pass + + +class HttpClientConnectionAsync(HttpClientConnectionAsyncUnified): + """ + An async HTTP/1.1 client connection. + + Use `HttpClientConnectionAsync.new()` to establish a new connection. + """ + + @classmethod + async def new(cls, + host_name: str, + port: int, + bootstrap: Optional[ClientBootstrap] = None, + socket_options: Optional[SocketOptions] = None, + tls_connection_options: Optional[TlsConnectionOptions] = None, + proxy_options: Optional['HttpProxyOptions'] = None) -> "HttpClientConnectionAsync": + """ + Asynchronously establish a new HttpClientConnectionAsync. + + Args: + host_name (str): Connect to host. + + port (int): Connect to port. + + bootstrap (Optional [ClientBootstrap]): Client bootstrap to use when initiating socket connection. + If None is provided, the default singleton is used. + + socket_options (Optional[SocketOptions]): Optional socket options. + If None is provided, then default options are used. + + tls_connection_options (Optional[TlsConnectionOptions]): Optional TLS + connection options. If None is provided, then the connection will + be attempted over plain-text. + + proxy_options (Optional[HttpProxyOptions]): Optional proxy options. + If None is provided then a proxy is not used. + + Returns: + HttpClientConnectionAsync: A new HTTP client connection. + """ + future = cls._generic_new( + host_name, + port, + bootstrap, + socket_options, + tls_connection_options, + proxy_options, + expected_version=HttpVersion.Http1_1, + asyncio_connection=True) + return await asyncio.wrap_future(future) + + def request(self, + request: 'HttpRequest', + async_body: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> 'HttpClientStreamAsync': """Create `HttpClientStreamAsync` to carry out the request/response exchange. @@ -101,7 +170,7 @@ def request(self, return HttpClientStreamAsync(self, request, loop) -class Http2ClientConnectionAsync(HttpClientConnectionBase): +class Http2ClientConnectionAsync(HttpClientConnectionAsyncUnified): """ An async HTTP/2 client connection. @@ -141,24 +210,12 @@ async def new(cls, socket_options, tls_connection_options, proxy_options, - HttpVersion.Http2, - initial_settings, - on_remote_settings_changed, + expected_version=HttpVersion.Http2, + initial_settings=initial_settings, + on_remote_settings_changed=on_remote_settings_changed, asyncio_connection=True) return await asyncio.wrap_future(future) - async def close(self) -> None: - """Close the connection asynchronously. - - Shutdown is asynchronous. This call has no effect if the connection is already - closing. - - Returns: - None: When shutdown is complete. - """ - _awscrt.http_connection_close(self._binding) - await asyncio.wrap_future(self.shutdown_future) - def request(self, request: 'HttpRequest', async_body: AsyncIterator[bytes] = None, @@ -177,7 +234,7 @@ def request(self, return Http2ClientStreamAsync(self, request, async_body, loop) -class HttpClientStreamAsyncBase(HttClientStreamBase): +class HttpClientStreamAsyncUnified(HttClientStreamBase): __slots__ = ( '_response_status_future', '_response_headers_future', @@ -188,10 +245,11 @@ class HttpClientStreamAsyncBase(HttClientStreamBase): '_status_code', '_loop') - def _init_common(self, connection: HttpClientConnectionBase, - request: HttpRequest, - async_body: AsyncIterator[bytes] = None, - loop: Optional[asyncio.AbstractEventLoop] = None) -> None: + def __init__(self, + connection: HttpClientConnectionAsync, + request: HttpRequest, + async_body: AsyncIterator[bytes] = None, + loop: Optional[asyncio.AbstractEventLoop] = None) -> None: # Initialize the parent class http2_manual_write = async_body is not None and connection.version is HttpVersion.Http2 super()._init_common(connection, request, http2_manual_write=http2_manual_write) @@ -300,8 +358,11 @@ async def wait_for_completion(self) -> int: """ return await self._completion_future + async def _set_async_body(self, body_iterator: AsyncIterator[bytes]): + ... + -class HttpClientStreamAsync(HttpClientStreamAsyncBase): +class HttpClientStreamAsync(HttpClientStreamAsyncUnified): """Async HTTP stream that sends a request and receives a response. Create an HttpClientStreamAsync with `HttpClientConnectionAsync.request()`. @@ -329,10 +390,10 @@ def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. If None, the current event loop is used. """ - super()._init_common(connection, request, loop=loop) + super().__init__(connection, request, loop=loop) -class Http2ClientStreamAsync(HttpClientStreamAsyncBase): +class Http2ClientStreamAsync(HttpClientStreamAsyncUnified): """HTTP/2 stream that sends a request and receives a response. Create an Http2ClientStreamAsync with `Http2ClientConnectionAsync.request()`. @@ -343,7 +404,7 @@ def __init__(self, request: HttpRequest, async_body: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: - super()._init_common(connection, request, async_body=async_body, loop=loop) + super().__init__(connection, request, async_body=async_body, loop=loop) async def _write_data(self, body, end_stream): future: Future = Future() @@ -362,16 +423,6 @@ def on_write_complete(error_code: int) -> None: await asyncio.wrap_future(future) async def _set_async_body(self, body_iterator: AsyncIterator[bytes]): - """Write data to the stream asynchronously. - - Args: - data_stream (AsyncIterator[bytes]): Async iterator that yields bytes to write. - Can be None to write an empty body, which is useful to finalize a request - with end_stream=True. - - Returns: - None: When the write completes. - """ try: async for chunk in body_iterator: await self._write_data(io.BytesIO(chunk), False) From 0fe5d76ae209ab070813e6a9a920c0033a267ab5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 Jun 2025 09:30:55 -0700 Subject: [PATCH 23/40] fix tests and doc --- awscrt/aio/http_asyncio.py | 20 +++++--- test/test_http_asyncio.py | 100 ++++++++++--------------------------- 2 files changed, 39 insertions(+), 81 deletions(-) diff --git a/awscrt/aio/http_asyncio.py b/awscrt/aio/http_asyncio.py index fd2ea9070..b0c311daa 100644 --- a/awscrt/aio/http_asyncio.py +++ b/awscrt/aio/http_asyncio.py @@ -26,7 +26,7 @@ class HttpClientConnectionAsyncUnified(HttpClientConnectionBase): """ - An async HTTP client connection. + An async unified HTTP client connection for either a HTTP/1 or HTTP/2 connection. Use `HttpClientConnectionAsync.new()` to establish a new connection. """ @@ -89,23 +89,24 @@ def request(self, request: 'HttpRequest', async_body: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> 'HttpClientStreamAsyncUnified': - """Create `HttpClientStreamAsync` to carry out the request/response exchange. + """Create `HttpClientStreamAsyncUnified` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. + async_body (AsyncIterator[bytes], optional): Async iterator providing chunks of the request body. + If provided, the body will be sent incrementally as chunks become available. loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. If None, the current event loop is used. Returns: - HttpClientStreamAsync: Stream for the HTTP request/response exchange. + HttpClientStreamAsyncUnified: Stream for the HTTP request/response exchange. """ - # return HttpClientStreamAsyncBase(self, request, loop) - pass + raise NotImplementedError("Subclasses must implement request") class HttpClientConnectionAsync(HttpClientConnectionAsyncUnified): """ - An async HTTP/1.1 client connection. + An async HTTP/1.1 only client connection. Use `HttpClientConnectionAsync.new()` to establish a new connection. """ @@ -161,6 +162,8 @@ def request(self, Args: request (HttpRequest): Definition for outgoing request. + async_body (AsyncIterator[bytes], optional): Async iterator providing chunks of the request body. + Not supported for HTTP/1.1 connections yet, use the request's body_stream instead. loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. If None, the current event loop is used. @@ -172,7 +175,7 @@ def request(self, class Http2ClientConnectionAsync(HttpClientConnectionAsyncUnified): """ - An async HTTP/2 client connection. + An async HTTP/2 only client connection. Use `Http2ClientConnectionAsync.new()` to establish a new connection. """ @@ -224,7 +227,8 @@ def request(self, Args: request (HttpRequest): Definition for outgoing request. - manual_write (bool): If True, enables manual data writing on the stream. + async_body (AsyncIterator[bytes], optional): Async iterator providing chunks of the request body. + If provided, the body will be sent incrementally as chunks become available from the iterator. loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. If None, the current event loop is used. diff --git a/test/test_http_asyncio.py b/test/test_http_asyncio.py index 00d996e6b..75ea7b3d8 100644 --- a/test/test_http_asyncio.py +++ b/test/test_http_asyncio.py @@ -317,17 +317,22 @@ async def _test_h2_manual_write_exception(self): request = HttpRequest('GET', url.path) request.headers.add('host', url.hostname) + + # Create stream without using async_body parameter + # (which would be needed to properly configure it for writing) stream = connection.request(request) + # The stream should have write_data attribute but using it should raise an exception + # since the stream isn't properly configured for manual writing exception = None try: - # If the stream is not configured to allow manual writes, this should throw an exception - await stream.write_data(BytesIO(b'hello'), False) - except RuntimeError as e: + # Attempt to access internal write_data method which should raise an exception + # since the stream wasn't created with async_body + await stream._write_data(BytesIO(b'hello'), False) + except (RuntimeError, AttributeError) as e: exception = e self.assertIsNotNone(exception) - await connection.close() def test_connect_http(self): @@ -563,13 +568,18 @@ async def _test_h2_mock_server_manual_write(self): request = HttpRequest('POST', self.mock_server_url.path) request.headers.add('host', self.mock_server_url.hostname) - stream = connection.request(request, manual_write=True) - # Write data in chunks - await stream.write_data(BytesIO(b'hello'), False) - await stream.write_data(BytesIO(b'he123123'), False) - await stream.write_data(None, False) - await stream.write_data(BytesIO(b'hello'), True) + # Create an async generator for the request body + body_chunks = [b'hello', b'he123123', b'', b'hello'] + total_length = 0 + for i in body_chunks: + total_length = total_length + len(i) + + async def body_generator(): + for i in body_chunks: + yield i + + stream = connection.request(request, async_body=body_generator()) # Collect response response = Response() @@ -578,7 +588,8 @@ async def _test_h2_mock_server_manual_write(self): # Check result self.assertEqual(200, status_code) self.assertEqual(200, response.status_code) - + # mock server response the total length received, check if it matches what we sent + self.assertEqual(total_length, int(response.body.decode())) await connection.close() class DelayStream: @@ -598,60 +609,6 @@ def read(self, _len): self._read = True return b'hello' - async def _test_h2_mock_server_manual_write_read_exception(self): - connection = await self._new_mock_connection() - # check we set an h2 connection - self.assertEqual(connection.version, HttpVersion.Http2) - - request = HttpRequest('POST', self.mock_server_url.path) - request.headers.add('host', self.mock_server_url.hostname) - stream = connection.request(request, manual_write=True) - - # Try to write data with a bad stream that raises an exception - exception = None - data = self.DelayStream(bad_read=True) - try: - await stream.write_data(data, False) - except Exception as e: - exception = e - stream_completion_exception = None - try: - await stream.wait_for_completion() - except Exception as e: - stream_completion_exception = e - - self.assertIsNotNone(exception) - self.assertIsNotNone(stream_completion_exception) - # assert that the exception is the same as the one we got from write_data. - self.assertEqual(str(exception), str(stream_completion_exception)) - await connection.close() - - async def _test_h2_mock_server_manual_write_lifetime(self): - connection = await self._new_mock_connection() - # check we set an h2 connection - self.assertEqual(connection.version, HttpVersion.Http2) - - request = HttpRequest('POST', self.mock_server_url.path) - request.headers.add('host', self.mock_server_url.hostname) - stream = connection.request(request, manual_write=True) - - # Create data stream and immediately delete the reference after writing - data = self.DelayStream(bad_read=False) - await stream.write_data(data, False) - del data - - # Finish the request - await stream.write_data(None, True) - - # Collect response - response = Response() - status_code = await response.collect_response(stream) - - # Check result - self.assertEqual(200, status_code) - - await connection.close() - async def _test_h2_mock_server_settings(self): # Test with invalid settings - should throw an exception exception = None @@ -669,9 +626,12 @@ async def _test_h2_mock_server_settings(self): request = HttpRequest('POST', self.mock_server_url.path) request.headers.add('host', self.mock_server_url.hostname) - stream = connection.request(request, manual_write=True) - await stream.write_data(BytesIO(b'hello'), True) + # Create an async generator for the request body + async def body_generator(): + yield b'hello' + + stream = connection.request(request, async_body=body_generator()) response = Response() status_code = await response.collect_response(stream) @@ -684,12 +644,6 @@ async def _test_h2_mock_server_settings(self): def test_h2_mock_server_manual_write(self): asyncio.run(self._test_h2_mock_server_manual_write()) - def test_h2_mock_server_manual_write_read_exception(self): - asyncio.run(self._test_h2_mock_server_manual_write_read_exception()) - - def test_h2_mock_server_manual_write_lifetime(self): - asyncio.run(self._test_h2_mock_server_manual_write_lifetime()) - def test_h2_mock_server_settings(self): asyncio.run(self._test_h2_mock_server_settings()) From f18b979b413cc70b4964b9b75f9f701f6f08a353 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 Jun 2025 09:47:41 -0700 Subject: [PATCH 24/40] naming update --- awscrt/aio/http_asyncio.py | 34 +++++++++++----------- awscrt/http.py | 58 ++++++++++++++++++++++++++++++++++++++ test/test_http_asyncio.py | 8 +++--- 3 files changed, 79 insertions(+), 21 deletions(-) diff --git a/awscrt/aio/http_asyncio.py b/awscrt/aio/http_asyncio.py index b0c311daa..2e52d7385 100644 --- a/awscrt/aio/http_asyncio.py +++ b/awscrt/aio/http_asyncio.py @@ -87,13 +87,13 @@ async def close(self) -> None: def request(self, request: 'HttpRequest', - async_body: AsyncIterator[bytes] = None, + request_body_generator: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> 'HttpClientStreamAsyncUnified': """Create `HttpClientStreamAsyncUnified` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. - async_body (AsyncIterator[bytes], optional): Async iterator providing chunks of the request body. + request_body_generator (AsyncIterator[bytes], optional): Async iterator providing chunks of the request body. If provided, the body will be sent incrementally as chunks become available. loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. If None, the current event loop is used. @@ -101,7 +101,7 @@ def request(self, Returns: HttpClientStreamAsyncUnified: Stream for the HTTP request/response exchange. """ - raise NotImplementedError("Subclasses must implement request") + return HttpClientStreamAsyncUnified(self, request, request_body_generator, loop) class HttpClientConnectionAsync(HttpClientConnectionAsyncUnified): @@ -156,13 +156,13 @@ async def new(cls, def request(self, request: 'HttpRequest', - async_body: AsyncIterator[bytes] = None, + request_body_generator: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> 'HttpClientStreamAsync': """Create `HttpClientStreamAsync` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. - async_body (AsyncIterator[bytes], optional): Async iterator providing chunks of the request body. + request_body_generator (AsyncIterator[bytes], optional): Async iterator providing chunks of the request body. Not supported for HTTP/1.1 connections yet, use the request's body_stream instead. loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. If None, the current event loop is used. @@ -221,13 +221,13 @@ async def new(cls, def request(self, request: 'HttpRequest', - async_body: AsyncIterator[bytes] = None, + request_body_generator: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> 'Http2ClientStreamAsync': """Create `Http2ClientStreamAsync` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. - async_body (AsyncIterator[bytes], optional): Async iterator providing chunks of the request body. + request_body_generator (AsyncIterator[bytes], optional): Async iterator providing chunks of the request body. If provided, the body will be sent incrementally as chunks become available from the iterator. loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. If None, the current event loop is used. @@ -235,7 +235,7 @@ def request(self, Returns: Http2ClientStreamAsync: Stream for the HTTP/2 request/response exchange. """ - return Http2ClientStreamAsync(self, request, async_body, loop) + return Http2ClientStreamAsync(self, request, request_body_generator, loop) class HttpClientStreamAsyncUnified(HttClientStreamBase): @@ -252,10 +252,10 @@ class HttpClientStreamAsyncUnified(HttClientStreamBase): def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, - async_body: AsyncIterator[bytes] = None, + request_body_generator: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: # Initialize the parent class - http2_manual_write = async_body is not None and connection.version is HttpVersion.Http2 + http2_manual_write = request_body_generator is not None and connection.version is HttpVersion.Http2 super()._init_common(connection, request, http2_manual_write=http2_manual_write) # Attach the event loop for async operations @@ -278,9 +278,9 @@ def __init__(self, self._response_headers_future = self._loop.create_future() self._status_code = None - self._async_body = async_body - if self._async_body is not None: - self._writer = self._loop.create_task(self._set_async_body(self._async_body)) + self._request_body_generator = request_body_generator + if self._request_body_generator is not None: + self._writer = self._loop.create_task(self._set_request_body_generator(self._request_body_generator)) # Activate the stream immediately _awscrt.http_client_stream_activate(self) @@ -362,7 +362,7 @@ async def wait_for_completion(self) -> int: """ return await self._completion_future - async def _set_async_body(self, body_iterator: AsyncIterator[bytes]): + async def _set_request_body_generator(self, body_iterator: AsyncIterator[bytes]): ... @@ -406,9 +406,9 @@ class Http2ClientStreamAsync(HttpClientStreamAsyncUnified): def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, - async_body: AsyncIterator[bytes] = None, + request_body_generator: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: - super().__init__(connection, request, async_body=async_body, loop=loop) + super().__init__(connection, request, request_body_generator=request_body_generator, loop=loop) async def _write_data(self, body, end_stream): future: Future = Future() @@ -426,7 +426,7 @@ def on_write_complete(error_code: int) -> None: _awscrt.http2_client_stream_write_data(self, body_stream, end_stream, on_write_complete) await asyncio.wrap_future(future) - async def _set_async_body(self, body_iterator: AsyncIterator[bytes]): + async def _set_request_body_generator(self, body_iterator: AsyncIterator[bytes]): try: async for chunk in body_iterator: await self._write_data(io.BytesIO(chunk), False) diff --git a/awscrt/http.py b/awscrt/http.py index 244f8ea61..bea4ff1f3 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -343,6 +343,45 @@ def request(self, on_response: Optional[Callable[..., None]] = None, on_body: Optional[Callable[..., None]] = None, manual_write: bool = False) -> 'Http2ClientStream': + """Create `Http2ClientStream` to carry out the request/response exchange. + + NOTE: The HTTP stream sends no data until `Http2ClientStream.activate()` + is called. Call activate() when you're ready for callbacks and events to fire. + + Args: + request (HttpRequest): Definition for outgoing request. + + on_response: Optional callback invoked once main response headers are received. + The function should take the following arguments and return nothing: + + * `http_stream` (`Http2ClientStream`): HTTP/2 stream carrying + out this request/response exchange. + + * `status_code` (int): Response status code. + + * `headers` (List[Tuple[str, str]]): Response headers as a + list of (name,value) pairs. + + * `**kwargs` (dict): Forward compatibility kwargs. + + on_body: Optional callback invoked 0+ times as response body data is received. + The function should take the following arguments and return nothing: + + * `http_stream` (`Http2ClientStream`): HTTP/2 stream carrying + out this request/response exchange. + + * `chunk` (buffer): Response body data (not necessarily + a whole "chunk" of chunked encoding). + + * `**kwargs` (dict): Forward-compatibility kwargs. + + manual_write (bool): If True, enables manual data writing on the stream. + This allows calling `write_data()` to stream the request body in chunks. + Note: In the asyncio version, this is replaced by the async_body parameter. + + Returns: + Http2ClientStream: Stream for the HTTP/2 request/response exchange. + """ return Http2ClientStream(self, request, on_response, on_body, manual_write) def close(self) -> "concurrent.futures.Future": @@ -487,6 +526,25 @@ def activate(self) -> None: def write_data(self, data_stream: Union[InputStream, Any], end_stream: bool = False) -> "concurrent.futures.Future": + """Write a chunk of data to the request body stream. + + This method is only available when the stream was created with + manual_write=True. This allows incremental writing of request data. + + Note: In the asyncio version, this is replaced by the request_body_generator parameter + which accepts an async generator. + + Args: + data_stream (Union[InputStream, Any]): Data to write. If not an InputStream, + it will be wrapped in one. Can be None to send an empty chunk. + + end_stream (bool): True to indicate this is the last chunk and no more data + will be sent. False if more chunks will follow. + + Returns: + concurrent.futures.Future: Future that completes when the write operation + is done. The future will contain None on success, or an exception on failure. + """ future: Future = Future() body_stream: InputStream = InputStream.wrap(data_stream, allow_none=True) diff --git a/test/test_http_asyncio.py b/test/test_http_asyncio.py index 75ea7b3d8..f22d6709a 100644 --- a/test/test_http_asyncio.py +++ b/test/test_http_asyncio.py @@ -318,7 +318,7 @@ async def _test_h2_manual_write_exception(self): request = HttpRequest('GET', url.path) request.headers.add('host', url.hostname) - # Create stream without using async_body parameter + # Create stream without using request_body_generator parameter # (which would be needed to properly configure it for writing) stream = connection.request(request) @@ -327,7 +327,7 @@ async def _test_h2_manual_write_exception(self): exception = None try: # Attempt to access internal write_data method which should raise an exception - # since the stream wasn't created with async_body + # since the stream wasn't created with request_body_generator await stream._write_data(BytesIO(b'hello'), False) except (RuntimeError, AttributeError) as e: exception = e @@ -579,7 +579,7 @@ async def body_generator(): for i in body_chunks: yield i - stream = connection.request(request, async_body=body_generator()) + stream = connection.request(request, request_body_generator=body_generator()) # Collect response response = Response() @@ -631,7 +631,7 @@ async def _test_h2_mock_server_settings(self): async def body_generator(): yield b'hello' - stream = connection.request(request, async_body=body_generator()) + stream = connection.request(request, request_body_generator=body_generator()) response = Response() status_code = await response.collect_response(stream) From 324ec18d24c30b2f0733f1efc4f382a9148fe452 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 Jun 2025 10:22:16 -0700 Subject: [PATCH 25/40] typo... --- awscrt/http.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/awscrt/http.py b/awscrt/http.py index bea4ff1f3..ab56e8be4 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -399,7 +399,12 @@ def close(self) -> "concurrent.futures.Future": class HttpStreamBase(NativeResource): - """Base for HTTP stream classes""" + """Base for HTTP stream classes. + + Attributes: + connection: The HTTP connection this stream belongs to. + completion_future: Future that completes when the operation finishes. + """ __slots__ = ('_connection', '_completion_future', '_on_body_cb') def __init__(self, connection: HttpConnectionBase, on_body: Optional[Callable[..., None]] = None) -> None: @@ -421,7 +426,16 @@ def _on_body(self, chunk: bytes) -> None: self._on_body_cb(http_stream=self, chunk=chunk) -class HttClientStreamBase(HttpStreamBase): +class HttpClientStreamBase(HttpStreamBase): + """Base for HTTP client stream classes. + + Attributes: + connection: This stream's connection. + + completion_future: Future that completes when + the request/response exchange is finished. + """ + __slots__ = ('_response_status_code', '_on_response_cb', '_on_body_cb', '_request', '_version') def _init_common(self, @@ -473,7 +487,7 @@ def _on_complete(self, error_code: int) -> None: self._completion_future.set_exception(awscrt.exceptions.from_code(error_code)) -class HttpClientStream(HttClientStreamBase): +class HttpClientStream(HttpClientStreamBase): """HTTP stream that sends a request and receives a response. Create an HttpClientStream with :meth:`HttpClientConnection.request()`. @@ -506,7 +520,7 @@ def activate(self) -> None: _awscrt.http_client_stream_activate(self) -class Http2ClientStream(HttClientStreamBase): +class Http2ClientStream(HttpClientStreamBase): def __init__(self, connection: HttpClientConnection, request: 'HttpRequest', From fdbef55be47b3cf888b6578617dc854a07ab6c69 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 Jun 2025 10:24:31 -0700 Subject: [PATCH 26/40] hmmm --- awscrt/aio/http_asyncio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awscrt/aio/http_asyncio.py b/awscrt/aio/http_asyncio.py index 2e52d7385..58ccc413c 100644 --- a/awscrt/aio/http_asyncio.py +++ b/awscrt/aio/http_asyncio.py @@ -15,7 +15,7 @@ import awscrt.exceptions from typing import List, Tuple, Optional, Union, Callable, Any, AsyncIterator from awscrt.http import ( - HttpClientConnectionBase, HttpRequest, HttClientStreamBase, HttpProxyOptions, + HttpClientConnectionBase, HttpRequest, HttpClientStreamBase, HttpProxyOptions, Http2Setting, HttpVersion ) from awscrt.io import ( @@ -238,7 +238,7 @@ def request(self, return Http2ClientStreamAsync(self, request, request_body_generator, loop) -class HttpClientStreamAsyncUnified(HttClientStreamBase): +class HttpClientStreamAsyncUnified(HttpClientStreamBase): __slots__ = ( '_response_status_future', '_response_headers_future', From d3e3ba4aede987fa0a1e57fdd4a1dccc5b3cec31 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 Jun 2025 10:47:21 -0700 Subject: [PATCH 27/40] why??? --- awscrt/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awscrt/http.py b/awscrt/http.py index ab56e8be4..834e809d8 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -407,7 +407,7 @@ class HttpStreamBase(NativeResource): """ __slots__ = ('_connection', '_completion_future', '_on_body_cb') - def __init__(self, connection: HttpConnectionBase, on_body: Optional[Callable[..., None]] = None) -> None: + def __init__(self, connection, on_body: Optional[Callable[..., None]] = None) -> None: super().__init__() self._connection: HttpConnectionBase = connection self._completion_future: Future = Future() From c48d68f01c77e630ef805636baa53049d1ee449c Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 Jun 2025 11:52:43 -0700 Subject: [PATCH 28/40] renaming the module and class --- awscrt/__init__.py | 2 +- awscrt/aio/{http_asyncio.py => aiohttp.py} | 72 +++++++++---------- awscrt/http.py | 6 +- examples/http2_asyncio_demo.py | 26 +++---- ...http_asyncio.py => test_aiohttp_client.py} | 8 +-- 5 files changed, 57 insertions(+), 57 deletions(-) rename awscrt/aio/{http_asyncio.py => aiohttp.py} (86%) rename test/{test_http_asyncio.py => test_aiohttp_client.py} (98%) diff --git a/awscrt/__init__.py b/awscrt/__init__.py index 1726b5e16..c2a348827 100644 --- a/awscrt/__init__.py +++ b/awscrt/__init__.py @@ -7,7 +7,7 @@ 'auth', 'crypto', 'http', - 'http_asyncio', + 'aio.aiohttp', 'io', 'mqtt', 'mqtt5', diff --git a/awscrt/aio/http_asyncio.py b/awscrt/aio/aiohttp.py similarity index 86% rename from awscrt/aio/http_asyncio.py rename to awscrt/aio/aiohttp.py index 58ccc413c..e3611e2c2 100644 --- a/awscrt/aio/http_asyncio.py +++ b/awscrt/aio/aiohttp.py @@ -24,11 +24,11 @@ from collections import deque -class HttpClientConnectionAsyncUnified(HttpClientConnectionBase): +class AIOHttpClientConnectionUnified(HttpClientConnectionBase): """ An async unified HTTP client connection for either a HTTP/1 or HTTP/2 connection. - Use `HttpClientConnectionAsync.new()` to establish a new connection. + Use `AIOHttpClientConnection.new()` to establish a new connection. """ @classmethod @@ -38,9 +38,9 @@ async def new(cls, bootstrap: Optional[ClientBootstrap] = None, socket_options: Optional[SocketOptions] = None, tls_connection_options: Optional[TlsConnectionOptions] = None, - proxy_options: Optional['HttpProxyOptions'] = None) -> "HttpClientConnectionAsyncUnified": + proxy_options: Optional['HttpProxyOptions'] = None) -> "AIOHttpClientConnectionUnified": """ - Asynchronously establish a new HttpClientConnectionAsyncUnified. + Asynchronously establish a new AIOHttpClientConnectionUnified. Args: host_name (str): Connect to host. @@ -61,7 +61,7 @@ async def new(cls, If None is provided then a proxy is not used. Returns: - HttpClientConnectionAsync: A new HTTP client connection. + AIOHttpClientConnection: A new HTTP client connection. """ future = cls._generic_new( host_name, @@ -88,8 +88,8 @@ async def close(self) -> None: def request(self, request: 'HttpRequest', request_body_generator: AsyncIterator[bytes] = None, - loop: Optional[asyncio.AbstractEventLoop] = None) -> 'HttpClientStreamAsyncUnified': - """Create `HttpClientStreamAsyncUnified` to carry out the request/response exchange. + loop: Optional[asyncio.AbstractEventLoop] = None) -> 'AIOHttpClientStreamUnified': + """Create `AIOHttpClientStreamUnified` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. @@ -99,16 +99,16 @@ def request(self, If None, the current event loop is used. Returns: - HttpClientStreamAsyncUnified: Stream for the HTTP request/response exchange. + AIOHttpClientStreamUnified: Stream for the HTTP request/response exchange. """ - return HttpClientStreamAsyncUnified(self, request, request_body_generator, loop) + return AIOHttpClientStreamUnified(self, request, request_body_generator, loop) -class HttpClientConnectionAsync(HttpClientConnectionAsyncUnified): +class AIOHttpClientConnection(AIOHttpClientConnectionUnified): """ An async HTTP/1.1 only client connection. - Use `HttpClientConnectionAsync.new()` to establish a new connection. + Use `AIOHttpClientConnection.new()` to establish a new connection. """ @classmethod @@ -118,9 +118,9 @@ async def new(cls, bootstrap: Optional[ClientBootstrap] = None, socket_options: Optional[SocketOptions] = None, tls_connection_options: Optional[TlsConnectionOptions] = None, - proxy_options: Optional['HttpProxyOptions'] = None) -> "HttpClientConnectionAsync": + proxy_options: Optional['HttpProxyOptions'] = None) -> "AIOHttpClientConnection": """ - Asynchronously establish a new HttpClientConnectionAsync. + Asynchronously establish a new AIOHttpClientConnection. Args: host_name (str): Connect to host. @@ -141,7 +141,7 @@ async def new(cls, If None is provided then a proxy is not used. Returns: - HttpClientConnectionAsync: A new HTTP client connection. + AIOHttpClientConnection: A new HTTP client connection. """ future = cls._generic_new( host_name, @@ -157,8 +157,8 @@ async def new(cls, def request(self, request: 'HttpRequest', request_body_generator: AsyncIterator[bytes] = None, - loop: Optional[asyncio.AbstractEventLoop] = None) -> 'HttpClientStreamAsync': - """Create `HttpClientStreamAsync` to carry out the request/response exchange. + loop: Optional[asyncio.AbstractEventLoop] = None) -> 'AIOHttpClientStream': + """Create `AIOHttpClientStream` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. @@ -168,16 +168,16 @@ def request(self, If None, the current event loop is used. Returns: - HttpClientStreamAsync: Stream for the HTTP request/response exchange. + AIOHttpClientStream: Stream for the HTTP request/response exchange. """ - return HttpClientStreamAsync(self, request, loop) + return AIOHttpClientStream(self, request, loop) -class Http2ClientConnectionAsync(HttpClientConnectionAsyncUnified): +class AIOHttp2ClientConnection(AIOHttpClientConnectionUnified): """ An async HTTP/2 only client connection. - Use `Http2ClientConnectionAsync.new()` to establish a new connection. + Use `AIOHttp2ClientConnection.new()` to establish a new connection. """ @classmethod @@ -190,12 +190,12 @@ async def new(cls, proxy_options: Optional['HttpProxyOptions'] = None, initial_settings: Optional[List[Http2Setting]] = None, on_remote_settings_changed: Optional[Callable[[List[Http2Setting]], - None]] = None) -> "Http2ClientConnectionAsync": + None]] = None) -> "AIOHttp2ClientConnection": """ Asynchronously establish an HTTP/2 client connection. Notes: to set up the connection, the server must support HTTP/2 and TlsConnectionOptions - This class extends HttpClientConnectionAsync with HTTP/2 specific functionality. + This class extends AIOHttpClientConnection with HTTP/2 specific functionality. HTTP/2 specific args: initial_settings (List[Http2Setting]): The initial settings to change for the connection. @@ -222,8 +222,8 @@ async def new(cls, def request(self, request: 'HttpRequest', request_body_generator: AsyncIterator[bytes] = None, - loop: Optional[asyncio.AbstractEventLoop] = None) -> 'Http2ClientStreamAsync': - """Create `Http2ClientStreamAsync` to carry out the request/response exchange. + loop: Optional[asyncio.AbstractEventLoop] = None) -> 'AIOHttp2ClientStream': + """Create `AIOHttp2ClientStream` to carry out the request/response exchange. Args: request (HttpRequest): Definition for outgoing request. @@ -233,12 +233,12 @@ def request(self, If None, the current event loop is used. Returns: - Http2ClientStreamAsync: Stream for the HTTP/2 request/response exchange. + AIOHttp2ClientStream: Stream for the HTTP/2 request/response exchange. """ - return Http2ClientStreamAsync(self, request, request_body_generator, loop) + return AIOHttp2ClientStream(self, request, request_body_generator, loop) -class HttpClientStreamAsyncUnified(HttpClientStreamBase): +class AIOHttpClientStreamUnified(HttpClientStreamBase): __slots__ = ( '_response_status_future', '_response_headers_future', @@ -250,7 +250,7 @@ class HttpClientStreamAsyncUnified(HttpClientStreamBase): '_loop') def __init__(self, - connection: HttpClientConnectionAsync, + connection: AIOHttpClientConnection, request: HttpRequest, request_body_generator: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: @@ -366,13 +366,13 @@ async def _set_request_body_generator(self, body_iterator: AsyncIterator[bytes]) ... -class HttpClientStreamAsync(HttpClientStreamAsyncUnified): +class AIOHttpClientStream(AIOHttpClientStreamUnified): """Async HTTP stream that sends a request and receives a response. - Create an HttpClientStreamAsync with `HttpClientConnectionAsync.request()`. + Create an AIOHttpClientStream with `AIOHttpClientConnection.request()`. Attributes: - connection (HttpClientConnectionAsync): This stream's connection. + connection (AIOHttpClientConnection): This stream's connection. completion_future (asyncio.Future): Future that will contain the response status code (int) when the request/response exchange @@ -384,12 +384,12 @@ class HttpClientStreamAsync(HttpClientStreamAsyncUnified): thread that owns the event loop used to create the stream """ - def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, + def __init__(self, connection: AIOHttpClientConnection, request: HttpRequest, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: """Initialize an HTTP client stream. Args: - connection (HttpClientConnectionAsync): The connection to send the request on. + connection (AIOHttpClientConnection): The connection to send the request on. request (HttpRequest): The HTTP request to send. loop (Optional[asyncio.AbstractEventLoop]): Event loop to use for async operations. If None, the current event loop is used. @@ -397,14 +397,14 @@ def __init__(self, connection: HttpClientConnectionAsync, request: HttpRequest, super().__init__(connection, request, loop=loop) -class Http2ClientStreamAsync(HttpClientStreamAsyncUnified): +class AIOHttp2ClientStream(AIOHttpClientStreamUnified): """HTTP/2 stream that sends a request and receives a response. - Create an Http2ClientStreamAsync with `Http2ClientConnectionAsync.request()`. + Create an AIOHttp2ClientStream with `AIOHttp2ClientConnection.request()`. """ def __init__(self, - connection: HttpClientConnectionAsync, + connection: AIOHttpClientConnection, request: HttpRequest, request_body_generator: AsyncIterator[bytes] = None, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: diff --git a/awscrt/http.py b/awscrt/http.py index 834e809d8..659ae061c 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -938,11 +938,11 @@ def _on_connection_setup(self, binding: Any, error_code: int, http_version: Http return if self._asyncio_connection: # Import is done here to avoid circular import issues - from awscrt.aio.http_asyncio import HttpClientConnectionAsync, Http2ClientConnectionAsync + from awscrt.aio.aiohttp import AIOHttpClientConnection, AIOHttp2ClientConnection if http_version == HttpVersion.Http2: - connection = Http2ClientConnectionAsync() + connection = AIOHttp2ClientConnection() else: - connection = HttpClientConnectionAsync() + connection = AIOHttpClientConnection() else: if http_version == HttpVersion.Http2: connection = Http2ClientConnection() diff --git a/examples/http2_asyncio_demo.py b/examples/http2_asyncio_demo.py index 7880ccda3..c856a1a44 100644 --- a/examples/http2_asyncio_demo.py +++ b/examples/http2_asyncio_demo.py @@ -12,7 +12,7 @@ import io from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions from awscrt.http import HttpHeaders, HttpRequest, Http2Setting, Http2SettingID -from awscrt.aio.http_asyncio import Http2ClientConnectionAsync +from awscrt.aio.aiohttp import AIOHttp2ClientConnection # from awscrt_python_logging_example import PythonLoggingRedirector import awscrt.io import logging @@ -91,7 +91,7 @@ async def make_concurrent_requests(): ] print(f"Connecting to {host_name}:{port} using HTTP/2...") - connection = await Http2ClientConnectionAsync.new( + connection = await AIOHttp2ClientConnection.new( host_name=host_name, port=port, bootstrap=bootstrap, @@ -236,9 +236,9 @@ async def async_data_generator(): print(f"Yielding chunk of size: {len(chunk)} bytes") yield chunk # Simulate some async processing between chunks - await asyncio.sleep(2) + await asyncio.sleep(0.1) - stream = connection.request(request, async_body=async_data_generator()) + stream = connection.request(request, request_body_generator=async_data_generator()) # Process the response print("hey") @@ -250,15 +250,15 @@ async def async_data_generator(): print(f"{name}: {value}") # Get the response body - # body = bytearray() - # while True: - # chunk = await stream.get_next_response_chunk() - # if not chunk: - # break - # body.extend(chunk) - - # print("\nStream Response body:") - # print(body.decode("utf-8")) + body = bytearray() + while True: + chunk = await stream.get_next_response_chunk() + if not chunk: + break + body.extend(chunk) + + print("\nStream Response body:") + print(body.decode("utf-8")) await stream.wait_for_completion() # Return the status code from the async iterator example diff --git a/test/test_http_asyncio.py b/test/test_aiohttp_client.py similarity index 98% rename from test/test_http_asyncio.py rename to test/test_aiohttp_client.py index f22d6709a..c57ebe37c 100644 --- a/test/test_http_asyncio.py +++ b/test/test_aiohttp_client.py @@ -18,7 +18,7 @@ from awscrt import io from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsContextOptions, TlsCipherPref from awscrt.http import HttpHeaders, HttpRequest, HttpVersion, Http2Setting, Http2SettingID -from awscrt.aio.http_asyncio import HttpClientConnectionAsync, Http2ClientConnectionAsync +from awscrt.aio.aiohttp import AIOHttpClientConnection, AIOHttp2ClientConnection import threading @@ -103,7 +103,7 @@ async def _new_client_connection(self, secure, proxy_options=None): event_loop_group = EventLoopGroup() host_resolver = DefaultHostResolver(event_loop_group) bootstrap = ClientBootstrap(event_loop_group, host_resolver) - return await HttpClientConnectionAsync.new( + return await AIOHttpClientConnection.new( host_name=self.hostname, port=self.port, bootstrap=bootstrap, @@ -279,7 +279,7 @@ async def _new_h2_client_connection(self, url): tls_conn_opt.set_server_name(url.hostname) tls_conn_opt.set_alpn_list(["h2"]) - connection = await Http2ClientConnectionAsync.new( + connection = await AIOHttp2ClientConnection.new( host_name=url.hostname, port=port, bootstrap=bootstrap, @@ -551,7 +551,7 @@ async def _new_mock_connection(self, initial_settings=None): if initial_settings is None: initial_settings = [Http2Setting(Http2SettingID.ENABLE_PUSH, 0)] - connection = await Http2ClientConnectionAsync.new( + connection = await AIOHttp2ClientConnection.new( host_name=self.mock_server_url.hostname, port=port, bootstrap=bootstrap, From c3a442d4b0fafc1bc19614571cae0443c7184868 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 18 Jun 2025 11:58:19 -0700 Subject: [PATCH 29/40] update the path --- test/test_aiohttp_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_aiohttp_client.py b/test/test_aiohttp_client.py index c57ebe37c..2cffcf8aa 100644 --- a/test/test_aiohttp_client.py +++ b/test/test_aiohttp_client.py @@ -129,7 +129,7 @@ async def _test_get(self, secure): connection = await self._new_client_connection(secure) self.assertTrue(connection.is_open()) - test_asset_path = 'test/test_http_asyncio.py' + test_asset_path = 'test/test_aiohttp_client.py' # Create request and get stream - stream is already activated request = HttpRequest('GET', '/' + test_asset_path) @@ -157,7 +157,7 @@ async def _test_put(self, secure): self._start_server(secure) try: connection = await self._new_client_connection(secure) - test_asset_path = 'test/test_http_asyncio.py' + test_asset_path = 'test/test_aiohttp_client.py' with open(test_asset_path, 'rb') as outgoing_body_stream: outgoing_body_bytes = outgoing_body_stream.read() headers = HttpHeaders([ @@ -217,7 +217,7 @@ async def _test_stream_lives_until_complete(self, secure): try: connection = await self._new_client_connection(secure) - request = HttpRequest('GET', '/test/test_http_asyncio.py') + request = HttpRequest('GET', '/test/test_aiohttp_client.py') stream = connection.request(request) # Store stream but delete all local references @@ -394,7 +394,7 @@ async def _test_cross_thread_http_client(self, secure): # Function to run in a different thread with a different event loop async def thread_func(conn): # Create new event loop for this thread - test_asset_path = 'test/test_http_asyncio.py' + test_asset_path = 'test/test_aiohttp_client.py' request = HttpRequest('GET', '/' + test_asset_path) # Use the connection but with the current thread's event loop From 33914babde4a0721e6384e9c345b1a0af49ce833 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 19 Jun 2025 09:43:39 -0700 Subject: [PATCH 30/40] address comments --- awscrt/aio/aiohttp.py | 14 ++++++-------- awscrt/http.py | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/awscrt/aio/aiohttp.py b/awscrt/aio/aiohttp.py index e3611e2c2..d39207d4d 100644 --- a/awscrt/aio/aiohttp.py +++ b/awscrt/aio/aiohttp.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: Apache-2.0. import asyncio -import io +from io import BytesIO import _awscrt from concurrent.futures import Future import awscrt.exceptions @@ -317,8 +317,8 @@ def _set_completion(self, error_code: int) -> None: else: self._completion_future.set_exception(awscrt.exceptions.from_code(error_code)) - if self._chunk_futures: - # the stream is completed, so we need to set the futures + # Resolve all pending chunk futures with an empty string to indicate end of stream + while self._chunk_futures: future = self._chunk_futures.popleft() future.set_result("") @@ -411,8 +411,8 @@ def __init__(self, super().__init__(connection, request, request_body_generator=request_body_generator, loop=loop) async def _write_data(self, body, end_stream): - future: Future = Future() - body_stream: InputStream = InputStream.wrap(body, allow_none=True) + future = Future() + body_stream = InputStream.wrap(body, allow_none=True) def on_write_complete(error_code: int) -> None: if future.cancelled(): @@ -429,8 +429,6 @@ def on_write_complete(error_code: int) -> None: async def _set_request_body_generator(self, body_iterator: AsyncIterator[bytes]): try: async for chunk in body_iterator: - await self._write_data(io.BytesIO(chunk), False) - except Exception: - raise + await self._write_data(BytesIO(chunk), False) finally: await self._write_data(None, True) diff --git a/awscrt/http.py b/awscrt/http.py index 659ae061c..ff5c20b42 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -409,8 +409,8 @@ class HttpStreamBase(NativeResource): def __init__(self, connection, on_body: Optional[Callable[..., None]] = None) -> None: super().__init__() - self._connection: HttpConnectionBase = connection - self._completion_future: Future = Future() + self._connection = connection + self._completion_future = Future() self._on_body_cb: Optional[Callable[..., None]] = on_body @property @@ -455,8 +455,8 @@ def _init_common(self, self._response_status_code: Optional[int] = None # keep HttpRequest alive until stream completes - self._request: 'HttpRequest' = request - self._version: HttpVersion = connection.version + self._request = request + self._version = connection.version self._binding = _awscrt.http_client_stream_new(self, connection, request, http2_manual_write) @property @@ -559,8 +559,8 @@ def write_data(self, concurrent.futures.Future: Future that completes when the write operation is done. The future will contain None on success, or an exception on failure. """ - future: Future = Future() - body_stream: InputStream = InputStream.wrap(data_stream, allow_none=True) + future = Future() + body_stream = InputStream.wrap(data_stream, allow_none=True) def on_write_complete(error_code: int) -> None: if future.cancelled(): @@ -587,7 +587,7 @@ def __init__(self, binding: Any, headers: 'HttpHeaders', super().__init__() self._binding = binding - self._headers: HttpHeaders = headers + self._headers = headers self._body_stream: Optional[InputStream] = None if body_stream: @@ -891,13 +891,13 @@ def __init__(self, auth_username: Optional[str] = None, auth_password: Optional[str] = None, connection_type: HttpProxyConnectionType = HttpProxyConnectionType.Legacy) -> None: - self.host_name: str = host_name - self.port: int = port - self.tls_connection_options: Optional[TlsConnectionOptions] = tls_connection_options - self.auth_type: HttpProxyAuthenticationType = auth_type - self.auth_username: Optional[str] = auth_username - self.auth_password: Optional[str] = auth_password - self.connection_type: HttpProxyConnectionType = connection_type + self.host_name = host_name + self.port = port + self.tls_connection_options = tls_connection_options + self.auth_type = auth_type + self.auth_username = auth_username + self.auth_password = auth_password + self.connection_type = connection_type class _HttpClientConnectionCore: From 5bced4af8fa701d3fe05cd60bf5e14003485f38f Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 19 Jun 2025 10:00:54 -0700 Subject: [PATCH 31/40] remove the demo --- examples/http2_asyncio_demo.py | 291 --------------------------------- 1 file changed, 291 deletions(-) delete mode 100644 examples/http2_asyncio_demo.py diff --git a/examples/http2_asyncio_demo.py b/examples/http2_asyncio_demo.py deleted file mode 100644 index c856a1a44..000000000 --- a/examples/http2_asyncio_demo.py +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env python3 -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0. - -""" -This example demonstrates how to use the asyncio HTTP/2 client in awscrt. -It performs multiple concurrent requests to httpbin.org and shows HTTP/2 features. -""" - -import asyncio -import sys -import io -from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions -from awscrt.http import HttpHeaders, HttpRequest, Http2Setting, Http2SettingID -from awscrt.aio.aiohttp import AIOHttp2ClientConnection -# from awscrt_python_logging_example import PythonLoggingRedirector -import awscrt.io -import logging -from typing import List, Tuple, Optional, Union, Callable, Any, AsyncIterator - - -class Response: - """Holds contents of incoming response""" - - def __init__(self, request_name): - self.request_name = request_name - self.status_code = None - self.headers = None - self.body = bytearray() - - def on_response(self, http_stream, status_code, headers, **kwargs): - print(f"[{self.request_name}] Received response status: {status_code}") - self.status_code = status_code - self.headers = HttpHeaders(headers) - for name, value in headers: - print(f"[{self.request_name}] Header: {name}: {value}") - - def on_body(self, http_stream, chunk, **kwargs): - print(f"[{self.request_name}] Received body chunk of size: {len(chunk)} bytes") - self.body.extend(chunk) - - -# Create an event for synchronizing remote settings -remote_settings_event = None -event_loop = None - - -def on_remote_settings_changed(settings): - """Handler for when the server updates HTTP/2 settings""" - print("Remote HTTP/2 settings changed:") - for setting in settings: - print(f" - {setting.id.name} = {setting.value}") - # Signal that remote settings have been received - # This callback is called from a different thread, so we need to use call_soon_threadsafe - if event_loop and remote_settings_event: - event_loop.call_soon_threadsafe(remote_settings_event.set) - - -async def make_concurrent_requests(): - """Perform multiple concurrent HTTP/2 requests asynchronously.""" - global remote_settings_event, event_loop - - # Get the current event loop and create the event - event_loop = asyncio.get_running_loop() - remote_settings_event = asyncio.Event() - - # Create an event loop group and default host resolver - event_loop_group = EventLoopGroup() - host_resolver = DefaultHostResolver(event_loop_group) - bootstrap = ClientBootstrap(event_loop_group, host_resolver) - - # Connect to httpbin.org - # host_name = "postman-echo.com" # Change to "httpbin.org" for real requests - # port = 443 - host_name = "localhost" # Change to "httpbin.org" for real requests - port = 3443 - - # TLS options for HTTP/2 - tls_ctx_opt = TlsContextOptions() - tls_ctx_opt.verify_peer = False - tls_ctx = ClientTlsContext(tls_ctx_opt) - tls_conn_opt = tls_ctx.new_connection_options() - tls_conn_opt.set_server_name(host_name) - tls_conn_opt.set_alpn_list(["h2"]) # Set ALPN to HTTP/2 - - # Initial HTTP/2 settings - initial_settings = [ - Http2Setting(Http2SettingID.ENABLE_PUSH, 0), - Http2Setting(Http2SettingID.MAX_CONCURRENT_STREAMS, 100), - Http2Setting(Http2SettingID.INITIAL_WINDOW_SIZE, 65535), - ] - - print(f"Connecting to {host_name}:{port} using HTTP/2...") - connection = await AIOHttp2ClientConnection.new( - host_name=host_name, - port=port, - bootstrap=bootstrap, - tls_connection_options=tls_conn_opt, - initial_settings=initial_settings, - on_remote_settings_changed=on_remote_settings_changed - ) - print("HTTP/2 Connection established!") - - # Wait for remote settings to be received - print("Waiting for remote settings...") - await remote_settings_event.wait() - print("Remote settings received, proceeding with requests...") - - try: - # Create several requests to be executed concurrently - tasks = [] - - # Request 1: Simple GET - # tasks.append(send_get_request(connection, host_name)) - - # Request 2: POST with JSON body - # tasks.append(send_post_request(connection, host_name)) - - # Request 3: Stream data using manual write mode - tasks.append(send_stream_request(connection, host_name)) - - # Wait for all requests to complete - results = await asyncio.gather(*tasks, return_exceptions=True) - - # Check for any exceptions - for i, result in enumerate(results): - if isinstance(result, Exception): - print(f"Task {i} failed with exception: {result}") - - finally: - # Add a small delay to ensure all responses are received - await asyncio.sleep(1) - - # Close the connection - print("Closing connection...") - await connection.close() - print("Connection closed!") - - -async def send_get_request(connection, host_name): - """Send a GET request using the HTTP/2 connection.""" - print("Sending GET request...") - request = HttpRequest("GET", "/delete") - request.headers.add("host", host_name) - - # Set up response handler - response = Response("GET") - stream = connection.request(request) - headers = await stream.get_response_headers() - print(headers) - - # Wait for completion - status_code = await stream.wait_for_completion() - print(f"GET request completed with status code: {status_code}") - print("\nGET Response body:") - print(response.body.decode("utf-8")) - return status_code - - -async def send_post_request(connection, host_name): - """Send a POST request with JSON body using the HTTP/2 connection.""" - print("Sending POST request with JSON body...") - - # Prepare JSON payload - json_payload = '{"name": "Example User", "id": 12345}' - - # Create request with headers - request = HttpRequest("POST", "/post") - request.headers.add("host", host_name) - request.headers.add("content-type", "application/json") - request.headers.add("content-length", str(len(json_payload))) - - # Set the body using BytesIO stream - request.body_stream = io.BytesIO(json_payload.encode("utf-8")) - - # Set up response handler - response = Response("POST") - stream = connection.request(request) - - # Wait for completion - status_code = await stream.wait_for_completion() - print(f"POST request completed with status code: {status_code}") - print("\nPOST Response body:") - print(response.body.decode("utf-8")) - return status_code - - -async def data_generator() -> AsyncIterator[bytes]: - for i in range(5): - yield f"chunk {i}".encode() - await asyncio.sleep(0.1) # Simulate delay between chunks - - -async def send_stream_request(connection, host_name): - """Send a request with streamed data using manual write mode.""" - # print("Sending request with manual data streaming...") - - # # Create request - # request = HttpRequest("PUT", "/put") - # request.headers.add("host", host_name) - # request.headers.add("content-type", "text/plain") - # # Note: We don't set content-length as we're streaming the data - - # # Set up response handler - # stream = connection.request(request, manual_write=True) - - # # Stream data in chunks using BytesIO (legacy method) - # print("\nMethod 1: Using BytesIO (legacy method)") - # data_chunks = [ - # b"This is the first chunk of data.\n", - # b"This is the second chunk of data.\n" - # ] - - # await stream.write_data(io.BytesIO(data_chunks[0]), end_stream=False) - # await stream.write_data(io.BytesIO(data_chunks[1]), end_stream=True) - - # # Wait for completion - # status_code = await stream.wait_for_completion() - # print(f"Stream request completed with status code: {status_code}") - - # Create a new stream for the AsyncIterator demo - print("\nMethod 2: Using AsyncIterator[bytes] (new method)") - request = HttpRequest("PUT", "/put") - request.headers.add("host", host_name) - request.headers.add("content-type", "text/plain") - - # Create an async generator function that yields chunks - - async def async_data_generator(): - chunks = [ - b"This is the first async chunk.\n", - b"This is the second async chunk.\n", - b"This is the final async chunk." - ] - for chunk in chunks: - print(f"Yielding chunk of size: {len(chunk)} bytes") - yield chunk - # Simulate some async processing between chunks - await asyncio.sleep(0.1) - - stream = connection.request(request, request_body_generator=async_data_generator()) - - # Process the response - print("hey") - status_code = await stream.get_response_status_code() - print(f"Async iterator stream request completed with status code: {status_code}") - headers = await stream.get_response_headers() - print("\nStream Response headers:") - for name, value in headers: - print(f"{name}: {value}") - - # Get the response body - body = bytearray() - while True: - chunk = await stream.get_next_response_chunk() - if not chunk: - break - body.extend(chunk) - - print("\nStream Response body:") - print(body.decode("utf-8")) - await stream.wait_for_completion() - - # Return the status code from the async iterator example - return status_code - - -def main(): - """Entry point for the example.""" - try: - - # Set up Python logging - logging.basicConfig(level=logging.DEBUG) - # awscrt.io.init_logging(awscrt.io.LogLevel.Trace, "stdout") - # Create and activate redirector - # redirector = PythonLoggingRedirector(base_logger_name="myapp.awscrt") - # redirector.activate(aws_log_level=awscrt.io.LogLevel.Trace) - - asyncio.run(make_concurrent_requests()) - # Your AWS CRT operations here... - # Logs will now appear in Python's logging system - - # redirector.deactivate() - return 0 - except Exception as e: - print(f"Exception: {e}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) From 0ab3899bbc7f5343a2e91f64e8652df772445b39 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 Jun 2025 10:08:25 -0700 Subject: [PATCH 32/40] trivial --- awscrt/aio/aiohttp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/awscrt/aio/aiohttp.py b/awscrt/aio/aiohttp.py index d39207d4d..22459ceca 100644 --- a/awscrt/aio/aiohttp.py +++ b/awscrt/aio/aiohttp.py @@ -2,18 +2,14 @@ HTTP AsyncIO support This module provides asyncio wrappers around the awscrt.http module. -All network operations in `awscrt.http_asyncio` are asynchronous and use Python's asyncio framework. +All network operations in `awscrt.aio.aiohttp` are asynchronous and use Python's asyncio framework. """ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0. -import asyncio -from io import BytesIO import _awscrt -from concurrent.futures import Future import awscrt.exceptions -from typing import List, Tuple, Optional, Union, Callable, Any, AsyncIterator from awscrt.http import ( HttpClientConnectionBase, HttpRequest, HttpClientStreamBase, HttpProxyOptions, Http2Setting, HttpVersion @@ -21,7 +17,11 @@ from awscrt.io import ( ClientBootstrap, SocketOptions, TlsConnectionOptions, InputStream ) +import asyncio from collections import deque +from io import BytesIO +from concurrent.futures import Future +from typing import List, Tuple, Optional, Union, Callable, Any, AsyncIterator class AIOHttpClientConnectionUnified(HttpClientConnectionBase): From 7fdc1823abb6a1c8d5cf77022a76fa12c732ebc7 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 Jun 2025 16:04:45 -0700 Subject: [PATCH 33/40] remove unneeded forward declaration --- awscrt/aio/aiohttp.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/awscrt/aio/aiohttp.py b/awscrt/aio/aiohttp.py index 22459ceca..c0552b068 100644 --- a/awscrt/aio/aiohttp.py +++ b/awscrt/aio/aiohttp.py @@ -38,7 +38,7 @@ async def new(cls, bootstrap: Optional[ClientBootstrap] = None, socket_options: Optional[SocketOptions] = None, tls_connection_options: Optional[TlsConnectionOptions] = None, - proxy_options: Optional['HttpProxyOptions'] = None) -> "AIOHttpClientConnectionUnified": + proxy_options: Optional[HttpProxyOptions] = None) -> "AIOHttpClientConnectionUnified": """ Asynchronously establish a new AIOHttpClientConnectionUnified. @@ -118,7 +118,7 @@ async def new(cls, bootstrap: Optional[ClientBootstrap] = None, socket_options: Optional[SocketOptions] = None, tls_connection_options: Optional[TlsConnectionOptions] = None, - proxy_options: Optional['HttpProxyOptions'] = None) -> "AIOHttpClientConnection": + proxy_options: Optional[HttpProxyOptions] = None) -> "AIOHttpClientConnection": """ Asynchronously establish a new AIOHttpClientConnection. @@ -187,7 +187,7 @@ async def new(cls, bootstrap: Optional[ClientBootstrap] = None, socket_options: Optional[SocketOptions] = None, tls_connection_options: Optional[TlsConnectionOptions] = None, - proxy_options: Optional['HttpProxyOptions'] = None, + proxy_options: Optional[HttpProxyOptions] = None, initial_settings: Optional[List[Http2Setting]] = None, on_remote_settings_changed: Optional[Callable[[List[Http2Setting]], None]] = None) -> "AIOHttp2ClientConnection": @@ -322,6 +322,9 @@ def _set_completion(self, error_code: int) -> None: future = self._chunk_futures.popleft() future.set_result("") + async def _set_request_body_generator(self, body_iterator: AsyncIterator[bytes]): + ... + async def get_response_status_code(self) -> int: """Get the response status code asynchronously. @@ -362,9 +365,6 @@ async def wait_for_completion(self) -> int: """ return await self._completion_future - async def _set_request_body_generator(self, body_iterator: AsyncIterator[bytes]): - ... - class AIOHttpClientStream(AIOHttpClientStreamUnified): """Async HTTP stream that sends a request and receives a response. @@ -424,7 +424,7 @@ def on_write_complete(error_code: int) -> None: future.set_result(None) _awscrt.http2_client_stream_write_data(self, body_stream, end_stream, on_write_complete) - await asyncio.wrap_future(future) + await asyncio.wrap_future(future, self._loop) async def _set_request_body_generator(self, body_iterator: AsyncIterator[bytes]): try: From eff3d57cacde88b410668b1d2ee4459cc1abaae5 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 23 Jun 2025 16:08:42 -0700 Subject: [PATCH 34/40] keyword required --- awscrt/aio/aiohttp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awscrt/aio/aiohttp.py b/awscrt/aio/aiohttp.py index c0552b068..e68c7ba8c 100644 --- a/awscrt/aio/aiohttp.py +++ b/awscrt/aio/aiohttp.py @@ -424,7 +424,7 @@ def on_write_complete(error_code: int) -> None: future.set_result(None) _awscrt.http2_client_stream_write_data(self, body_stream, end_stream, on_write_complete) - await asyncio.wrap_future(future, self._loop) + await asyncio.wrap_future(future, loop=self._loop) async def _set_request_body_generator(self, body_iterator: AsyncIterator[bytes]): try: From 44e2640cada6d4457bd75bc8378478cc43e12efc Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Tue, 24 Jun 2025 09:42:54 -0700 Subject: [PATCH 35/40] use wrap_future for consistence --- awscrt/aio/aiohttp.py | 25 +++++++------------------ crt/aws-c-event-stream | 2 +- crt/aws-lc | 2 +- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/awscrt/aio/aiohttp.py b/awscrt/aio/aiohttp.py index e68c7ba8c..596a95c7a 100644 --- a/awscrt/aio/aiohttp.py +++ b/awscrt/aio/aiohttp.py @@ -21,7 +21,7 @@ from collections import deque from io import BytesIO from concurrent.futures import Future -from typing import List, Tuple, Optional, Union, Callable, Any, AsyncIterator +from typing import List, Tuple, Optional, Callable, AsyncIterator class AIOHttpClientConnectionUnified(HttpClientConnectionBase): @@ -273,9 +273,9 @@ def __init__(self, self._stream_completed = False # Create futures for async operations - self._completion_future = self._loop.create_future() - self._response_status_future = self._loop.create_future() - self._response_headers_future = self._loop.create_future() + self._completion_future = Future() + self._response_status_future = Future() + self._response_headers_future = Future() self._status_code = None self._request_body_generator = request_body_generator @@ -288,17 +288,10 @@ def __init__(self, def _on_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: self._status_code = status_code # invoked from the C thread, so we need to schedule the result setting on the event loop - self._loop.call_soon_threadsafe(self._set_response, status_code, name_value_pairs) - - def _set_response(self, status_code: int, name_value_pairs: List[Tuple[str, str]]) -> None: - """Set the response status and headers in the futures.""" self._response_status_future.set_result(status_code) self._response_headers_future.set_result(name_value_pairs) def _on_body(self, chunk: bytes) -> None: - self._loop.call_soon_threadsafe(self._set_body_chunk, chunk) - - def _set_body_chunk(self, chunk: bytes) -> None: """Process body chunk on the correct event loop thread.""" if self._chunk_futures: future = self._chunk_futures.popleft() @@ -307,10 +300,6 @@ def _set_body_chunk(self, chunk: bytes) -> None: self._received_chunks.append(chunk) def _on_complete(self, error_code: int) -> None: - # invoked from the C thread, so we need to schedule the result setting on the event loop - self._loop.call_soon_threadsafe(self._set_completion, error_code) - - def _set_completion(self, error_code: int) -> None: """Set the completion status of the stream.""" if error_code == 0: self._completion_future.set_result(self._status_code) @@ -331,7 +320,7 @@ async def get_response_status_code(self) -> int: Returns: int: The response status code. """ - return await self._response_status_future + return await asyncio.wrap_future(self._response_status_future, loop=self._loop) async def get_response_headers(self) -> List[Tuple[str, str]]: """Get the response headers asynchronously. @@ -339,7 +328,7 @@ async def get_response_headers(self) -> List[Tuple[str, str]]: Returns: List[Tuple[str, str]]: The response headers as a list of (name, value) tuples. """ - return await self._response_headers_future + return await asyncio.wrap_future(self._response_headers_future, loop=self._loop) async def get_next_response_chunk(self) -> bytes: """Get the next chunk from the response body. @@ -363,7 +352,7 @@ async def wait_for_completion(self) -> int: Returns: int: The response status code. """ - return await self._completion_future + return await asyncio.wrap_future(self._completion_future, loop=self._loop) class AIOHttpClientStream(AIOHttpClientStreamUnified): diff --git a/crt/aws-c-event-stream b/crt/aws-c-event-stream index 9312b0525..8f8f599e7 160000 --- a/crt/aws-c-event-stream +++ b/crt/aws-c-event-stream @@ -1 +1 @@ -Subproject commit 9312b052583183b98526aaeb91e5c72ec3db9627 +Subproject commit 8f8f599e78864188fe8547dafaa695a1d4855d6a diff --git a/crt/aws-lc b/crt/aws-lc index 0f76ff194..8b4e504c7 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit 0f76ff194bd410a45dd5d3cf75fc790033899b54 +Subproject commit 8b4e504c71fb129047e1b1e85fb5639154196884 From 7b73c8cb3d28fe5dd940e83738634039233b568d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 10 Sep 2025 14:30:37 -0700 Subject: [PATCH 36/40] renaming --- awscrt/__init__.py | 2 +- awscrt/aio/{aiohttp.py => http.py} | 2 +- test/test_aiohttp_client.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename awscrt/aio/{aiohttp.py => http.py} (99%) diff --git a/awscrt/__init__.py b/awscrt/__init__.py index c2a348827..ab3ee339f 100644 --- a/awscrt/__init__.py +++ b/awscrt/__init__.py @@ -7,7 +7,7 @@ 'auth', 'crypto', 'http', - 'aio.aiohttp', + 'aio.http', 'io', 'mqtt', 'mqtt5', diff --git a/awscrt/aio/aiohttp.py b/awscrt/aio/http.py similarity index 99% rename from awscrt/aio/aiohttp.py rename to awscrt/aio/http.py index 596a95c7a..4b0a33242 100644 --- a/awscrt/aio/aiohttp.py +++ b/awscrt/aio/http.py @@ -2,7 +2,7 @@ HTTP AsyncIO support This module provides asyncio wrappers around the awscrt.http module. -All network operations in `awscrt.aio.aiohttp` are asynchronous and use Python's asyncio framework. +All network operations in `awscrt.aio.http` are asynchronous and use Python's asyncio framework. """ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/test/test_aiohttp_client.py b/test/test_aiohttp_client.py index 2cffcf8aa..f15365d6b 100644 --- a/test/test_aiohttp_client.py +++ b/test/test_aiohttp_client.py @@ -18,7 +18,7 @@ from awscrt import io from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsContextOptions, TlsCipherPref from awscrt.http import HttpHeaders, HttpRequest, HttpVersion, Http2Setting, Http2SettingID -from awscrt.aio.aiohttp import AIOHttpClientConnection, AIOHttp2ClientConnection +from awscrt.aio.http import AIOHttpClientConnection, AIOHttp2ClientConnection import threading From 0c0a5213134fa01b39d8c7e90a4cb51aaf10c649 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 10 Sep 2025 14:50:16 -0700 Subject: [PATCH 37/40] why this was included --- awscrt/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awscrt/http.py b/awscrt/http.py index ff5c20b42..e8a9c2a73 100644 --- a/awscrt/http.py +++ b/awscrt/http.py @@ -938,7 +938,7 @@ def _on_connection_setup(self, binding: Any, error_code: int, http_version: Http return if self._asyncio_connection: # Import is done here to avoid circular import issues - from awscrt.aio.aiohttp import AIOHttpClientConnection, AIOHttp2ClientConnection + from awscrt.aio.http import AIOHttpClientConnection, AIOHttp2ClientConnection if http_version == HttpVersion.Http2: connection = AIOHttp2ClientConnection() else: From f28d1fc4fab9fbbacc983ee355eee296be8c8043 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 11 Sep 2025 16:55:21 -0700 Subject: [PATCH 38/40] update documentation --- awscrt/aio/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awscrt/aio/http.py b/awscrt/aio/http.py index 4b0a33242..d37bec401 100644 --- a/awscrt/aio/http.py +++ b/awscrt/aio/http.py @@ -28,7 +28,7 @@ class AIOHttpClientConnectionUnified(HttpClientConnectionBase): """ An async unified HTTP client connection for either a HTTP/1 or HTTP/2 connection. - Use `AIOHttpClientConnection.new()` to establish a new connection. + Use `AIOHttpClientConnectionUnified.new()` to establish a new connection. """ @classmethod @@ -61,7 +61,7 @@ async def new(cls, If None is provided then a proxy is not used. Returns: - AIOHttpClientConnection: A new HTTP client connection. + AIOHttpClientConnectionUnified: A new unified HTTP client connection. """ future = cls._generic_new( host_name, From 9841c240a434f69595a3818ca40dcad59436c5cb Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 18 Sep 2025 09:46:42 -0700 Subject: [PATCH 39/40] latest submodules --- crt/aws-c-auth | 2 +- crt/aws-c-cal | 2 +- crt/aws-c-common | 2 +- crt/aws-c-event-stream | 2 +- crt/aws-c-io | 2 +- crt/aws-c-s3 | 2 +- crt/aws-lc | 2 +- crt/s2n | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crt/aws-c-auth b/crt/aws-c-auth index cd9d6afcd..ab03bdd99 160000 --- a/crt/aws-c-auth +++ b/crt/aws-c-auth @@ -1 +1 @@ -Subproject commit cd9d6afcd42035d49bb2d0d3bef24b9faed57773 +Subproject commit ab03bdd996437d9097953ebb9495de71b6adc537 diff --git a/crt/aws-c-cal b/crt/aws-c-cal index 8703b3e59..cdd052bf0 160000 --- a/crt/aws-c-cal +++ b/crt/aws-c-cal @@ -1 +1 @@ -Subproject commit 8703b3e5930c9fd508025b268ab837fc9df3c4fd +Subproject commit cdd052bf0ac38d72177d6376ea668755fca13df4 diff --git a/crt/aws-c-common b/crt/aws-c-common index 2b67a658e..31578beb2 160000 --- a/crt/aws-c-common +++ b/crt/aws-c-common @@ -1 +1 @@ -Subproject commit 2b67a658e461520f1de20d64342b91ddcedc7ebb +Subproject commit 31578beb2309330fece3fb3a66035a568a2641e7 diff --git a/crt/aws-c-event-stream b/crt/aws-c-event-stream index 2a0f7c9fe..31a44ff91 160000 --- a/crt/aws-c-event-stream +++ b/crt/aws-c-event-stream @@ -1 +1 @@ -Subproject commit 2a0f7c9fe656c4789fa762a4ff04d06401abd282 +Subproject commit 31a44ff9108840a8f3fec54006218f4bc6c505e1 diff --git a/crt/aws-c-io b/crt/aws-c-io index 9c7f98dcb..db7a1bddc 160000 --- a/crt/aws-c-io +++ b/crt/aws-c-io @@ -1 +1 @@ -Subproject commit 9c7f98dcb083bd705eeb323e77868b1e2c9d4e73 +Subproject commit db7a1bddc9a29eca18734d0af189c3924775dcf1 diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 3afa5d08b..43d33d681 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 3afa5d08be95e82199a153e3abbe59bbb42638d7 +Subproject commit 43d33d681da4fed34b8ae1e6b98700ab08291628 diff --git a/crt/aws-lc b/crt/aws-lc index 04875dbbd..2294510cd 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit 04875dbbd6610a91855dcdc8edc268da289cb6d9 +Subproject commit 2294510cd0ecb2d5946461e3dbb038363b7b94cb diff --git a/crt/s2n b/crt/s2n index 418313c27..792d36671 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 418313c274d9cb72984dcd6e5e917740bc180664 +Subproject commit 792d36671f11d79c448519130c1b77f5540942fb From 1f8bf1a0a0971cfcb838db0fc3de060bd6cd7755 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 18 Sep 2025 09:51:01 -0700 Subject: [PATCH 40/40] include this for new aws-lc change --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 2a8f6f649..829c4f8b8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -44,6 +44,7 @@ graft crt/aws-lc/tests/compiler_features_tests prune crt/aws-lc/third_party graft crt/aws-lc/third_party/fiat graft crt/aws-lc/third_party/s2n-bignum +graft crt/aws-lc/third_party/jitterentropy prune crt/aws-lc/tool prune crt/aws-lc/util include crt/aws-lc/util/fipstools/CMakeLists.txt