diff --git a/.changes/next-release/enhancement-HTTP-34569.json b/.changes/next-release/enhancement-HTTP-34569.json new file mode 100644 index 000000000000..86e68003779b --- /dev/null +++ b/.changes/next-release/enhancement-HTTP-34569.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "HTTP", + "description": "Move 100-continue behavior to use `HTTPConnections` request interface." +} diff --git a/.changes/next-release/enhancement-urllib3-48038.json b/.changes/next-release/enhancement-urllib3-48038.json new file mode 100644 index 000000000000..8e3702e5ffe6 --- /dev/null +++ b/.changes/next-release/enhancement-urllib3-48038.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "urllib3", + "description": "Update urllib3 to version 2.6.3" +} diff --git a/awscli/botocore/awsrequest.py b/awscli/botocore/awsrequest.py index e06c89d2c548..f7a18a3d8ab6 100644 --- a/awscli/botocore/awsrequest.py +++ b/awscli/botocore/awsrequest.py @@ -66,34 +66,34 @@ class AWSConnection: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._original_response_cls = self.response_class - # We'd ideally hook into httplib's states, but they're all - # __mangled_vars so we use our own state var. This variable is set - # when we receive an early response from the server. If this value is - # set to True, any calls to send() are noops. This value is reset to - # false every time _send_request is called. This is to workaround the - # fact that py2.6 (and only py2.6) has a separate send() call for the - # body in _send_request, as opposed to endheaders(), which is where the - # body is sent in all versions > 2.6. + # This variable is set when we receive an early response from the + # server. If this value is set to True, any calls to send() are noops. + # This value is reset to false every time _send_request is called. + # This is to workaround changes in urllib3 2.0 which uses separate + # send() calls in request() instead of delegating to endheaders(), + # which is where the body is sent in CPython's HTTPConnection. self._response_received = False self._expect_header_set = False + self._send_called = False def close(self): super().close() # Reset all of our instance state we were tracking. self._response_received = False self._expect_header_set = False + self._send_called = False self.response_class = self._original_response_cls - def _send_request(self, method, url, body, headers, *args, **kwargs): + def request(self, method, url, body=None, headers=None, *args, **kwargs): + if headers is None: + headers = {} self._response_received = False if headers.get('Expect', b'') == b'100-continue': self._expect_header_set = True else: self._expect_header_set = False self.response_class = self._original_response_cls - rval = super()._send_request( - method, url, body, headers, *args, **kwargs - ) + rval = super().request(method, url, body, headers, *args, **kwargs) self._expect_header_set = False return rval @@ -210,10 +210,15 @@ def _send_message_body(self, message_body): def send(self, str): if self._response_received: - logger.debug( - "send() called, but reseponse already received. " - "Not sending data." - ) + if not self._send_called: + # urllib3 2.0 chunks and calls send potentially + # thousands of times inside `request` unlike the + # standard library. Only log this once for sanity. + logger.debug( + "send() called, but response already received. " + "Not sending data." + ) + self._send_called = True return return super().send(str) diff --git a/awscli/botocore/httpsession.py b/awscli/botocore/httpsession.py index 95d99ac9bc9f..dcddba5ae90f 100644 --- a/awscli/botocore/httpsession.py +++ b/awscli/botocore/httpsession.py @@ -4,6 +4,7 @@ import socket import sys from base64 import b64encode +from concurrent.futures import CancelledError from urllib3 import PoolManager, Timeout, proxy_from_url from urllib3.exceptions import ( @@ -17,6 +18,7 @@ ) from urllib3.exceptions import ReadTimeoutError as URLLib3ReadTimeoutError from urllib3.exceptions import SSLError as URLLib3SSLError +from urllib3.poolmanager import PoolKey from urllib3.util.retry import Retry from urllib3.util.ssl_ import ( OP_NO_COMPRESSION, @@ -28,8 +30,6 @@ ) from urllib3.util.url import parse_url -from concurrent.futures import CancelledError - try: from urllib3.util.ssl_ import OP_NO_TICKET, PROTOCOL_TLS_CLIENT except ImportError: @@ -75,6 +75,15 @@ DEFAULT_TIMEOUT = 60 MAX_POOL_CONNECTIONS = 10 DEFAULT_CA_BUNDLE = os.path.join(os.path.dirname(__file__), 'cacert.pem') +BUFFER_SIZE = None +if hasattr(PoolKey, 'key_blocksize'): + # urllib3 2.0 implemented its own chunking logic and set + # a default blocksize of 16KB. This creates a noticeable + # performance bottleneck when transferring objects + # larger than 100MB. Based on experiments, a blocksize + # of 128KB significantly improves throughput before + # getting diminishing returns. + BUFFER_SIZE = 1024 * 128 try: from certifi import where @@ -330,7 +339,6 @@ def _proxies_kwargs(self, **kwargs): def _get_pool_manager_kwargs(self, **extra_kwargs): pool_manager_kwargs = { - 'strict': True, 'timeout': self._timeout, 'maxsize': self._max_pool_connections, 'ssl_context': self._get_ssl_context(), @@ -338,6 +346,8 @@ def _get_pool_manager_kwargs(self, **extra_kwargs): 'cert_file': self._cert_file, 'key_file': self._key_file, } + if BUFFER_SIZE: + pool_manager_kwargs['blocksize'] = BUFFER_SIZE pool_manager_kwargs.update(**extra_kwargs) return pool_manager_kwargs diff --git a/hssyoo.sh b/hssyoo.sh new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/pyproject.toml b/pyproject.toml index 708e1345626a..9a3d2b277bd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "awscrt==0.29.1", "python-dateutil>=2.1,<=2.9.0", "jmespath>=0.7.1,<1.1.0", - "urllib3>=1.25.4,<1.27", + "urllib3>=1.25.4,<=2.6.3", ] dynamic = ["version"] diff --git a/requirements/download-deps/portable-exe-lock.txt b/requirements/download-deps/portable-exe-lock.txt index ab6ad0eac274..c70db7e10782 100644 --- a/requirements/download-deps/portable-exe-lock.txt +++ b/requirements/download-deps/portable-exe-lock.txt @@ -100,9 +100,9 @@ pyinstaller==6.11.1 \ --hash=sha256:ddc0fddd75f07f7e423da1f0822e389a42af011f9589e0269b87e0d89aa48c1f \ --hash=sha256:e21c7806e34f40181e7606926a14579f848bfb1dc52cbca7eea66eccccbfe977 # via -r requirements/portable-exe-extras.txt -pyinstaller-hooks-contrib==2025.10 \ - --hash=sha256:a1a737e5c0dccf1cf6f19a25e2efd109b9fec9ddd625f97f553dac16ee884881 \ - --hash=sha256:aa7a378518772846221f63a84d6306d9827299323243db890851474dfd1231a9 +pyinstaller-hooks-contrib==2025.11 \ + --hash=sha256:777e163e2942474aa41a8e6d31ac1635292d63422c3646c176d584d04d971c34 \ + --hash=sha256:dfe18632e06655fa88d218e0d768fd753e1886465c12a6d4bce04f1aaeec917d # via pyinstaller python-dateutil==2.9.0 \ --hash=sha256:78e73e19c63f5b20ffa567001531680d939dc042bf7850431877645523c66709 \ @@ -164,9 +164,9 @@ six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via python-dateutil -urllib3==1.26.20 \ - --hash=sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e \ - --hash=sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32 +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via awscli (pyproject.toml) wcwidth==0.2.14 \ --hash=sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605 \ diff --git a/requirements/download-deps/portable-exe-win-lock.txt b/requirements/download-deps/portable-exe-win-lock.txt index ce4254839cbf..4ad0f9d035a6 100644 --- a/requirements/download-deps/portable-exe-win-lock.txt +++ b/requirements/download-deps/portable-exe-win-lock.txt @@ -98,9 +98,9 @@ pyinstaller==6.11.1 \ --hash=sha256:ddc0fddd75f07f7e423da1f0822e389a42af011f9589e0269b87e0d89aa48c1f \ --hash=sha256:e21c7806e34f40181e7606926a14579f848bfb1dc52cbca7eea66eccccbfe977 # via -r D:/a/aws-cli/aws-cli/requirements/portable-exe-extras.txt -pyinstaller-hooks-contrib==2025.10 \ - --hash=sha256:a1a737e5c0dccf1cf6f19a25e2efd109b9fec9ddd625f97f553dac16ee884881 \ - --hash=sha256:aa7a378518772846221f63a84d6306d9827299323243db890851474dfd1231a9 +pyinstaller-hooks-contrib==2025.11 \ + --hash=sha256:777e163e2942474aa41a8e6d31ac1635292d63422c3646c176d584d04d971c34 \ + --hash=sha256:dfe18632e06655fa88d218e0d768fd753e1886465c12a6d4bce04f1aaeec917d # via pyinstaller python-dateutil==2.9.0 \ --hash=sha256:78e73e19c63f5b20ffa567001531680d939dc042bf7850431877645523c66709 \ @@ -166,9 +166,9 @@ six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via python-dateutil -urllib3==1.26.20 \ - --hash=sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e \ - --hash=sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32 +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via awscli (D:/a/aws-cli/aws-cli/pyproject.toml) wcwidth==0.2.14 \ --hash=sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605 \ diff --git a/requirements/download-deps/system-sandbox-lock.txt b/requirements/download-deps/system-sandbox-lock.txt index 547e9266d2ff..60187b1d1ece 100644 --- a/requirements/download-deps/system-sandbox-lock.txt +++ b/requirements/download-deps/system-sandbox-lock.txt @@ -126,9 +126,9 @@ six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via python-dateutil -urllib3==1.26.20 \ - --hash=sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e \ - --hash=sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32 +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via awscli (pyproject.toml) wcwidth==0.2.14 \ --hash=sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605 \ diff --git a/requirements/download-deps/system-sandbox-win-lock.txt b/requirements/download-deps/system-sandbox-win-lock.txt index 2825e6a46773..0fa8278526cb 100644 --- a/requirements/download-deps/system-sandbox-win-lock.txt +++ b/requirements/download-deps/system-sandbox-win-lock.txt @@ -126,9 +126,9 @@ six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via python-dateutil -urllib3==1.26.20 \ - --hash=sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e \ - --hash=sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32 +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via awscli (D:/a/aws-cli/aws-cli/pyproject.toml) wcwidth==0.2.14 \ --hash=sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605 \ diff --git a/tests/unit/botocore/test_http_session.py b/tests/unit/botocore/test_http_session.py index e2202e8ad95a..1165f3fdcc53 100644 --- a/tests/unit/botocore/test_http_session.py +++ b/tests/unit/botocore/test_http_session.py @@ -1,4 +1,5 @@ import socket +from concurrent.futures import CancelledError import pytest from botocore.awsrequest import ( @@ -12,13 +13,13 @@ ProxyConnectionError, ) from botocore.httpsession import ( + BUFFER_SIZE, ProxyConfiguration, URLLib3Session, get_cert_path, mask_proxy_url, ) from urllib3.exceptions import NewConnectionError, ProtocolError, ProxyError -from concurrent.futures import CancelledError from tests import mock, unittest @@ -149,7 +150,6 @@ def assert_request_sent( def _assert_manager_call(self, manager, *assert_args, **assert_kwargs): call_kwargs = { - 'strict': True, 'maxsize': mock.ANY, 'timeout': mock.ANY, 'ssl_context': mock.ANY, @@ -157,6 +157,8 @@ def _assert_manager_call(self, manager, *assert_args, **assert_kwargs): 'cert_file': None, 'key_file': None, } + if BUFFER_SIZE: + call_kwargs['blocksize'] = BUFFER_SIZE call_kwargs.update(assert_kwargs) manager.assert_called_with(*assert_args, **call_kwargs)