From e5941a630729fba4bc83974ec8d2fba26779c321 Mon Sep 17 00:00:00 2001 From: Contributor Date: Thu, 13 Aug 2026 19:16:22 +0000 Subject: [PATCH 1/2] fix: strip misleading Content-Encoding header from mocked responses responses always serves `body` as literal, uncompressed bytes -- it never actually applies the encoding named in a Content-Encoding header. When such a header was present on a registered response (e.g. copied verbatim from a real recorded response into a hand-edited/legacy fixture file, or passed directly via headers=), requests/urllib3 would try to decompress the already-uncompressed body and raise ContentDecodingError/DecodeError instead of returning the mocked response. Strip the header in BaseResponse.get_headers(), the single choke point used by both Response and CallbackResponse, so every path that constructs a mocked response (add(), add_callback(), and _add_from_file()) is protected, not just freshly recorded and re-dumped fixture files. Adds two regression tests: one for the direct responses.add() usage and one for the _add_from_file() replay path used by the recorder documentation, both of which reproduce the exact failure from the reported issue before this fix and pass after it. Fixes #724 --- CHANGES | 7 +++ responses/__init__.py | 15 ++++++ responses/tests/test_responses.py | 76 +++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) diff --git a/CHANGES b/CHANGES index 66bc5f17..ad24935f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,13 @@ 0.26.3 ------ +* Fixed a mocked response with a ``Content-Encoding`` header (e.g. + ``gzip``) and a plain, uncompressed ``body`` causing ``requests``/ + ``urllib3`` to raise ``ContentDecodingError`` when reading the response. + ``responses`` never actually compresses ``body``, so the header is now + stripped when the response is served, whether registered directly via + ``add()``/``headers=`` or replayed from a fixture file via + ``_add_from_file()``. See #724 * Fixed the element type exposed by `CallList` so static type checkers infer `Call` values when iterating, indexing, or filtering recorded calls. See #722 * Fixed `query_string_matcher` (and the query matching auto-applied to a diff --git a/responses/__init__.py b/responses/__init__.py index 41f5fde8..48ec0254 100644 --- a/responses/__init__.py +++ b/responses/__init__.py @@ -503,6 +503,21 @@ def get_headers(self) -> HTTPHeaderDict: if self.headers: headers.extend(self.headers) + # ``responses`` always serves ``body`` as literal, uncompressed bytes: + # it never actually applies the encoding named in a "Content-Encoding" + # header. If such a header is present (e.g. copied over verbatim from + # a real recorded response, or from a hand-written/legacy fixture + # file produced by an older version of ``responses``), HTTP clients + # that honor the header -- such as ``requests``/``urllib3`` -- will + # try to decompress the already-uncompressed body and raise a + # confusing ``ContentDecodingError``/``DecodeError``. Since the + # header would always be a lie in that scenario, strip it here so + # the mocked response is internally consistent regardless of how it + # was registered (``add()``, ``_add_from_file()``, or the recorder's + # in-memory registry). + if "Content-Encoding" in headers: + del headers["Content-Encoding"] + return headers def get_response(self, request: "PreparedRequest") -> HTTPResponse: diff --git a/responses/tests/test_responses.py b/responses/tests/test_responses.py index b3740ed9..bb07b663 100644 --- a/responses/tests/test_responses.py +++ b/responses/tests/test_responses.py @@ -1441,6 +1441,82 @@ def run(): assert_reset() +def test_content_encoding_header_is_stripped(): + """Test that a stale/misleading 'Content-Encoding' header does not + cause 'requests' to fail trying to decompress an uncompressed body. + + ``responses`` always serves ``body`` verbatim -- it never actually + compresses it -- so a "Content-Encoding: gzip" header (e.g. copied over + from a real recorded response, or present in a hand-written/legacy + fixture file) is always inaccurate and causes ``requests``/``urllib3`` + to raise ``ContentDecodingError`` when they try to honor it. + + For more details see https://github.com/getsentry/responses/issues/724 + """ + + @responses.activate + def run(): + responses.add( + responses.GET, + "https://example.org/", + body='{"first_name": true}', + status=200, + content_type="application/json", + headers={"content-encoding": "gzip", "x-request-id": "abc123"}, + ) + + resp = requests.get("https://example.org/") + + assert resp.status_code == 200 + # The body should come through untouched and be readable without + # raising a ContentDecodingError. + assert resp.json() == {"first_name": True} + assert "Content-Encoding" not in resp.headers + # Unrelated headers must be preserved. + assert resp.headers["x-request-id"] == "abc123" + + run() + assert_reset() + + +def test_content_encoding_header_is_stripped_when_loaded_from_file(tmp_path): + """Same as ``test_content_encoding_header_is_stripped`` but exercising + the ``_add_from_file`` replay path, which is how fixtures recorded by + older versions of ``responses`` (or hand-crafted/imported files) are + typically loaded. + + For more details see https://github.com/getsentry/responses/issues/724 + """ + fixture = tmp_path / "recorded.yaml" + fixture.write_text( + """\ +responses: +- response: + auto_calculate_content_length: false + body: '{"first_name": true}' + content_type: application/json + headers: + content-encoding: gzip + method: GET + status: 200 + url: https://example.org/lookup +""" + ) + + @responses.activate + def run(): + responses._add_from_file(file_path=str(fixture)) + + resp = requests.get("https://example.org/lookup") + + assert resp.status_code == 200 + assert resp.json() == {"first_name": True} + assert "Content-Encoding" not in resp.headers + + run() + assert_reset() + + def test_content_length_error(monkeypatch): """ Currently 'requests' does not enforce content length validation, From 224d7821ead6a97d86158e80d50b26042636b4db Mon Sep 17 00:00:00 2001 From: Sohel2309 Date: Sun, 23 Aug 2026 11:35:57 +0530 Subject: [PATCH 2/2] fix: strip stale Content-Encoding when recording, not when serving --- CHANGES | 18 +++++--- responses/__init__.py | 15 ------- responses/_recorder.py | 26 +++++++++++ responses/tests/test_recorder.py | 64 +++++++++++++++++++++++++++ responses/tests/test_responses.py | 73 +++++++++---------------------- 5 files changed, 121 insertions(+), 75 deletions(-) diff --git a/CHANGES b/CHANGES index ad24935f..10a613f9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,13 +1,17 @@ 0.26.3 ------ -* Fixed a mocked response with a ``Content-Encoding`` header (e.g. - ``gzip``) and a plain, uncompressed ``body`` causing ``requests``/ - ``urllib3`` to raise ``ContentDecodingError`` when reading the response. - ``responses`` never actually compresses ``body``, so the header is now - stripped when the response is served, whether registered directly via - ``add()``/``headers=`` or replayed from a fixture file via - ``_add_from_file()``. See #724 +* Fixed ``_recorder.record`` capturing a real response's ``Content-Encoding`` + header (e.g. ``gzip``) alongside the already-decompressed body that + ``requests`` exposes via ``response.text``. The mismatched header caused + ``requests``/``urllib3`` to raise ``ContentDecodingError`` when the + recorded fixture was later replayed. The recorder now drops + ``Content-Encoding`` at capture time, so both the in-memory recorded + response and any file dumped from it are consistent. Genuinely + gzip-compressed mocked responses (real compressed bytes paired with a + matching ``Content-Encoding`` header, registered directly via ``add()``) + are unaffected and continue to be decoded normally by ``requests``. See + #724 * Fixed the element type exposed by `CallList` so static type checkers infer `Call` values when iterating, indexing, or filtering recorded calls. See #722 * Fixed `query_string_matcher` (and the query matching auto-applied to a diff --git a/responses/__init__.py b/responses/__init__.py index 48ec0254..41f5fde8 100644 --- a/responses/__init__.py +++ b/responses/__init__.py @@ -503,21 +503,6 @@ def get_headers(self) -> HTTPHeaderDict: if self.headers: headers.extend(self.headers) - # ``responses`` always serves ``body`` as literal, uncompressed bytes: - # it never actually applies the encoding named in a "Content-Encoding" - # header. If such a header is present (e.g. copied over verbatim from - # a real recorded response, or from a hand-written/legacy fixture - # file produced by an older version of ``responses``), HTTP clients - # that honor the header -- such as ``requests``/``urllib3`` -- will - # try to decompress the already-uncompressed body and raise a - # confusing ``ContentDecodingError``/``DecodeError``. Since the - # header would always be a lie in that scenario, strip it here so - # the mocked response is internally consistent regardless of how it - # was registered (``add()``, ``_add_from_file()``, or the recorder's - # in-memory registry). - if "Content-Encoding" in headers: - del headers["Content-Encoding"] - return headers def get_response(self, request: "PreparedRequest") -> HTTPResponse: diff --git a/responses/_recorder.py b/responses/_recorder.py index e0fac897..84949ede 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -38,6 +38,28 @@ def _remove_nones(d: "Any") -> "Any": return d +def _strip_content_encoding(headers: "Dict[str, str]") -> "Dict[str, str]": + """Drop a "Content-Encoding" header (case-insensitively) from a headers dict. + + ``requests`` transparently decompresses the response body before it is + exposed via ``response.text``, so by the time the recorder captures a + response, the body is already plain/uncompressed. A "Content-Encoding" + header copied verbatim from the real response would therefore no longer + match the (decompressed) body, and would cause any client honoring that + header -- such as ``requests``/``urllib3`` during replay -- to attempt to + decompress already-uncompressed content and fail. See #724. + + HTTP header names are case-insensitive, and HTTP/2 servers send them + lowercase, so match without regard to case (consistent with + ``_remove_default_headers`` below). + """ + return { + key: value + for key, value in headers.items() + if key.lower() != "content-encoding" + } + + def _remove_default_headers(data: "Any") -> "Any": """ It would be too verbose to store these headers in the file generated by the @@ -152,6 +174,10 @@ def _on_request( headers_values = { key: value for key, value in requests_response.headers.items() } + # ``requests_response.text`` below is already decompressed by + # ``requests``, so a captured "Content-Encoding" header would no + # longer describe the body we are about to store. See #724. + headers_values = _strip_content_encoding(headers_values) responses_response = Response( method=str(request.method), url=str(requests_response.request.url), diff --git a/responses/tests/test_recorder.py b/responses/tests/test_recorder.py index b761dc80..ccafccbb 100644 --- a/responses/tests/test_recorder.py +++ b/responses/tests/test_recorder.py @@ -1,3 +1,4 @@ +import gzip from pathlib import Path import pytest @@ -220,6 +221,69 @@ def run(): _recorder.recorder.reset() assert not _recorder.recorder.get_registry().registered + def test_recorder_strips_content_encoding(self, httpserver): + """A real gzip-encoded response should not retain a misleading + "Content-Encoding" header once recorded. + + ``requests`` transparently decompresses the response body before + the recorder captures it via ``response.text``, so keeping the + original "Content-Encoding: gzip" header alongside that already + decompressed body would make the recorded response internally + inconsistent: replaying it would cause ``requests``/``urllib3`` to + attempt to decompress content that is no longer compressed. + + This checks the in-memory registered response *before* it is + dumped to a file: ``_remove_default_headers`` already strips + "Content-Encoding" at dump time (see + ``test_remove_default_headers_is_case_insensitive`` above), so + only inspecting the dumped file would not prove that the recorder + itself -- ``Recorder._on_request`` -- avoids capturing the stale + header in the first place (which matters e.g. for the REPL usage + exercised by ``test_use_recorder_without_decorator``, where a file + may never be written at all). + + See https://github.com/getsentry/responses/issues/724 + """ + original_body = b'{"first_name": true}' + httpserver.expect_request("/gzipped").respond_with_data( + gzip.compress(original_body), + status=200, + content_type="application/json", + headers={"Content-Encoding": "gzip"}, + ) + url = httpserver.url_for("/gzipped") + + _recorder.recorder.start() + requests.get(url) + _recorder.recorder.stop() + + # The in-memory recorded response must not retain the header... + [recorded] = _recorder.recorder.get_registry().registered + assert recorded.headers is not None + assert "Content-Encoding" not in recorded.headers + assert "content-encoding" not in recorded.headers + assert recorded.body == original_body.decode() + + # ...and neither should the file dumped from it. + _recorder.recorder.dump_to_file(self.out_file) + with open(self.out_file) as file: + data = yaml.safe_load(file) + dumped_headers = data["responses"][0]["response"].get("headers", {}) + assert "Content-Encoding" not in dumped_headers + assert "content-encoding" not in dumped_headers + + _recorder.recorder.reset() + + # Replaying the recorded fixture must not raise ContentDecodingError. + @responses.activate + def replay(): + responses._add_from_file(file_path=self.out_file) + resp = requests.get(url) + assert resp.status_code == 200 + assert resp.content == original_body + + replay() + class TestReplay: def setup_method(self): diff --git a/responses/tests/test_responses.py b/responses/tests/test_responses.py index bb07b663..5a055711 100644 --- a/responses/tests/test_responses.py +++ b/responses/tests/test_responses.py @@ -1,3 +1,4 @@ +import gzip import inspect import os import re @@ -1441,37 +1442,41 @@ def run(): assert_reset() -def test_content_encoding_header_is_stripped(): - """Test that a stale/misleading 'Content-Encoding' header does not - cause 'requests' to fail trying to decompress an uncompressed body. +def test_genuinely_gzip_encoded_response_is_decoded(): + """A mocked response CAN legitimately be gzip-encoded: if ``body`` is + real compressed bytes and a matching ``Content-Encoding: gzip`` header + is registered, ``requests``/``urllib3`` should transparently decompress + it on read, exactly as they would for a real server response. - ``responses`` always serves ``body`` verbatim -- it never actually - compresses it -- so a "Content-Encoding: gzip" header (e.g. copied over - from a real recorded response, or present in a hand-written/legacy - fixture file) is always inaccurate and causes ``requests``/``urllib3`` - to raise ``ContentDecodingError`` when they try to honor it. - - For more details see https://github.com/getsentry/responses/issues/724 + This is the counterpart to the recorder-side fix for + https://github.com/getsentry/responses/issues/724: that issue was + caused by the recorder pairing a *decompressed* body with a leftover + ``Content-Encoding`` header, not by ``responses`` mishandling genuinely + encoded bodies. ``get_headers()``/response serving must not strip a + ``Content-Encoding`` header unconditionally, or this legitimate case + breaks. """ + original_body = b'{"first_name": true}' + compressed_body = gzip.compress(original_body) @responses.activate def run(): responses.add( responses.GET, "https://example.org/", - body='{"first_name": true}', + body=compressed_body, status=200, content_type="application/json", - headers={"content-encoding": "gzip", "x-request-id": "abc123"}, + headers={"Content-Encoding": "gzip", "x-request-id": "abc123"}, ) resp = requests.get("https://example.org/") assert resp.status_code == 200 - # The body should come through untouched and be readable without - # raising a ContentDecodingError. + # urllib3/requests should decode the real gzip bytes back to the + # original, uncompressed content. + assert resp.content == original_body assert resp.json() == {"first_name": True} - assert "Content-Encoding" not in resp.headers # Unrelated headers must be preserved. assert resp.headers["x-request-id"] == "abc123" @@ -1479,44 +1484,6 @@ def run(): assert_reset() -def test_content_encoding_header_is_stripped_when_loaded_from_file(tmp_path): - """Same as ``test_content_encoding_header_is_stripped`` but exercising - the ``_add_from_file`` replay path, which is how fixtures recorded by - older versions of ``responses`` (or hand-crafted/imported files) are - typically loaded. - - For more details see https://github.com/getsentry/responses/issues/724 - """ - fixture = tmp_path / "recorded.yaml" - fixture.write_text( - """\ -responses: -- response: - auto_calculate_content_length: false - body: '{"first_name": true}' - content_type: application/json - headers: - content-encoding: gzip - method: GET - status: 200 - url: https://example.org/lookup -""" - ) - - @responses.activate - def run(): - responses._add_from_file(file_path=str(fixture)) - - resp = requests.get("https://example.org/lookup") - - assert resp.status_code == 200 - assert resp.json() == {"first_name": True} - assert "Content-Encoding" not in resp.headers - - run() - assert_reset() - - def test_content_length_error(monkeypatch): """ Currently 'requests' does not enforce content length validation,