diff --git a/CHANGES b/CHANGES index 66bc5f17..10a613f9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,17 @@ 0.26.3 ------ +* Fixed ``_recorder.record`` capturing a real response's ``Content-Encoding`` + header (e.g. ``gzip``) alongside the already-decompressed body that + ``requests`` exposes via ``response.text``. The mismatched header caused + ``requests``/``urllib3`` to raise ``ContentDecodingError`` when the + recorded fixture was later replayed. The recorder now drops + ``Content-Encoding`` at capture time, so both the in-memory recorded + response and any file dumped from it are consistent. Genuinely + gzip-compressed mocked responses (real compressed bytes paired with a + matching ``Content-Encoding`` header, registered directly via ``add()``) + are unaffected and continue to be decoded normally by ``requests``. See + #724 * Fixed the element type exposed by `CallList` so static type checkers infer `Call` values when iterating, indexing, or filtering recorded calls. See #722 * Fixed `query_string_matcher` (and the query matching auto-applied to a diff --git a/responses/_recorder.py b/responses/_recorder.py index e0fac897..84949ede 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -38,6 +38,28 @@ def _remove_nones(d: "Any") -> "Any": return d +def _strip_content_encoding(headers: "Dict[str, str]") -> "Dict[str, str]": + """Drop a "Content-Encoding" header (case-insensitively) from a headers dict. + + ``requests`` transparently decompresses the response body before it is + exposed via ``response.text``, so by the time the recorder captures a + response, the body is already plain/uncompressed. A "Content-Encoding" + header copied verbatim from the real response would therefore no longer + match the (decompressed) body, and would cause any client honoring that + header -- such as ``requests``/``urllib3`` during replay -- to attempt to + decompress already-uncompressed content and fail. See #724. + + HTTP header names are case-insensitive, and HTTP/2 servers send them + lowercase, so match without regard to case (consistent with + ``_remove_default_headers`` below). + """ + return { + key: value + for key, value in headers.items() + if key.lower() != "content-encoding" + } + + def _remove_default_headers(data: "Any") -> "Any": """ It would be too verbose to store these headers in the file generated by the @@ -152,6 +174,10 @@ def _on_request( headers_values = { key: value for key, value in requests_response.headers.items() } + # ``requests_response.text`` below is already decompressed by + # ``requests``, so a captured "Content-Encoding" header would no + # longer describe the body we are about to store. See #724. + headers_values = _strip_content_encoding(headers_values) responses_response = Response( method=str(request.method), url=str(requests_response.request.url), diff --git a/responses/tests/test_recorder.py b/responses/tests/test_recorder.py index b761dc80..ccafccbb 100644 --- a/responses/tests/test_recorder.py +++ b/responses/tests/test_recorder.py @@ -1,3 +1,4 @@ +import gzip from pathlib import Path import pytest @@ -220,6 +221,69 @@ def run(): _recorder.recorder.reset() assert not _recorder.recorder.get_registry().registered + def test_recorder_strips_content_encoding(self, httpserver): + """A real gzip-encoded response should not retain a misleading + "Content-Encoding" header once recorded. + + ``requests`` transparently decompresses the response body before + the recorder captures it via ``response.text``, so keeping the + original "Content-Encoding: gzip" header alongside that already + decompressed body would make the recorded response internally + inconsistent: replaying it would cause ``requests``/``urllib3`` to + attempt to decompress content that is no longer compressed. + + This checks the in-memory registered response *before* it is + dumped to a file: ``_remove_default_headers`` already strips + "Content-Encoding" at dump time (see + ``test_remove_default_headers_is_case_insensitive`` above), so + only inspecting the dumped file would not prove that the recorder + itself -- ``Recorder._on_request`` -- avoids capturing the stale + header in the first place (which matters e.g. for the REPL usage + exercised by ``test_use_recorder_without_decorator``, where a file + may never be written at all). + + See https://github.com/getsentry/responses/issues/724 + """ + original_body = b'{"first_name": true}' + httpserver.expect_request("/gzipped").respond_with_data( + gzip.compress(original_body), + status=200, + content_type="application/json", + headers={"Content-Encoding": "gzip"}, + ) + url = httpserver.url_for("/gzipped") + + _recorder.recorder.start() + requests.get(url) + _recorder.recorder.stop() + + # The in-memory recorded response must not retain the header... + [recorded] = _recorder.recorder.get_registry().registered + assert recorded.headers is not None + assert "Content-Encoding" not in recorded.headers + assert "content-encoding" not in recorded.headers + assert recorded.body == original_body.decode() + + # ...and neither should the file dumped from it. + _recorder.recorder.dump_to_file(self.out_file) + with open(self.out_file) as file: + data = yaml.safe_load(file) + dumped_headers = data["responses"][0]["response"].get("headers", {}) + assert "Content-Encoding" not in dumped_headers + assert "content-encoding" not in dumped_headers + + _recorder.recorder.reset() + + # Replaying the recorded fixture must not raise ContentDecodingError. + @responses.activate + def replay(): + responses._add_from_file(file_path=self.out_file) + resp = requests.get(url) + assert resp.status_code == 200 + assert resp.content == original_body + + replay() + class TestReplay: def setup_method(self): diff --git a/responses/tests/test_responses.py b/responses/tests/test_responses.py index b3740ed9..5a055711 100644 --- a/responses/tests/test_responses.py +++ b/responses/tests/test_responses.py @@ -1,3 +1,4 @@ +import gzip import inspect import os import re @@ -1441,6 +1442,48 @@ def run(): assert_reset() +def test_genuinely_gzip_encoded_response_is_decoded(): + """A mocked response CAN legitimately be gzip-encoded: if ``body`` is + real compressed bytes and a matching ``Content-Encoding: gzip`` header + is registered, ``requests``/``urllib3`` should transparently decompress + it on read, exactly as they would for a real server response. + + This is the counterpart to the recorder-side fix for + https://github.com/getsentry/responses/issues/724: that issue was + caused by the recorder pairing a *decompressed* body with a leftover + ``Content-Encoding`` header, not by ``responses`` mishandling genuinely + encoded bodies. ``get_headers()``/response serving must not strip a + ``Content-Encoding`` header unconditionally, or this legitimate case + breaks. + """ + original_body = b'{"first_name": true}' + compressed_body = gzip.compress(original_body) + + @responses.activate + def run(): + responses.add( + responses.GET, + "https://example.org/", + body=compressed_body, + status=200, + content_type="application/json", + headers={"Content-Encoding": "gzip", "x-request-id": "abc123"}, + ) + + resp = requests.get("https://example.org/") + + assert resp.status_code == 200 + # urllib3/requests should decode the real gzip bytes back to the + # original, uncompressed content. + assert resp.content == original_body + assert resp.json() == {"first_name": True} + # Unrelated headers must be preserved. + assert resp.headers["x-request-id"] == "abc123" + + run() + assert_reset() + + def test_content_length_error(monkeypatch): """ Currently 'requests' does not enforce content length validation,