fix: strip misleading Content-Encoding header from mocked responses

CHANGES

@@ -1,6 +1,17 @@
 0.26.3
 ------
 
+* Fixed ``_recorder.record`` capturing a real response's ``Content-Encoding``
+  header (e.g. ``gzip``) alongside the already-decompressed body that
+  ``requests`` exposes via ``response.text``. The mismatched header caused
+  ``requests``/``urllib3`` to raise ``ContentDecodingError`` when the
+  recorded fixture was later replayed. The recorder now drops
+  ``Content-Encoding`` at capture time, so both the in-memory recorded
+  response and any file dumped from it are consistent. Genuinely
+  gzip-compressed mocked responses (real compressed bytes paired with a
+  matching ``Content-Encoding`` header, registered directly via ``add()``)
+  are unaffected and continue to be decoded normally by ``requests``. See
+  #724
 * Fixed the element type exposed by `CallList` so static type checkers infer
   `Call` values when iterating, indexing, or filtering recorded calls. See #722
 * Fixed `query_string_matcher` (and the query matching auto-applied to a

responses/_recorder.py

@@ -38,6 +38,28 @@ def _remove_nones(d: "Any") -> "Any":
     return d
 
 
+def _strip_content_encoding(headers: "Dict[str, str]") -> "Dict[str, str]":
+    """Drop a "Content-Encoding" header (case-insensitively) from a headers dict.
+
+    ``requests`` transparently decompresses the response body before it is
+    exposed via ``response.text``, so by the time the recorder captures a
+    response, the body is already plain/uncompressed. A "Content-Encoding"
+    header copied verbatim from the real response would therefore no longer
+    match the (decompressed) body, and would cause any client honoring that
+    header -- such as ``requests``/``urllib3`` during replay -- to attempt to
+    decompress already-uncompressed content and fail. See #724.
+
+    HTTP header names are case-insensitive, and HTTP/2 servers send them
+    lowercase, so match without regard to case (consistent with
+    ``_remove_default_headers`` below).
+    """
+    return {
+        key: value
+        for key, value in headers.items()
+        if key.lower() != "content-encoding"
+    }
+
+
 def _remove_default_headers(data: "Any") -> "Any":
     """
     It would be too verbose to store these headers in the file generated by the
@@ -152,6 +174,10 @@ def _on_request(
         headers_values = {
             key: value for key, value in requests_response.headers.items()
         }
+        # ``requests_response.text`` below is already decompressed by
+        # ``requests``, so a captured "Content-Encoding" header would no
+        # longer describe the body we are about to store. See #724.
+        headers_values = _strip_content_encoding(headers_values)
         responses_response = Response(
             method=str(request.method),
             url=str(requests_response.request.url),

responses/tests/test_recorder.py

@@ -1,3 +1,4 @@
+import gzip
 from pathlib import Path
 
 import pytest
@@ -220,6 +221,69 @@ def run():
         _recorder.recorder.reset()
         assert not _recorder.recorder.get_registry().registered
 
+    def test_recorder_strips_content_encoding(self, httpserver):
+        """A real gzip-encoded response should not retain a misleading
+        "Content-Encoding" header once recorded.
+
+        ``requests`` transparently decompresses the response body before
+        the recorder captures it via ``response.text``, so keeping the
+        original "Content-Encoding: gzip" header alongside that already
+        decompressed body would make the recorded response internally
+        inconsistent: replaying it would cause ``requests``/``urllib3`` to
+        attempt to decompress content that is no longer compressed.
+
+        This checks the in-memory registered response *before* it is
+        dumped to a file: ``_remove_default_headers`` already strips
+        "Content-Encoding" at dump time (see
+        ``test_remove_default_headers_is_case_insensitive`` above), so
+        only inspecting the dumped file would not prove that the recorder
+        itself -- ``Recorder._on_request`` -- avoids capturing the stale
+        header in the first place (which matters e.g. for the REPL usage
+        exercised by ``test_use_recorder_without_decorator``, where a file
+        may never be written at all).
+
+        See https://github.com/getsentry/responses/issues/724
+        """
+        original_body = b'{"first_name": true}'
+        httpserver.expect_request("/gzipped").respond_with_data(
+            gzip.compress(original_body),
+            status=200,
+            content_type="application/json",
+            headers={"Content-Encoding": "gzip"},
+        )
+        url = httpserver.url_for("/gzipped")
+
+        _recorder.recorder.start()
+        requests.get(url)
+        _recorder.recorder.stop()
+
+        # The in-memory recorded response must not retain the header...
+        [recorded] = _recorder.recorder.get_registry().registered
+        assert recorded.headers is not None
+        assert "Content-Encoding" not in recorded.headers
+        assert "content-encoding" not in recorded.headers
+        assert recorded.body == original_body.decode()
+
+        # ...and neither should the file dumped from it.
+        _recorder.recorder.dump_to_file(self.out_file)
+        with open(self.out_file) as file:
+            data = yaml.safe_load(file)
+        dumped_headers = data["responses"][0]["response"].get("headers", {})
+        assert "Content-Encoding" not in dumped_headers
+        assert "content-encoding" not in dumped_headers
+
+        _recorder.recorder.reset()
+
+        # Replaying the recorded fixture must not raise ContentDecodingError.
+        @responses.activate
+        def replay():
+            responses._add_from_file(file_path=self.out_file)
+            resp = requests.get(url)
+            assert resp.status_code == 200
+            assert resp.content == original_body
+
+        replay()
+
 
 class TestReplay:
     def setup_method(self):

responses/tests/test_responses.py

@@ -1,3 +1,4 @@
+import gzip
 import inspect
 import os
 import re
@@ -1441,6 +1442,48 @@ def run():
     assert_reset()
 
 
+def test_genuinely_gzip_encoded_response_is_decoded():
+    """A mocked response CAN legitimately be gzip-encoded: if ``body`` is
+    real compressed bytes and a matching ``Content-Encoding: gzip`` header
+    is registered, ``requests``/``urllib3`` should transparently decompress
+    it on read, exactly as they would for a real server response.
+
+    This is the counterpart to the recorder-side fix for
+    https://github.com/getsentry/responses/issues/724: that issue was
+    caused by the recorder pairing a *decompressed* body with a leftover
+    ``Content-Encoding`` header, not by ``responses`` mishandling genuinely
+    encoded bodies. ``get_headers()``/response serving must not strip a
+    ``Content-Encoding`` header unconditionally, or this legitimate case
+    breaks.
+    """
+    original_body = b'{"first_name": true}'
+    compressed_body = gzip.compress(original_body)
+
+    @responses.activate
+    def run():
+        responses.add(
+            responses.GET,
+            "https://example.org/",
+            body=compressed_body,
+            status=200,
+            content_type="application/json",
+            headers={"Content-Encoding": "gzip", "x-request-id": "abc123"},
+        )
+
+        resp = requests.get("https://example.org/")
+
+        assert resp.status_code == 200
+        # urllib3/requests should decode the real gzip bytes back to the
+        # original, uncompressed content.
+        assert resp.content == original_body
+        assert resp.json() == {"first_name": True}
+        # Unrelated headers must be preserved.
+        assert resp.headers["x-request-id"] == "abc123"
+
+    run()
+    assert_reset()
+
+
 def test_content_length_error(monkeypatch):
     """
     Currently 'requests' does not enforce content length validation,