fix(robots): disallow crawling when robots.txt returns a 5xx (#2065)

`RobotsTxtFile.load` only special-cased 4xx responses (file unavailable
-> allow
all) and parsed the raw response body as robots rules for every other
status. The
HTTP clients do not raise on 5xx, so a server-error response reached
that branch and
its body was handed to Protego. Protego reads a non-robots body (for
example an HTML
`500` error page, or an empty body) as allow-all, so a transient 5xx on
`robots.txt`
silently permitted crawling every URL, the opposite of what the site
expects while it
is failing.

Per RFC 9309 section 2.3.1.4, a `robots.txt` that is unreachable due to
server errors
(5xx) is undefined and a crawler "MUST assume complete disallow". This
PR handles 5xx
explicitly with a disallow-all directive, using the existing
`is_status_code_server_error` helper. The 4xx (allow-all, section
2.3.1.3), 2xx/3xx
(parse the body), and network-error paths are left unchanged.

This only affects users who opt in with `respect_robots_txt_file=True`,
and only when
`robots.txt` returns a 5xx: previously such a response allowed crawling
everything, now
it correctly disallows until the file is reachable again. It also
matches crawlee-JS,
which never parses a non-2xx `robots.txt` body as rules.

Author: anxkhn

src/crawlee/_utils/robots.py

@@ -8,7 +8,7 @@
 
 from crawlee._utils.sitemap import Sitemap
 from crawlee._utils.urls import filter_url
-from crawlee._utils.web import is_status_code_client_error
+from crawlee._utils.web import is_status_code_client_error, is_status_code_server_error
 
 if TYPE_CHECKING:
     from typing_extensions import Self
@@ -57,11 +57,15 @@ async def load(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | N
         try:
             response = await http_client.send_request(url, proxy_info=proxy_info)
 
-            body = (
-                b'User-agent: *\nAllow: /'
-                if is_status_code_client_error(response.status_code)
-                else await response.read()
-            )
+            status_code = response.status_code
+            if is_status_code_client_error(status_code):
+                # A 4xx response means the file is unavailable, so crawling is unrestricted (RFC 9309, section 2.3.1.3).
+                body = b'User-agent: *\nAllow: /'
+            elif is_status_code_server_error(status_code):
+                # A 5xx response means the file is unreachable, so assume complete disallow (RFC 9309, section 2.3.1.4).
+                body = b'User-agent: *\nDisallow: /'
+            else:
+                body = await response.read()
             robots = Protego.parse(body.decode('utf-8'))
 
         except Exception as e:

tests/unit/_utils/test_robots.py

@@ -25,6 +25,16 @@ async def test_allow_disallow_robots_txt(server_url: URL, http_client: HttpClien
     assert not robots.is_allowed(str(server_url / 'deny_all/page.html'))
 
 
+async def test_server_error_disallows_all(server_url: URL, http_client: HttpClient) -> None:
+    """A 5xx robots.txt response is treated as complete disallow (RFC 9309, section 2.3.1.4).
+
+    The response body must not be parsed as rules; otherwise an error page would be misread as allow-all.
+    """
+    robots = await RobotsTxtFile.load(str(server_url / 'status/500'), http_client)
+    assert not robots.is_allowed(str(server_url / 'admin/page.html'))
+    assert not robots.is_allowed(str(server_url / 'something/page.html'))
+
+
 async def test_extract_sitemaps_urls(server_url: URL, http_client: HttpClient) -> None:
     """Cross-host sitemap entries are dropped under the `'same-hostname'` enqueue strategy."""
     robots = await RobotsTxtFile.find(str(server_url), http_client)