Merge branch 'fix/rate_limiting_for_all_deployments' into 'master'

Feature - rate limiting for all deployments See merge request team-6/openeo-sentinelhub-python-driver!321
Open-EO · Oct 13, 2023 · b7fdf90 · b7fdf90
2 parents 08a4fe9 + 36eaa96
commit b7fdf90
Show file tree

Hide file tree

Showing 2 changed files with 73 additions and 4 deletions.
diff --git a/rest/processing/processing_api_request.py b/rest/processing/processing_api_request.py
@@ -60,6 +60,10 @@ def make_request(self):
         return requests.post(self.url, data=json.dumps(self.data), headers=self.get_headers())
 
     def has_rate_limiting_with_backoff(self):
-        if self.url.startswith(SentinelhubDeployments.USWEST):
+        if (
+            self.url.startswith(SentinelhubDeployments.MAIN)
+            or self.url.startswith(SentinelhubDeployments.USWEST)
+            or self.url.startswith(SentinelhubDeployments.CREODIAS)
+        ):
             return True
         return False
diff --git a/tests/test_units.py b/tests/test_units.py
@@ -986,9 +986,40 @@ def test_get_roles(filename, expected_roles):
     "endpoint,access_token,api_responses,min_exec_time,should_raise_error,expected_error",
     [
         ("https://services.sentinel-hub.com", None, [{"status": 200}], 0, False, None),
-        ("https://services.sentinel-hub.com", None, [{"status": 429}], 0, True, "Too Many Requests"),
-        ("https://services.sentinel-hub.com", "some-token", [{"status": 200}], 0, False, None),
-        ("https://services.sentinel-hub.com", "some-token", [{"status": 429}], 0, True, "Too Many Requests"),
+        (
+            "https://services.sentinel-hub.com",
+            None,
+            [
+                {"status": 429, "headers": {"retry-after": "2"}},
+                {"status": 429, "headers": {"retry-after": "7"}},
+                {"status": 200},
+            ],
+            9,
+            False,
+            None,
+        ),
+        (
+            "https://services.sentinel-hub.com",
+            None,
+            [
+                {"status": 429, "headers": {"retry-after": "2"}},
+                {"status": 429, "headers": {"retry-after": "2"}},
+                {"status": 429, "headers": {"retry-after": "2"}},
+                {"status": 429, "headers": {"retry-after": "2"}},
+            ],
+            0,
+            True,
+            "Out of retries.",
+        ),
+        (
+            "https://services.sentinel-hub.com",
+            None,
+            [{"status": 429, "headers": {"retry-after": "2"}}, {"status": 500}],
+            0,
+            True,
+            "Internal Server Error",
+        ),
+        ("https://services-uswest2.sentinel-hub.com", None, [{"status": 200}], 0, False, None),
         (
             "https://services-uswest2.sentinel-hub.com",
             None,
@@ -1022,6 +1053,40 @@ def test_get_roles(filename, expected_roles):
             True,
             "Internal Server Error",
         ),
+        ("https://creodias.sentinel-hub.com", None, [{"status": 200}], 0, False, None),
+        (
+            "https://creodias.sentinel-hub.com",
+            None,
+            [
+                {"status": 429, "headers": {"retry-after": "2"}},
+                {"status": 429, "headers": {"retry-after": "7"}},
+                {"status": 200},
+            ],
+            9,
+            False,
+            None,
+        ),
+        (
+            "https://creodias.sentinel-hub.com",
+            None,
+            [
+                {"status": 429, "headers": {"retry-after": "2"}},
+                {"status": 429, "headers": {"retry-after": "2"}},
+                {"status": 429, "headers": {"retry-after": "2"}},
+                {"status": 429, "headers": {"retry-after": "2"}},
+            ],
+            0,
+            True,
+            "Out of retries.",
+        ),
+        (
+            "https://creodias.sentinel-hub.com",
+            None,
+            [{"status": 429, "headers": {"retry-after": "2"}}, {"status": 500}],
+            0,
+            True,
+            "Internal Server Error",
+        ),
     ],
 )
 def test_processing_api_request(