Default retry time (#505)

* parametrize retry time

* add a placeholder for sentinelhub service class

* update defaults to 30s

* also prepare things for release

CHANGELOG.MD

@@ -1,3 +1,8 @@
+## [Version 3.9.5] - 2023-12-07
+
+- The `SentinelHubDownloadClient` class now has a `default_retry_time` parameter, which allows control over the waiting time when a request gets a 429 TOO_MANY_REQUESTS response without a specific retry time in the headers. The default value for this behavior has been changed from 0s to 30s to avoid edge-cases where SH services were bombarded with requests.
+
+
 ## [Version 3.9.4] - 2023-11-13
 
 - Fixed a problem with `dataclasses_json 0.6.2` that broke BYOC functionalities

sentinelhub/_version.py

@@ -1,3 +1,3 @@
 """Version of the sentinelhub package."""
 
-__version__ = "3.9.4"
+__version__ = "3.9.5"

sentinelhub/api/base.py

@@ -25,6 +25,8 @@
 class SentinelHubService(metaclass=ABCMeta):
     """A base class for classes interacting with different Sentinel Hub APIs"""
 
+    _DEFAULT_RETRY_TIME = 30
+
     def __init__(self, config: Optional[SHConfig] = None):
         """
         :param config: A configuration object with required parameters `sh_client_id`, `sh_client_secret`, and
@@ -36,7 +38,7 @@ def __init__(self, config: Optional[SHConfig] = None):
         base_url = self.config.sh_base_url.rstrip("/")
         self.service_url = self._get_service_url(base_url)
 
-        self.client = SentinelHubDownloadClient(config=self.config)
+        self.client = SentinelHubDownloadClient(config=self.config, default_retry_time=self._DEFAULT_RETRY_TIME)
 
     @staticmethod
     @abstractmethod

sentinelhub/download/rate_limit.py

@@ -50,9 +50,9 @@ def register_next(self) -> float:
 
         return wait_time
 
-    def update(self, headers: dict) -> None:
+    def update(self, headers: dict, *, default: float) -> None:
         """Update the next possible download time if the service has responded with the rate limit"""
-        retry_after: float = int(headers.get(self.RETRY_HEADER, 0))  # can be a string representation of a number
+        retry_after: float = int(headers.get(self.RETRY_HEADER, default))  # can be a string representation of a number
         retry_after = retry_after / 1000
 
         if retry_after:

sentinelhub/download/sentinelhub_client.py

@@ -34,11 +34,13 @@ class SentinelHubDownloadClient(DownloadClient):
     _CACHED_SESSIONS: ClassVar[dict[tuple[str, str], SentinelHubSession]] = {}
     _UNIVERSAL_CACHE_KEY = "universal-user", "default-url"
 
-    def __init__(self, *, session: SentinelHubSession | None = None, **kwargs: Any):
+    def __init__(self, *, session: SentinelHubSession | None = None, default_retry_time: float = 30, **kwargs: Any):
         """
         :param session: If a session object is provided here then this client instance will always use only the
             provided session. Otherwise, it will either use a cached session or create a new session and cache
             it.
+        :param default_retry_time: The default waiting time when retrying after getting a TOO_MANY_REQUESTS response
+            without appropriate retry headers.
         :param kwargs: Optional parameters from DownloadClient
         """
         super().__init__(**kwargs)
@@ -49,6 +51,7 @@ def __init__(self, *, session: SentinelHubSession | None = None, **kwargs: Any):
                 f"{session} was given"
             )
         self.session = session
+        self.default_retry_time = default_retry_time
 
         self.rate_limit = SentinelHubRateLimit(num_processes=self.config.number_of_download_processes)
         self.lock: Lock | None = None
@@ -84,10 +87,9 @@ def _execute_download(self, request: DownloadRequest) -> DownloadResponse:
                 )
                 response = self._do_download(request)
 
-                self._execute_thread_safe(self.rate_limit.update, response.headers)
-
                 if response.status_code == requests.status_codes.codes.TOO_MANY_REQUESTS:
                     warnings.warn("Download rate limit hit", category=SHRateLimitWarning)
+                    self._execute_thread_safe(self.rate_limit.update, response.headers, default=self.default_retry_time)
                     continue
 
                 response.raise_for_status()

tests/download/test_rate_limit.py

@@ -185,7 +185,7 @@ def run_interaction(
             rate_limit_hits += 1
             logger.info("Process %d: rate limit hit %s", index, response_headers)
 
-        rate_limit.update(response_headers)
+        rate_limit.update(response_headers, default=0)
 
     return rate_limit_hits