From f21e394014ce7721e1fa9a829f80b3096d61b15d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Thu, 23 Nov 2023 10:15:01 +0100 Subject: [PATCH 1/2] add version bound to typing extensions (#502) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fd32fc48..3dfeff58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ dependencies = [ "tomli", "tomli_w", "tqdm", - "typing-extensions", + "typing-extensions>=4.5.0", "utm", ] From 6bed47c77f8870436c9e5a0d5b206e48b0dcd68b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?= <31988337+zigaLuksic@users.noreply.github.com> Date: Thu, 7 Dec 2023 12:51:48 +0100 Subject: [PATCH 2/2] Default retry time (#505) * parametrize retry time * add a placeholder for sentinelhub service class * update defaults to 30s * also prepare things for release --- CHANGELOG.MD | 5 +++++ sentinelhub/_version.py | 2 +- sentinelhub/api/base.py | 4 +++- sentinelhub/download/rate_limit.py | 4 ++-- sentinelhub/download/sentinelhub_client.py | 8 +++++--- tests/download/test_rate_limit.py | 2 +- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 5a5bbc1c..c1d1447a 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,3 +1,8 @@ +## [Version 3.9.5] - 2023-12-07 + +- The `SentinelHubDownloadClient` class now has a `default_retry_time` parameter, which allows control over the waiting time when a request gets a 429 TOO_MANY_REQUESTS response without a specific retry time in the headers. The default value for this behavior has been changed from 0s to 30s to avoid edge-cases where SH services were bombarded with requests. + + ## [Version 3.9.4] - 2023-11-13 - Fixed a problem with `dataclasses_json 0.6.2` that broke BYOC functionalities diff --git a/sentinelhub/_version.py b/sentinelhub/_version.py index b98c7f0b..b2c3350a 100644 --- a/sentinelhub/_version.py +++ b/sentinelhub/_version.py @@ -1,3 +1,3 @@ """Version of the sentinelhub package.""" -__version__ = "3.9.4" +__version__ = "3.9.5" diff --git a/sentinelhub/api/base.py b/sentinelhub/api/base.py index 9911767d..d079f2c6 100644 --- a/sentinelhub/api/base.py +++ b/sentinelhub/api/base.py @@ -25,6 +25,8 @@ class SentinelHubService(metaclass=ABCMeta): """A base class for classes interacting with different Sentinel Hub APIs""" + _DEFAULT_RETRY_TIME = 30 + def __init__(self, config: Optional[SHConfig] = None): """ :param config: A configuration object with required parameters `sh_client_id`, `sh_client_secret`, and @@ -36,7 +38,7 @@ def __init__(self, config: Optional[SHConfig] = None): base_url = self.config.sh_base_url.rstrip("/") self.service_url = self._get_service_url(base_url) - self.client = SentinelHubDownloadClient(config=self.config) + self.client = SentinelHubDownloadClient(config=self.config, default_retry_time=self._DEFAULT_RETRY_TIME) @staticmethod @abstractmethod diff --git a/sentinelhub/download/rate_limit.py b/sentinelhub/download/rate_limit.py index 6efd5ebb..aec22a0a 100644 --- a/sentinelhub/download/rate_limit.py +++ b/sentinelhub/download/rate_limit.py @@ -50,9 +50,9 @@ def register_next(self) -> float: return wait_time - def update(self, headers: dict) -> None: + def update(self, headers: dict, *, default: float) -> None: """Update the next possible download time if the service has responded with the rate limit""" - retry_after: float = int(headers.get(self.RETRY_HEADER, 0)) # can be a string representation of a number + retry_after: float = int(headers.get(self.RETRY_HEADER, default)) # can be a string representation of a number retry_after = retry_after / 1000 if retry_after: diff --git a/sentinelhub/download/sentinelhub_client.py b/sentinelhub/download/sentinelhub_client.py index e714eb40..8b1f455a 100644 --- a/sentinelhub/download/sentinelhub_client.py +++ b/sentinelhub/download/sentinelhub_client.py @@ -34,11 +34,13 @@ class SentinelHubDownloadClient(DownloadClient): _CACHED_SESSIONS: ClassVar[dict[tuple[str, str], SentinelHubSession]] = {} _UNIVERSAL_CACHE_KEY = "universal-user", "default-url" - def __init__(self, *, session: SentinelHubSession | None = None, **kwargs: Any): + def __init__(self, *, session: SentinelHubSession | None = None, default_retry_time: float = 30, **kwargs: Any): """ :param session: If a session object is provided here then this client instance will always use only the provided session. Otherwise, it will either use a cached session or create a new session and cache it. + :param default_retry_time: The default waiting time when retrying after getting a TOO_MANY_REQUESTS response + without appropriate retry headers. :param kwargs: Optional parameters from DownloadClient """ super().__init__(**kwargs) @@ -49,6 +51,7 @@ def __init__(self, *, session: SentinelHubSession | None = None, **kwargs: Any): f"{session} was given" ) self.session = session + self.default_retry_time = default_retry_time self.rate_limit = SentinelHubRateLimit(num_processes=self.config.number_of_download_processes) self.lock: Lock | None = None @@ -84,10 +87,9 @@ def _execute_download(self, request: DownloadRequest) -> DownloadResponse: ) response = self._do_download(request) - self._execute_thread_safe(self.rate_limit.update, response.headers) - if response.status_code == requests.status_codes.codes.TOO_MANY_REQUESTS: warnings.warn("Download rate limit hit", category=SHRateLimitWarning) + self._execute_thread_safe(self.rate_limit.update, response.headers, default=self.default_retry_time) continue response.raise_for_status() diff --git a/tests/download/test_rate_limit.py b/tests/download/test_rate_limit.py index 5f21a67f..8ab3d6be 100644 --- a/tests/download/test_rate_limit.py +++ b/tests/download/test_rate_limit.py @@ -185,7 +185,7 @@ def run_interaction( rate_limit_hits += 1 logger.info("Process %d: rate limit hit %s", index, response_headers) - rate_limit.update(response_headers) + rate_limit.update(response_headers, default=0) return rate_limit_hits