Release 3.10.2, Merge pull request #529 from sentinel-hub/develop

Release 3.10.2
sentinel-hub · Apr 24, 2024 · f9f29c9 · f9f29c9
2 parents c7f5912 + 13a71ab
commit f9f29c9
Show file tree

Hide file tree

Showing 27 changed files with 172 additions and 124 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: end-of-file-fixer
       - id: requirements-txt-fixer
@@ -13,18 +13,18 @@ repos:
       - id: debug-statements
 
   - repo: https://github.com/psf/black
-    rev: 23.12.1
+    rev: 24.4.0
     hooks:
       - id: black
         language_version: python3
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.1.11"
+    rev: "v0.4.1"
     hooks:
       - id: ruff
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.7.1
+    rev: 1.8.5
     hooks:
       - id: nbqa-black
       - id: nbqa-ruff
diff --git a/CHANGELOG.MD b/CHANGELOG.MD
@@ -1,3 +1,8 @@
+## [Version 3.10.2] - 2024-24-04
+
+- Added `max_retries` parameter to `SHConfig` class. It controls how many times the client will attempt to re-download before raising `OutOfRequestsException`. It is set to `None` by default, in which case it never stops trying. Contributed by @Regan-Koopmans.
+
+
 ## [Version 3.10.1] - 2024-01-10
 
 - Improved documentation for Copernicus Data Space Ecosystem.

diff --git a/examples/fis_request.ipynb b/examples/fis_request.ipynb
@@ -432,13 +432,15 @@
     "\n",
     "geometry1 = Geometry(Polygon([(-5.13, 48), (-5.23, 48.09), (-5.13, 48.17), (-5.03, 48.08), (-5.13, 48)]), CRS.WGS84)\n",
     "geometry2 = Geometry(\n",
-    "    Polygon([\n",
-    "        (1292344.0, 5205055.5),\n",
-    "        (1301479.5, 5195920.0),\n",
-    "        (1310615.0, 5205055.5),\n",
-    "        (1301479.5, 5214191.0),\n",
-    "        (1292344.0, 5205055.5),\n",
-    "    ]),\n",
+    "    Polygon(\n",
+    "        [\n",
+    "            (1292344.0, 5205055.5),\n",
+    "            (1301479.5, 5195920.0),\n",
+    "            (1310615.0, 5205055.5),\n",
+    "            (1301479.5, 5214191.0),\n",
+    "            (1292344.0, 5205055.5),\n",
+    "        ]\n",
+    "    ),\n",
     "    CRS.POP_WEB,\n",
     ")"
    ]

diff --git a/examples/process_request.ipynb b/examples/process_request.ipynb
@@ -989,13 +989,15 @@
     "request_raw_dict = {\n",
     "    \"input\": {\n",
     "        \"bounds\": {\"properties\": {\"crs\": betsiboka_bbox.crs.opengis_string}, \"bbox\": list(betsiboka_bbox)},\n",
-    "        \"data\": [{\n",
-    "            \"type\": \"S2L1C\",\n",
-    "            \"dataFilter\": {\n",
-    "                \"timeRange\": {\"from\": \"2020-06-01T00:00:00Z\", \"to\": \"2020-06-30T00:00:00Z\"},\n",
-    "                \"mosaickingOrder\": \"leastCC\",\n",
-    "            },\n",
-    "        }],\n",
+    "        \"data\": [\n",
+    "            {\n",
+    "                \"type\": \"S2L1C\",\n",
+    "                \"dataFilter\": {\n",
+    "                    \"timeRange\": {\"from\": \"2020-06-01T00:00:00Z\", \"to\": \"2020-06-30T00:00:00Z\"},\n",
+    "                    \"mosaickingOrder\": \"leastCC\",\n",
+    "                },\n",
+    "            }\n",
+    "        ],\n",
     "    },\n",
     "    \"output\": {\n",
     "        \"width\": betsiboka_size[0],\n",

diff --git a/examples/process_request_cdse.ipynb b/examples/process_request_cdse.ipynb
@@ -941,13 +941,15 @@
     "request_raw_dict = {\n",
     "    \"input\": {\n",
     "        \"bounds\": {\"properties\": {\"crs\": betsiboka_bbox.crs.opengis_string}, \"bbox\": list(betsiboka_bbox)},\n",
-    "        \"data\": [{\n",
-    "            \"type\": \"S2L1C\",\n",
-    "            \"dataFilter\": {\n",
-    "                \"timeRange\": {\"from\": \"2020-06-01T00:00:00Z\", \"to\": \"2020-06-30T00:00:00Z\"},\n",
-    "                \"mosaickingOrder\": \"leastCC\",\n",
-    "            },\n",
-    "        }],\n",
+    "        \"data\": [\n",
+    "            {\n",
+    "                \"type\": \"S2L1C\",\n",
+    "                \"dataFilter\": {\n",
+    "                    \"timeRange\": {\"from\": \"2020-06-01T00:00:00Z\", \"to\": \"2020-06-30T00:00:00Z\"},\n",
+    "                    \"mosaickingOrder\": \"leastCC\",\n",
+    "                },\n",
+    "            }\n",
+    "        ],\n",
     "    },\n",
     "    \"output\": {\n",
     "        \"width\": betsiboka_size[0],\n",

diff --git a/pyproject.toml b/pyproject.toml
@@ -63,7 +63,7 @@ docs = [
     "matplotlib",
     "nbsphinx",
     "sphinx==7.1.2",
-    "sphinx_mdinclude",
+    "sphinx_mdinclude==0.5.4",  # version fixed because 0.6.0 didnt work at release time of 3.10.2
     "sphinx_rtd_theme==1.3.0",
 ]
 dev = [
@@ -73,14 +73,13 @@ dev = [
     "click>=8.0.0",
     "fs",
     "mypy>=0.990",
-    "moto",
+    "moto[s3]>=5.0.0",
     "pandas",
     "pre-commit",
     "pylint>=2.14.0",
     "pytest>=4.0.0",
     "pytest-cov",
     "pytest-dependency",
-    "pytest-lazy-fixture",
     "pytest-mock",
     "ray[default]",
     "requests-mock",

diff --git a/sentinelhub/_version.py b/sentinelhub/_version.py
@@ -1,3 +1,3 @@
 """Version of the sentinelhub package."""
 
-__version__ = "3.10.1"
+__version__ = "3.10.2"
diff --git a/sentinelhub/api/base_request.py b/sentinelhub/api/base_request.py
@@ -134,11 +134,13 @@ def _get_base_url(self) -> str:
         settings from config object. In case different collections have different restrictions then
         `SHConfig.sh_base_url` breaks the tie in case it matches one of the data collection URLs.
         """
-        data_collection_urls = tuple({
-            input_data_dict.service_url.rstrip("/")
-            for input_data_dict in self.payload["input"]["data"]
-            if isinstance(input_data_dict, InputDataDict) and input_data_dict.service_url is not None
-        })
+        data_collection_urls = tuple(
+            {
+                input_data_dict.service_url.rstrip("/")
+                for input_data_dict in self.payload["input"]["data"]
+                if isinstance(input_data_dict, InputDataDict) and input_data_dict.service_url is not None
+            }
+        )
         config_base_url = self.config.sh_base_url.rstrip("/")
 
         if not data_collection_urls:

diff --git a/sentinelhub/api/batch/process.py b/sentinelhub/api/batch/process.py
@@ -150,17 +150,19 @@ def output(
         :param kwargs: Any other arguments to be added to a dictionary of parameters
         :return: A dictionary of output parameters
         """
-        return remove_undefined({
-            "defaultTilePath": default_tile_path,
-            "overwrite": overwrite,
-            "skipExisting": skip_existing,
-            "cogOutput": cog_output,
-            "cogParameters": cog_parameters,
-            "createCollection": create_collection,
-            "collectionId": collection_id,
-            "responses": responses,
-            **kwargs,
-        })
+        return remove_undefined(
+            {
+                "defaultTilePath": default_tile_path,
+                "overwrite": overwrite,
+                "skipExisting": skip_existing,
+                "cogOutput": cog_output,
+                "cogParameters": cog_parameters,
+                "createCollection": create_collection,
+                "collectionId": collection_id,
+                "responses": responses,
+                **kwargs,
+            }
+        )
 
     def iter_tiling_grids(self, **kwargs: Any) -> SentinelHubFeatureIterator:
         """An iterator over tiling grids

diff --git a/sentinelhub/api/byoc.py b/sentinelhub/api/byoc.py
@@ -220,11 +220,13 @@ def update_tile(self, collection: CollectionType, tile: TileType) -> Json:
         headers = {"Content-Type": MimeType.JSON.get_string()}
 
         _tile = self._to_dict(tile)
-        updates = remove_undefined({
-            "path": _tile["path"],
-            "coverGeometry": _tile.get("coverGeometry"),
-            "sensingTime": _tile.get("sensingTime"),
-        })
+        updates = remove_undefined(
+            {
+                "path": _tile["path"],
+                "coverGeometry": _tile.get("coverGeometry"),
+                "sensingTime": _tile.get("sensingTime"),
+            }
+        )
 
         return self.client.get_json(
             url=url, request_type=RequestType.PUT, post_values=updates, headers=headers, use_session=True

diff --git a/sentinelhub/api/catalog.py b/sentinelhub/api/catalog.py
@@ -140,20 +140,22 @@ def search(
         if geometry and geometry.crs is not CRS.WGS84:
             geometry = geometry.transform(CRS.WGS84)
 
-        payload = remove_undefined({
-            "collections": [collection_id],
-            "datetime": f"{start_time}/{end_time}" if time else None,
-            "bbox": list(bbox) if bbox else None,
-            "intersects": geometry.get_geojson(with_crs=False) if geometry else None,
-            "ids": ids,
-            "filter": self._prepare_filters(filter, collection, filter_lang),
-            "filter-lang": filter_lang,
-            "filter-crs": filter_crs,
-            "fields": fields,
-            "distinct": distinct,
-            "limit": limit,
-            **kwargs,
-        })
+        payload = remove_undefined(
+            {
+                "collections": [collection_id],
+                "datetime": f"{start_time}/{end_time}" if time else None,
+                "bbox": list(bbox) if bbox else None,
+                "intersects": geometry.get_geojson(with_crs=False) if geometry else None,
+                "ids": ids,
+                "filter": self._prepare_filters(filter, collection, filter_lang),
+                "filter-lang": filter_lang,
+                "filter-crs": filter_crs,
+                "fields": fields,
+                "distinct": distinct,
+                "limit": limit,
+                **kwargs,
+            }
+        )
 
         return CatalogSearchIterator(self.client, url, payload)
 

diff --git a/sentinelhub/config.py b/sentinelhub/config.py
@@ -24,7 +24,7 @@
 
 
 @dataclass(repr=False)
-class _SHConfig:
+class _SHConfig:  # pylint: disable=too-many-instance-attributes
     instance_id: str = ""
     sh_client_id: str = ""
     sh_client_secret: str = ""
@@ -46,6 +46,7 @@ class _SHConfig:
     download_sleep_time: float = 5.0
     download_timeout_seconds: float = 120.0
     number_of_download_processes: int = 1
+    max_retries: int | None = None
 
     def __post_init__(self) -> None:
         if self.sh_auth_base_url is not None:
@@ -94,6 +95,7 @@ class SHConfig(_SHConfig):
           attempt this number exponentially increases with factor `3`.
         - `download_timeout_seconds`: Maximum number of seconds before download attempt is canceled.
         - `number_of_download_processes`: Number of download processes, used to calculate rate-limit sleep time.
+        - `max_retries`: Maximum number of retries until an exception is raised.
 
     The location of `config.toml` for manual modification can be found with `SHConfig.get_config_location()`.
     """

diff --git a/sentinelhub/data_collections_bands.py b/sentinelhub/data_collections_bands.py
@@ -1,4 +1,4 @@
-""" Contains information about data collections used by SH """
+"""Contains information about data collections used by SH"""
 
 from __future__ import annotations
 

diff --git a/sentinelhub/download/handlers.py b/sentinelhub/download/handlers.py
@@ -52,7 +52,7 @@ def new_download_func(self: Self, request: DownloadRequest) -> T:
 
 
 def retry_temporary_errors(
-    download_func: Callable[[SelfWithConfig, DownloadRequest], T]
+    download_func: Callable[[SelfWithConfig, DownloadRequest], T],
 ) -> Callable[[SelfWithConfig, DownloadRequest], T]:
     """Decorator function for handling server and connection errors"""
     backoff_coefficient = 3

diff --git a/sentinelhub/download/sentinelhub_client.py b/sentinelhub/download/sentinelhub_client.py
@@ -15,7 +15,7 @@
 
 from ..config import SHConfig
 from ..constants import SHConstants
-from ..exceptions import SHRateLimitWarning, SHRuntimeWarning
+from ..exceptions import OutOfRequestsException, SHRateLimitWarning, SHRuntimeWarning
 from ..types import JsonDict
 from .client import DownloadClient
 from .handlers import fail_user_errors, retry_temporary_errors
@@ -75,10 +75,12 @@ def _execute_download(self, request: DownloadRequest) -> DownloadResponse:
         """
         Executes the download with a single thread and uses a rate limit object, which is shared between all threads
         """
+        download_attempts = 0
         while True:
             sleep_time = self._execute_thread_safe(self.rate_limit.register_next)
 
             if sleep_time == 0:
+                download_attempts += 1
                 LOGGER.debug(
                     "Sending %s request to %s. Hash of sent request is %s",
                     request.request_type.value,
@@ -89,6 +91,9 @@ def _execute_download(self, request: DownloadRequest) -> DownloadResponse:
 
                 if response.status_code == requests.status_codes.codes.TOO_MANY_REQUESTS:
                     warnings.warn("Download rate limit hit", category=SHRateLimitWarning)
+                    if self.config.max_retries is not None and download_attempts >= self.config.max_retries:
+                        raise OutOfRequestsException("Maximum number of download attempts reached")
+
                     self._execute_thread_safe(self.rate_limit.update, response.headers, default=self.default_retry_time)
                     continue
 

diff --git a/sentinelhub/geometry.py b/sentinelhub/geometry.py
@@ -150,7 +150,7 @@ def _to_tuple(cls, bbox: BBoxInputType) -> tuple[float, float, float, float]:
 
     @staticmethod
     def _tuple_from_list_or_tuple(
-        bbox: tuple[float, float, float, float] | tuple[tuple[float, float], tuple[float, float]]
+        bbox: tuple[float, float, float, float] | tuple[tuple[float, float], tuple[float, float]],
     ) -> tuple[float, float, float, float]:
         """Converts a list or tuple representation of a bbox into a flat tuple representation.
 

diff --git a/tests/api/batch/test_process.py b/tests/api/batch/test_process.py
@@ -67,21 +67,23 @@ def test_create_and_run_batch_request(batch_client: SentinelHubBatch, requests_m
     request_id = "mocked-id"
     requests_mock.post(
         "/api/v1/batch/process",
-        [{
-            "json": {
-                "id": request_id,
-                "processRequest": {
-                    "input": {
-                        "bounds": {
-                            "bbox": list(bbox),
-                            "properties": {"crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"},
+        [
+            {
+                "json": {
+                    "id": request_id,
+                    "processRequest": {
+                        "input": {
+                            "bounds": {
+                                "bbox": list(bbox),
+                                "properties": {"crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"},
+                            }
                         }
-                    }
-                },
-                "tileCount": 42,
-                "status": "CREATED",
+                    },
+                    "tileCount": 42,
+                    "status": "CREATED",
+                }
             }
-        }],
+        ],
     )
 
     batch_request = batch_client.create(