Release version 3.8.2, Merge pull request #387 from sentinel-hub/develop

Release version 3.8.2

.github/workflows/ci_action.yml

@@ -108,7 +108,7 @@ jobs:
           pytest -m "not sh_integration and not aws_integration"
 
       - name: Upload code coverage
-        if: ${{ matrix.full_test_suite }}
+        if: ${{ matrix.full_test_suite && github.event_name == 'push' }}
         uses: codecov/codecov-action@v2
         with:
           files: coverage.xml

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
         language_version: python3
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.11.4
+    rev: 5.12.0
     hooks:
       - id: isort
         name: isort (python)
@@ -46,7 +46,7 @@ repos:
           - flake8-typing-imports
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.6.0
+    rev: 1.6.1
     hooks:
       - id: nbqa-black
       - id: nbqa-isort

requirements-dev.txt

@@ -5,6 +5,7 @@ codecov
 fs
 moto
 mypy>=0.990
+pandas
 pre-commit
 pylint>=2.14.0
 pytest>=4.0.0

sentinelhub/_version.py

@@ -1,3 +1,3 @@
 """Version of the sentinelhub package."""
 
-__version__ = "3.8.1"
+__version__ = "3.8.2"

sentinelhub/api/batch/process.py

@@ -14,6 +14,7 @@
 
 from ...constants import RequestType
 from ...data_collections import DataCollection
+from ...exceptions import deprecated_function
 from ...geometry import CRS, BBox, Geometry
 from ...types import Json, JsonDict
 from ..base import BaseCollection, SentinelHubFeatureIterator
@@ -358,6 +359,7 @@ def get_tile(self, batch_request: BatchRequestType, tile_id: Optional[int]) -> J
         url = self._get_tiles_url(request_id, tile_id=tile_id)
         return self.client.get_json_dict(url, use_session=True)
 
+    @deprecated_function(message_suffix="The service endpoint will be removed soon. Please use `restart_job` instead.")
     def reprocess_tile(self, batch_request: BatchRequestType, tile_id: Optional[int]) -> Json:
         """Reprocess a single failed tile
 

sentinelhub/constants.py

@@ -16,9 +16,10 @@
 from aenum import extend_enum
 
 from ._version import __version__
-from .exceptions import SHUserWarning
+from .exceptions import SHUserWarning, deprecated_class
 
 
+@deprecated_class()
 class PackageProps:
     """Class for obtaining package properties. Currently, it supports obtaining package version."""
 
@@ -115,7 +116,7 @@ def _parse_crs(value: object) -> object:
         """
         if isinstance(value, dict) and "init" in value:
             value = value["init"]
-        if isinstance(value, pyproj.CRS):
+        if hasattr(value, "to_epsg"):
             if value == CRSMeta._UNSUPPORTED_CRS:
                 message = (
                     "sentinelhub-py supports only WGS 84 coordinate reference system with "
@@ -148,7 +149,7 @@ def _parse_crs(value: object) -> object:
                 value = match.group("code")
             if value.upper() == "CRS84":
                 return "4326"
-            return value.lower().strip("epsg: ")
+            return value.lower().replace("epsg:", "").strip()
         return value
 
 
@@ -451,4 +452,4 @@ class SHConstants:
     """
 
     LATEST = "latest"
-    HEADERS = {"User-Agent": f"sentinelhub-py/v{PackageProps.get_version()}"}
+    HEADERS = {"User-Agent": f"sentinelhub-py/v{__version__}"}

sentinelhub/download/client.py

@@ -4,10 +4,10 @@
 import json
 import logging
 import os
-import sys
 import warnings
-from concurrent.futures import Future, ThreadPoolExecutor, as_completed
-from typing import Any, List, Optional, Union, overload
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from contextlib import nullcontext
+from typing import Any, Iterable, List, Optional, Union
 from xml.etree import ElementTree
 
 import requests
@@ -19,6 +19,7 @@
     DownloadFailedException,
     HashedNameCollisionException,
     MissingDataInRequestException,
+    SHDeprecationWarning,
     SHRuntimeWarning,
 )
 from ..io_utils import read_data
@@ -56,77 +57,60 @@ def __init__(
 
         self.config = config or SHConfig()
 
-    @overload
     def download(
         self,
-        download_requests: DownloadRequest,
-        max_threads: Optional[int] = None,
-        decode_data: bool = True,
-        show_progress: bool = False,
-    ) -> Any:
-        ...
-
-    @overload
-    def download(
-        self,
-        download_requests: List[DownloadRequest],
+        download_requests: Iterable[DownloadRequest],
         max_threads: Optional[int] = None,
         decode_data: bool = True,
         show_progress: bool = False,
     ) -> List[Any]:
-        ...
-
-    def download(
-        self,
-        download_requests: Union[DownloadRequest, List[DownloadRequest]],
-        max_threads: Optional[int] = None,
-        decode_data: bool = True,
-        show_progress: bool = False,
-    ) -> Union[List[Any], Any]:
         """Download one or multiple requests, provided as a request list.
 
-        :param download_requests: A list of requests or a single request to be executed.
+        :param download_requests: A list of requests to be executed.
         :param max_threads: Maximum number of threads to be used for download in parallel. The default is
             `max_threads=None` which will use the number of processors on the system multiplied by 5.
         :param decode_data: If `True` it will decode data otherwise it will return it in form of a `DownloadResponse`
             objects which contain binary data and response metadata.
         :param show_progress: Whether a progress bar should be displayed while downloading
-        :return: A list of results or a single result, depending on input parameter `download_requests`
+        :return: A list of results
         """
-        downloads = [download_requests] if isinstance(download_requests, DownloadRequest) else download_requests
+        if isinstance(download_requests, DownloadRequest):
+            warnings.warn(
+                (
+                    "The parameter `download_requests` should be a sequence of requests. In future versions download of"
+                    " single requests will only be supported if provided as a singelton tuple or list."
+                ),
+                category=SHDeprecationWarning,
+            )
+            requests_list: List[DownloadRequest] = [download_requests]
+        else:
+            requests_list = list(download_requests)
 
-        data_list = [None] * len(downloads)
+        results = [None] * len(requests_list)
 
         single_download_method = self._single_download_decoded if decode_data else self._single_download
+
         with ThreadPoolExecutor(max_workers=max_threads) as executor:
-            download_list = [executor.submit(single_download_method, request) for request in downloads]
+            download_list = [executor.submit(single_download_method, request) for request in requests_list]
             future_order = {future: i for i, future in enumerate(download_list)}
 
-            # Consider using tqdm.contrib.concurrent.thread_map in the future
-            if show_progress:
-                with tqdm(total=len(download_list)) as pbar:
-                    for future in as_completed(download_list):
-                        data_list[future_order[future]] = self._process_download_future(future)
-                        pbar.update(1)
-            else:
+            progress_context = tqdm(total=len(download_list)) if show_progress else nullcontext()
+            with progress_context as progress_bar:
                 for future in as_completed(download_list):
-                    data_list[future_order[future]] = self._process_download_future(future)
+                    try:
+                        results[future_order[future]] = future.result()
+                    except DownloadFailedException as download_exception:
+                        if self.raise_download_errors:
+                            raise download_exception
+
+                        warnings.warn(str(download_exception), category=SHRuntimeWarning)
+
+                    if progress_bar:
+                        progress_bar.update(1)
 
         if isinstance(download_requests, DownloadRequest):
-            return data_list[0]
-        return data_list
-
-    def _process_download_future(self, future: Future) -> Any:
-        """Unpacks the future and correctly handles exceptions"""
-        try:
-            return future.result()
-        except DownloadFailedException as download_exception:
-            if self.raise_download_errors:
-                traceback = sys.exc_info()[2]
-                raise download_exception.with_traceback(traceback)
-
-            warnings.warn(str(download_exception), category=SHRuntimeWarning)
-            return None
+            return results[0]  # type: ignore[return-value] # will be removed in future version
+        return results
 
     def _single_download_decoded(self, request: DownloadRequest) -> Any:
         """Downloads a response and decodes it into data. By decoding a single response"""

sentinelhub/download/handlers.py

@@ -1,6 +1,7 @@
 """
 Module implementing error handlers which can occur during download procedure
 """
+import functools
 import logging
 import sys
 import time
@@ -36,6 +37,7 @@ class _HasConfig(Protocol):
 def fail_user_errors(download_func: Callable[[Self, DownloadRequest], T]) -> Callable[[Self, DownloadRequest], T]:
     """Decorator function for handling user errors"""
 
+    @functools.wraps(download_func)
     def new_download_func(self: Self, request: DownloadRequest) -> T:
         try:
             return download_func(self, request)
@@ -58,14 +60,17 @@ def retry_temporary_errors(
     """Decorator function for handling server and connection errors"""
     backoff_coefficient = 3
 
+    @functools.wraps(download_func)
     def new_download_func(self: SelfWithConfig, request: DownloadRequest) -> T:
         download_attempts = self.config.max_download_attempts
         sleep_time = self.config.download_sleep_time
 
-        for attempt_num in range(download_attempts):
+        for attempt_idx in range(download_attempts):
             try:
                 return download_func(self, request)
+
             except requests.RequestException as exception:
+                attempts_left = download_attempts - (attempt_idx + 1)
                 if not (
                     _is_temporary_problem(exception)
                     or (
@@ -75,14 +80,14 @@ def new_download_func(self: SelfWithConfig, request: DownloadRequest) -> T:
                 ):
                     raise exception from exception
 
-                if attempt_num == download_attempts - 1:
+                if attempts_left <= 0:
                     message = _create_download_failed_message(exception, request.url)
                     raise DownloadFailedException(message, request_exception=exception) from exception
 
                 LOGGER.debug(
                     "Download attempt failed: %s\n%d attempts left, will retry in %ds",
                     exception,
-                    download_attempts - attempt_num - 1,
+                    attempts_left,
                     sleep_time,
                 )
                 time.sleep(sleep_time)
@@ -98,6 +103,7 @@ def new_download_func(self: SelfWithConfig, request: DownloadRequest) -> T:
 def fail_missing_file(download_func: Callable[[Self, DownloadRequest], T]) -> Callable[[Self, DownloadRequest], T]:
     """A decorator for raising an error if a file is missing"""
 
+    @functools.wraps(download_func)
     def new_download_func(self: Self, request: DownloadRequest) -> T:
         try:
             return download_func(self, request)

sentinelhub/download/rate_limit.py

@@ -3,7 +3,6 @@
 """
 import time
 from enum import Enum
-from typing import Union
 
 from ..types import JsonDict
 
@@ -52,8 +51,7 @@ def register_next(self) -> float:
 
     def update(self, headers: dict) -> None:
         """Update the next possible download time if the service has responded with the rate limit"""
-        retry_after: float
-        retry_after = int(headers.get(self.RETRY_HEADER, 0))
+        retry_after: float = round(headers.get(self.RETRY_HEADER, 0))
         retry_after = retry_after / 1000
 
         if retry_after:
@@ -63,7 +61,7 @@ def update(self, headers: dict) -> None:
 class PolicyBucket:
     """A class representing Sentinel Hub policy bucket"""
 
-    def __init__(self, policy_type: Union[str, PolicyType], policy_payload: JsonDict):
+    def __init__(self, policy_type: PolicyType, policy_payload: JsonDict):
         """
         :param policy_type: A type of policy
         :param policy_payload: A dictionary of policy parameters
@@ -77,7 +75,7 @@ def __init__(self, policy_type: Union[str, PolicyType], policy_payload: JsonDict
         # The following is the same as if we would interpret samplingPeriod string
         self.refill_per_second = 10**9 / policy_payload["nanosBetweenRefills"]
 
-        self._content = self.capacity
+        self.content = self.capacity
 
     def __repr__(self) -> str:
         """Representation of the bucket content"""
@@ -86,16 +84,6 @@ def __repr__(self) -> str:
             f"refill_period={self.refill_period}, refill_per_second={self.refill_per_second})"
         )
 
-    @property
-    def content(self) -> float:
-        """Variable `content` can be accessed as a property"""
-        return self._content
-
-    @content.setter
-    def content(self, value: float) -> None:
-        """Variable `content` can be modified by external classes"""
-        self._content = value
-
     def count_cost_per_second(self, elapsed_time: float, new_content: float) -> float:
         """Calculates the cost per second for the bucket given the elapsed time and the new content.
 
@@ -121,9 +109,7 @@ def get_wait_time(
         expected_content = max(self.content + elapsed_time * self.refill_per_second - overall_completed_cost, 0)
 
         if self.is_fixed():
-            if expected_content < cost_per_request:
-                return -1
-            return 0
+            return -1 if expected_content < cost_per_request else 0
 
         return max(cost_per_request - expected_content + buffer_cost, 0) / self.refill_per_second
 

sentinelhub/testing_utils.py

@@ -2,7 +2,7 @@
 Utility tools for writing unit tests for packages which rely on `sentinelhub-py`
 """
 import os
-from typing import Any, Callable, Dict, Optional, Tuple
+from typing import Any, Callable, Dict, Optional, Tuple, Union
 
 import numpy as np
 from pytest import approx
@@ -24,7 +24,7 @@ def get_output_folder(current_file: str) -> str:
 def test_numpy_data(
     data: Optional[np.ndarray] = None,
     exp_shape: Optional[Tuple[int, ...]] = None,
-    exp_dtype: Optional[np.dtype] = None,
+    exp_dtype: Union[None, type, np.dtype] = None,
     exp_min: Optional[float] = None,
     exp_max: Optional[float] = None,
     exp_mean: Optional[float] = None,
@@ -43,7 +43,7 @@ def test_numpy_data(
 def assert_statistics_match(
     data: np.ndarray,
     exp_shape: Optional[Tuple[int, ...]] = None,
-    exp_dtype: Optional[np.dtype] = None,
+    exp_dtype: Union[None, type, np.dtype] = None,
     exp_min: Optional[float] = None,
     exp_max: Optional[float] = None,
     exp_mean: Optional[float] = None,