diff --git a/docs/source/geopedia.rst b/docs/source/geopedia.rst index 0647d9a6..38a73c33 100644 --- a/docs/source/geopedia.rst +++ b/docs/source/geopedia.rst @@ -4,5 +4,11 @@ geopedia .. module:: sentinelhub.geopedia .. autoclass:: GeopediaService :members: +.. autoclass:: GeopediaSession + :members: +.. autoclass:: GeopediaWmsService + :members: .. autoclass:: GeopediaImageService :members: +.. autoclass:: GeopediaFeatureIterator + :members: diff --git a/sentinelhub/__init__.py b/sentinelhub/__init__.py index 64bff34e..36df4f39 100644 --- a/sentinelhub/__init__.py +++ b/sentinelhub/__init__.py @@ -2,7 +2,8 @@ This module lists all externally useful classes and functions """ -from .data_request import WmsRequest, WcsRequest, AwsTileRequest, AwsProductRequest, GeopediaWmsRequest, \ +from .data_request import WmsRequest, WcsRequest, AwsTileRequest, AwsProductRequest, \ + GeopediaWmsRequest, GeopediaImageRequest, \ get_safe_format, download_safe_format from .aws import AwsProduct, AwsTile @@ -11,6 +12,7 @@ from .areas import BBoxSplitter, OsmSplitter, TileSplitter from .ogc import WebFeatureService +from .geopedia import GeopediaFeatureIterator, GeopediaSession from .common import BBox from .constants import DataSource, CustomUrlParam, CRS, MimeType, OgcConstants, AwsConstants, ServiceType diff --git a/sentinelhub/_version.py b/sentinelhub/_version.py index d67e15ab..d915c607 100644 --- a/sentinelhub/_version.py +++ b/sentinelhub/_version.py @@ -2,4 +2,4 @@ Version of sentinelhub package """ -__version__ = "2.4.3" +__version__ = "2.4.4" diff --git a/sentinelhub/aws.py b/sentinelhub/aws.py index ecda3bf0..66fa9d01 100644 --- a/sentinelhub/aws.py +++ b/sentinelhub/aws.py @@ -107,7 +107,7 @@ def get_base_url(self, force_http=False): :return: base url string :rtype: str """ - base_url = SHConfig().aws_metadata_base_url.rstrip('/') if force_http else 's3:/' + base_url = SHConfig().aws_metadata_url.rstrip('/') if force_http else 's3:/' aws_bucket = SHConfig().aws_s3_l1c_bucket if self.data_source is DataSource.SENTINEL2_L1C else \ SHConfig().aws_s3_l2a_bucket diff --git a/sentinelhub/config.json b/sentinelhub/config.json index d16d158c..922e41d5 100644 --- a/sentinelhub/config.json +++ b/sentinelhub/config.json @@ -3,8 +3,9 @@ "aws_access_key_id": "", "aws_secret_access_key": "", "ogc_base_url": "https://services.sentinel-hub.com/ogc/", - "gpd_base_url": "http://service.geopedia.world/", - "aws_metadata_base_url": "https://roda.sentinel-hub.com/", + "geopedia_wms_url": "http://service.geopedia.world/", + "geopedia_rest_url": "https://www.geopedia.world/rest/", + "aws_metadata_url": "https://roda.sentinel-hub.com/", "aws_s3_l1c_bucket": "sentinel-s2-l1c", "aws_s3_l2a_bucket": "sentinel-s2-l2a", "opensearch_url": "http://opensearch.sentinel-hub.com/resto/api/collections/Sentinel2/", @@ -14,4 +15,4 @@ "max_download_attempts": 4, "download_sleep_time": 5, "download_timeout_seconds": 120 -} \ No newline at end of file +} diff --git a/sentinelhub/config.py b/sentinelhub/config.py index 8b273e88..5bd53c8e 100644 --- a/sentinelhub/config.py +++ b/sentinelhub/config.py @@ -17,9 +17,10 @@ class SHConfig: of specifying it explicitly every time he/she creates new ogc request. - `aws_access_key_id`: Access key for AWS Requester Pays buckets. - `aws_secret_access_key`: Secret access key for AWS Requester Pays buckets. - - `ogc_base_url`: Base url for Sentinel Hub's services (should not be changed by the user). - - `gpd_base_url`: Base url for Geopedia's services (should not be changed by the user). - - `aws_metadata_base_url`: Base url for publicly available metadata files + - `ogc_base_url`: Base url for Sentinel Hub's services. + - `geopedia_wms_url`: Base url for Geopedia WMS services. + - `geopedia_rest_url`: Base url for Geopedia REST services. + - `aws_metadata_url`: Base url for publicly available metadata files - `aws_s3_l1c_bucket`: Name of Sentinel-2 L1C bucket at AWS s3 service. - `aws_s3_l2a_bucket`: Name of Sentinel-2 L2A bucket at AWS s3 service. - `opensearch_url`: Base url for Sentinelhub Opensearch service. @@ -46,8 +47,9 @@ class _SHConfig: ('aws_access_key_id', ''), ('aws_secret_access_key', ''), ('ogc_base_url', 'https://services.sentinel-hub.com/ogc/'), - ('gpd_base_url', 'http://service.geopedia.world/'), - ('aws_metadata_base_url', 'https://roda.sentinel-hub.com/'), + ('geopedia_wms_url', 'http://service.geopedia.world/'), + ('geopedia_rest_url', 'https://www.geopedia.world/rest/'), + ('aws_metadata_url', 'https://roda.sentinel-hub.com/'), ('aws_s3_l1c_bucket', 'sentinel-s2-l1c'), ('aws_s3_l2a_bucket', 'sentinel-s2-l2a'), ('opensearch_url', 'http://opensearch.sentinel-hub.com/resto/api/collections/Sentinel2/'), diff --git a/sentinelhub/constants.py b/sentinelhub/constants.py index 7896f521..d38e20cc 100644 --- a/sentinelhub/constants.py +++ b/sentinelhub/constants.py @@ -27,12 +27,13 @@ def get_version(): class ServiceType(Enum): """ Enum constant class for type of service - Supported types are WMS, WCS, WFS, AWS + Supported types are WMS, WCS, WFS, AWS, IMAGE """ WMS = 'wms' WCS = 'wcs' WFS = 'wfs' AWS = 'aws' + IMAGE = 'image' class _DataSourceMeta(EnumMeta): @@ -510,6 +511,13 @@ def get_expected_max_value(self): except IndexError: raise ValueError('Type {} is not supported by this method'.format(self)) + @staticmethod + def from_string(mime_type_str): + if mime_type_str == 'jpeg': + return MimeType.JPG + + return MimeType(mime_type_str) + class RequestType(Enum): """ Enum constant class for GET/POST request type """ diff --git a/sentinelhub/data_request.py b/sentinelhub/data_request.py index 561cc99e..34ba0912 100644 --- a/sentinelhub/data_request.py +++ b/sentinelhub/data_request.py @@ -12,7 +12,7 @@ from copy import deepcopy from .ogc import OgcImageService -from .geopedia import GeopediaImageService +from .geopedia import GeopediaWmsService, GeopediaImageService from .aws import AwsProduct, AwsTile from .aws_safe import SafeProduct, SafeTile from .download import download_data, ImageDecodingError, DownloadFailedException @@ -38,10 +38,10 @@ def __init__(self, *, data_folder=None): self.download_list = [] self.folder_list = [] - self.create_request() + self._create_request() @abstractmethod - def create_request(self): + def _create_request(self): raise NotImplementedError def get_download_list(self): @@ -314,7 +314,7 @@ def _check_custom_url_parameters(self): if param not in CustomUrlParam: raise ValueError('Parameter %s is not a valid custom url parameter. Please check and fix.' % param) - def create_request(self): + def _create_request(self): """Set download requests Create a list of DownloadRequests for all Sentinel-2 acquisitions within request's time interval and @@ -472,51 +472,77 @@ def __init__(self, *, resx='10m', resy='10m', **kwargs): class GeopediaRequest(DataRequest): - """ The base class for Geopedia's OGC-type requests (WMS and WCS) where all common parameters are - defined. + """ The base class for Geopedia requests where all common parameters are defined. - :param service_type: type of OGC service (WMS or WCS) + :param layer: Geopedia layer which contains requested data + :type layer: str + :param service_type: Type of the service, supported are ``ServiceType.WMS`` and ``ServiceType.IMAGE`` :type service_type: constants.ServiceType - :param size_x: number of pixels in x or resolution in x (i.e. ``512`` or ``10m``) - :type size_x: int or str - :param size_y: number of pixels in x or resolution in y (i.e. ``512`` or ``10m``) - :type size_y: int or str - :param bbox: Bounding box of the requested image. Coordinates must be in the specified coordinate reference system. + :param bbox: Bounding box of the requested data :type bbox: common.BBox - :param layer: the preconfigured layer (image) to be returned as comma separated layer names. Required. - :type layer: str :param theme: Geopedia's theme for which the layer is defined. :type theme: str - :param custom_url_params: dictionary of CustomUrlParameters and their values supported by Geopedia's WMS services. - At the moment only the transparancy is supported (CustomUrlParam.TRANSPARENT). - :type custom_url_params: dictionary of CustomUrlParameter enum and its value, i.e. - ``{constants.CustomUrlParam.TRANSPARENT:True}`` - :param image_format: format of the returned image by the Sentinel Hub's WMS getMap service. Default is PNG, but - in some cases 32-bit TIFF is required, i.e. if requesting unprocessed raw bands. - Default is ``constants.MimeType.PNG``. + :param image_format: Format of the returned image by the Sentinel Hub's WMS getMap service. Default is + ``constants.MimeType.PNG``. :type image_format: constants.MimeType - :param data_folder: location of the directory where the fetched data will be saved. + :param data_folder: Location of the directory where the fetched data will be saved. :type data_folder: str """ - def __init__(self, layer, bbox, theme, *, service_type=None, size_x=None, size_y=None, custom_url_params=None, - image_format=MimeType.PNG, **kwargs): + def __init__(self, layer, service_type, *, bbox=None, theme=None, image_format=MimeType.PNG, **kwargs): self.layer = layer - self.theme = theme + self.service_type = service_type + self.bbox = bbox + if bbox.crs is not CRS.POP_WEB: + raise ValueError('Geopedia Request at the moment supports only bounding boxes with coordinates in ' + '{}'.format(CRS.POP_WEB)) + + self.theme = theme self.image_format = MimeType(image_format) - self.service_type = service_type - self.size_x = size_x - self.size_y = size_y + + super().__init__(**kwargs) + + @abstractmethod + def _create_request(self): + raise NotImplementedError + + +class GeopediaWmsRequest(GeopediaRequest): + """ Web Map Service request class for Geopedia + + Creates an instance of Geopedia's WMS (Web Map Service) GetMap request, which provides access to WMS layers in + Geopedia. + + :param width: width (number of columns) of the returned image (array) + :type width: int or None + :param height: height (number of rows) of the returned image (array) + :type height: int or None + :param custom_url_params: dictionary of CustomUrlParameters and their values supported by Geopedia's WMS services. + At the moment only the transparency is supported (CustomUrlParam.TRANSPARENT). + :type custom_url_params: dictionary of CustomUrlParameter enum and its value, i.e. + ``{constants.CustomUrlParam.TRANSPARENT:True}`` + :param layer: Geopedia layer which contains requested data + :type layer: str + :param bbox: Bounding box of the requested data + :type bbox: common.BBox + :param theme: Geopedia's theme for which the layer is defined. + :type theme: str + :param image_format: Format of the returned image by the Sentinel Hub's WMS getMap service. Default is + ``constants.MimeType.PNG``. + :type image_format: constants.MimeType + :param data_folder: Location of the directory where the fetched data will be saved. + :type data_folder: str + """ + def __init__(self, *, width=None, height=None, custom_url_params=None, **kwargs): + self.size_x = width + self.size_y = height self.custom_url_params = custom_url_params if self.custom_url_params is not None: self._check_custom_url_parameters() - if bbox.crs is not CRS.POP_WEB: - raise ValueError('Geopedia Request at the moment supports only CRS = {}'.format(CRS.POP_WEB)) - - super().__init__(**kwargs) + super().__init__(service_type=ServiceType.WMS, **kwargs) def _check_custom_url_parameters(self): """Checks if custom url parameters are valid parameters. @@ -524,44 +550,66 @@ def _check_custom_url_parameters(self): Throws ValueError if the provided parameter is not a valid parameter. """ for param in self.custom_url_params.keys(): - if param not in [CustomUrlParam.TRANSPARENT]: - raise ValueError('Parameter %s is not a valid custom url parameter. Please check and fix.' % param) + if param is not CustomUrlParam.TRANSPARENT: + raise ValueError('Parameter {} is currently not supported.'.format(param)) - def create_request(self): + def _create_request(self): """Set download requests Create a list of DownloadRequests for all Sentinel-2 acquisitions within request's time interval and acceptable cloud coverage. """ - gpd_service = GeopediaImageService() + gpd_service = GeopediaWmsService() self.download_list = gpd_service.get_request(self) -class GeopediaWmsRequest(GeopediaRequest): - """ Web Map Service request class for Geopedia - - Creates an instance of Geopedia's WMS (Web Map Service) GetMap request, - which provides access to various layers in Geopedia. +class GeopediaImageRequest(GeopediaRequest): + """Request to access data in a Geopedia vector / raster layer. - :param width: width (number of columns) of the returned image (array) - :type width: int or None - :param height: height (number of rows) of the returned image (array) - :type height: int or None - :param data_source: Source of requested satellite data. Default is Sentinel-2 L1C data. - :type data_source: constants.DataSource - :param bbox: Bounding box of the requested image. Coordinates must be in the specified coordinate reference system. - :type bbox: common.BBox - :param layer: the preconfigured layer (image) to be returned. Required. + :param image_field_name: Name of the field in the data table which holds images + :type image_field_name: str + :param keep_image_names: If ``True`` images will be saved with the same names as in Geopedia otherwise Geopedia + hashes will be used as names. If there are multiple images with the same names in the Geopedia layer this + parameter should be set to ``False`` to prevent images being overwritten. + :type keep_image_names: bool + :param layer: Geopedia layer which contains requested data :type layer: str + :param bbox: Bounding box of the requested data + :type bbox: common.BBox :param theme: Geopedia's theme for which the layer is defined. :type theme: str - :param image_format: format of the returned image by the Geopedia WMS getMap service. Default is PNG. + :param image_format: Format of the returned image by the Sentinel Hub's WMS getMap service. Default is + ``constants.MimeType.PNG``. :type image_format: constants.MimeType - :param data_folder: location of the directory where the fetched data will be saved. + :param data_folder: Location of the directory where the fetched data will be saved. :type data_folder: str """ - def __init__(self, *, width=None, height=None, **kwargs): - super().__init__(service_type=ServiceType.WMS, size_x=width, size_y=height, **kwargs) + def __init__(self, *, image_field_name, keep_image_names=True, **kwargs): + self.image_field_name = image_field_name + self.keep_image_names = keep_image_names + + self.gpd_iterator = None + + super().__init__(service_type=ServiceType.IMAGE, **kwargs) + + def _create_request(self): + """Set a list of download requests + + Set a list of DownloadRequests for all images that are under the + given property of the Geopedia's Vector layer. + """ + gpd_service = GeopediaImageService() + self.download_list = gpd_service.get_request(self) + self.gpd_iterator = gpd_service.get_gpd_iterator() + + def get_items(self): + """Returns iterator over info about data used for this request + + :return: Iterator of dictionaries containing info about data used in + this request. + :rtype: Iterator[dict] or None + """ + return self.gpd_iterator class AwsRequest(DataRequest): @@ -592,7 +640,7 @@ def __init__(self, *, bands=None, metafiles=None, safe_format=False, **kwargs): super().__init__(**kwargs) @abstractmethod - def create_request(self): + def _create_request(self): raise NotImplementedError def get_aws_service(self): @@ -632,7 +680,7 @@ def __init__(self, product_id, *, tile_list=None, **kwargs): super().__init__(**kwargs) - def create_request(self): + def _create_request(self): if self.safe_format: self.aws_service = SafeProduct(self.product_id, tile_list=self.tile_list, bands=self.bands, metafiles=self.metafiles) @@ -680,7 +728,7 @@ def __init__(self, *, tile=None, time=None, aws_index=None, data_source=DataSour super().__init__(**kwargs) - def create_request(self): + def _create_request(self): if self.safe_format: self.aws_service = SafeTile(self.tile, self.time, self.aws_index, bands=self.bands, metafiles=self.metafiles, data_source=self.data_source) diff --git a/sentinelhub/download.py b/sentinelhub/download.py index 2b513da6..1a68d90b 100644 --- a/sentinelhub/download.py +++ b/sentinelhub/download.py @@ -245,12 +245,16 @@ def execute_download_request(request): try_num -= 1 if try_num > 0 and (_is_temporal_problem(exception) or (isinstance(exception, requests.HTTPError) and - exception.response.status_code >= requests.status_codes.codes.INTERNAL_SERVER_ERROR)): + exception.response.status_code >= requests.status_codes.codes.INTERNAL_SERVER_ERROR) or + _request_limit_reached(exception)): LOGGER.debug('Download attempt failed: %s\n%d attempts left, will retry in %ds', exception, try_num, SHConfig().download_sleep_time) - time.sleep(SHConfig().download_sleep_time) + sleep_time = SHConfig().download_sleep_time + if _request_limit_reached(exception): + sleep_time = max(sleep_time, 60) + time.sleep(sleep_time) else: - if request.url.startswith(SHConfig().aws_metadata_base_url) and \ + if request.url.startswith(SHConfig().aws_metadata_url) and \ isinstance(exception, requests.HTTPError) and \ exception.response.status_code == requests.status_codes.codes.NOT_FOUND: raise AwsDownloadFailedException('File in location %s is missing' % request.url) @@ -328,6 +332,19 @@ def _is_temporal_problem(exception): return isinstance(exception, requests.ConnectionError) +def _request_limit_reached(exception): + """ Checks if exception was raised because of too many executed requests. (This is a temporal solution and + will be changed in later package versions.) + + :param exception: Exception raised during download + :type exception: Exception + :return: True if exception is caused because too many requests were executed at once and False otherwise + :rtype: bool + """ + return isinstance(exception, requests.HTTPError) and \ + exception.response.status_code == requests.status_codes.codes.TOO_MANY_REQUESTS + + def _create_download_failed_message(exception, url): """ Creates message describing why download has failed diff --git a/sentinelhub/geopedia.py b/sentinelhub/geopedia.py index f6ce2e28..0e0ea026 100644 --- a/sentinelhub/geopedia.py +++ b/sentinelhub/geopedia.py @@ -3,9 +3,15 @@ """ import logging +import datetime -from .ogc import OgcImageService +from shapely.geometry import shape as geo_shape + +from .ogc import OgcImageService, MimeType from .config import SHConfig +from .download import DownloadRequest, get_json +from .constants import CRS +from .geo_utils import transform_bbox LOGGER = logging.getLogger(__name__) @@ -13,24 +19,75 @@ class GeopediaService: """ The class for Geopedia OGC services - :param base_url: Base url of Geopedia's OGC services. If ``None``, the url specified in the configuration + :param base_url: Base url of Geopedia REST services. If ``None``, the url specified in the configuration file is taken. :type base_url: str or None """ def __init__(self, base_url=None): - self.base_url = SHConfig().gpd_base_url if base_url is None else base_url + self.base_url = SHConfig().geopedia_rest_url if base_url is None else base_url + + +class GeopediaSession(GeopediaService): + """ For retrieving data from Geopedia vector and raster layers it is required to make a session. This class handles + starting and renewing of session. It provides session headers required by Geopedia REST requests. + + The session is created globally for all instances of this class. At the moment session duration is hardcoded to 1 + hour. After that this class will renew the session. + """ + SESSION_DURATION = datetime.timedelta(hours=1) + + _session_id = None + _session_end_timestamp = None + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.session_url = '{}data/v1/session/create?locale=en'.format(self.base_url) + + def get_session_headers(self, force_new_session=False): + """ Returns session headers + + :param force_new_session: If ``True`` it will always create a new session. Otherwise it will create a new + session only if no session exists or the previous session timed out. + :type force_new_session: bool + :return: A dictionary containing session headers + :rtype: dict + """ + return { + 'X-GPD-Session': self.get_session_id(force_new_session=force_new_session) + } + + def get_session_id(self, force_new_session=False): + """ Returns a session ID + + :param force_new_session: If ``True`` it will always create a new session. Otherwise it will create a new + session only if no session exists or the previous session timed out. + :type force_new_session: bool + :return: A session ID string + :rtype: str + """ + if self._session_id is None or force_new_session or datetime.datetime.now() > self._session_end_timestamp: + self._start_new_session() + + return self._session_id + + def _start_new_session(self): + """ Updates the session id and calculates when the new session will end. + """ + self._session_end_timestamp = datetime.datetime.now() + self.SESSION_DURATION + self._session_id = get_json(self.session_url)['sessionId'] -class GeopediaImageService(GeopediaService, OgcImageService): +class GeopediaWmsService(GeopediaService, OgcImageService): """Geopedia OGC services class for providing image data. Most of the methods are inherited from `sentinelhub.ogc.OgcImageService` class. - :param base_url: Base url of Geopedia's OGC services. If ``None``, the url specified in the configuration + :param base_url: Base url of Geopedia WMS services. If ``None``, the url specified in the configuration file is taken. :type base_url: str or None """ - def __init__(self, **kwargs): - super().__init__(**kwargs) + def __init__(self, base_url=None): + super().__init__(base_url=SHConfig().geopedia_wms_url if base_url is None else base_url) def get_dates(self, request): """ Geopedia does not support date queries @@ -46,3 +103,143 @@ def get_wfs_iterator(self): """ This method is inherited from OgcImageService but is not implemented. """ raise NotImplementedError + + +class GeopediaImageService(GeopediaService): + """Service class that provides images from a Geopedia vector layer. + + :param base_url: Base url of Geopedia REST services. If ``None``, the url + specified in the configuration file is taken. + :type base_url: str or None + """ + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.gpd_iterator = None + + def get_request(self, request): + """Get download requests + + Get a list of DownloadRequests for all data that are under the given field of the Geopedia Vector layer. + + :return: list of DownloadRequests + """ + return [DownloadRequest(url=self._get_url(item), + filename=self._get_filename(request, item), + data_type=request.image_format) + for item in self._get_items(request)] + + def _get_items(self, request): + self.gpd_iterator = GeopediaFeatureIterator(request.layer, bbox=request.bbox, base_url=self.base_url) + + field_iter = self.gpd_iterator.get_field_iterator(request.image_field_name) + items = [] + + for field_items in field_iter: # an image field can have multiple images + + for item in field_items: + if not item['mimeType'].startswith('image/'): + continue + + mime_type = MimeType.from_string(item['mimeType'][6:]) + + if mime_type is request.image_format: + items.append(item) + + return items + + @staticmethod + def _get_url(item): + return item.get('objectPath') + + @staticmethod + def _get_filename(request, item): + """ Creates a filename + """ + if request.keep_image_names: + filename = OgcImageService.finalize_filename(item['niceName'].replace(' ', '_')) + else: + filename = OgcImageService.finalize_filename( + '_'.join([str(request.layer), item['objectPath'].rsplit('/', 1)[-1]]), + request.image_format + ) + + LOGGER.debug("filename=%s", filename) + return filename + + def get_gpd_iterator(self): + """Returns iterator over info about data used for the + GeopediaVectorRequest + + :return: Iterator of dictionaries containing info about data used in the request. + :rtype: Iterator[dict] or None + """ + return self.gpd_iterator + + +class GeopediaFeatureIterator(GeopediaService): + """Iterator for Geopedia Vector Service + + :type layer: str + :param bbox: Bounding box of the requested image. Its coordinates must be + in the CRS.POP_WEB (EPSG:3857) coordinate system. + :type bbox: common.BBox + :param base_url: Base url of Geopedia REST services. If ``None``, the url specified in the configuration + file is taken. + :type base_url: str or None + """ + def __init__(self, layer, bbox=None, **kwargs): + super().__init__(**kwargs) + + self.layer = layer + + self.query = {} + if bbox is not None: + if bbox.crs is not CRS.POP_WEB: + bbox = transform_bbox(bbox, CRS.POP_WEB) + + self.query['filterExpression'] = 'bbox({},"EPSG:3857")'.format(bbox) + + self.gpd_session = GeopediaSession() + self.features = [] + self.index = 0 + + self.next_page_url = '{}data/v2/search/tables/{}/features'.format(self.base_url, layer) + + def __iter__(self): + self.index = 0 + + return self + + def __next__(self): + if self.index == len(self.features): + self._fetch_features() + + if self.index < len(self.features): + self.index += 1 + return self.features[self.index - 1] + + raise StopIteration + + def _fetch_features(self): + """ Retrieves a new page of features from Geopedia + """ + if self.next_page_url is None: + return + + response = get_json(self.next_page_url, post_values=self.query, headers=self.gpd_session.get_session_headers()) + + self.features.extend(response['features']) + self.next_page_url = response['pagination']['next'] + + def get_geometry_iterator(self): + """ Iterator over Geopedia feature geometries + """ + for feature in self: + yield geo_shape(feature['geometry']) + + def get_field_iterator(self, field): + """ Iterator over the specified field of Geopedia features + """ + for feature in self: + yield feature['properties'].get(field, []) diff --git a/sentinelhub/ogc.py b/sentinelhub/ogc.py index 5b961fb1..586d59b3 100644 --- a/sentinelhub/ogc.py +++ b/sentinelhub/ogc.py @@ -230,16 +230,31 @@ def get_filename(request, date, size_x, size_y): key=lambda parameter_item: parameter_item[0].value): filename = '_'.join([filename, param.value, str(value)]) + return OgcImageService.finalize_filename(filename, request.image_format) + + @staticmethod + def finalize_filename(filename, image_format=None): + """ Replaces invalid characters in filename string, adds image extension and reduces filename length + + :param filename: Incomplete filename string + :type filename: str + :param image_format: Format which will be used for filename extension + :type image_format: MimeType + :return: Final filename string + :rtype: str + """ for char in [' ', '/', '\\', '|', ';', ':', '\n', '\t']: filename = filename.replace(char, '') - suffix = str(request.image_format.value) - if request.image_format.is_tiff_format() and request.image_format is not MimeType.TIFF: - suffix = str(MimeType.TIFF.value) - filename = '_'.join([filename, str(request.image_format.value).replace(';', '_')]) + if image_format: + suffix = str(image_format.value) + if image_format.is_tiff_format() and image_format is not MimeType.TIFF: + suffix = str(MimeType.TIFF.value) + filename = '_'.join([filename, str(image_format.value).replace(';', '_')]) + + filename = '.'.join([filename[:254 - len(suffix)], suffix]) - filename = '.'.join([filename[:254 - len(suffix)], suffix]) - return filename # Even in UNIX systems filename must have at most 255 bytes + return filename # Even in UNIX systems filename must have at most 255 bytes def get_dates(self, request): """ Get available Sentinel-2 acquisitions at least time_difference apart diff --git a/tests/test_geopedia.py b/tests/test_geopedia.py index 5e463b24..645d295d 100644 --- a/tests/test_geopedia.py +++ b/tests/test_geopedia.py @@ -3,10 +3,10 @@ from tests_all import TestSentinelHub -from sentinelhub import GeopediaWmsRequest, CRS, MimeType, BBox +from sentinelhub import GeopediaWmsRequest, GeopediaImageRequest, GeopediaFeatureIterator, CRS, MimeType, BBox -class TestOgc(TestSentinelHub): +class TestGeopediaWms(TestSentinelHub): @classmethod def setUpClass(cls): @@ -25,22 +25,66 @@ def test_return_type(self): "Expected a list of length {}, got length {}".format(data_len, len(self.data))) def test_stats(self): - delta = 1e-1 if np.issubdtype(self.data[0].dtype, np.integer) else 1e-4 - - min_val = np.amin(self.data[0]) - min_exp = 0 - self.assertAlmostEqual(min_exp, min_val, delta=delta, msg="Expected min {}, got {}".format(min_exp, min_val)) - max_val = np.amax(self.data[0]) - max_exp = 255 - self.assertAlmostEqual(max_exp, max_val, delta=delta, msg="Expected max {}, got {}".format(max_exp, max_val)) - mean_val = np.mean(self.data[0]) - mean_exp = 150.9248 - self.assertAlmostEqual(mean_exp, mean_val, delta=delta, - msg="Expected mean {}, got {}".format(mean_exp, mean_val)) - median_val = np.median(self.data[0]) - media_exp = 255 - self.assertAlmostEqual(media_exp, median_val, delta=delta, - msg="Expected median {}, got {}".format(media_exp, median_val)) + self.test_numpy_stats(np.array(self.data), exp_min=0, exp_max=255, exp_mean=150.9248, exp_median=255) + + +class TestGeopediaImageService(TestSentinelHub): + + @classmethod + def setUpClass(cls): + bbox = BBox(bbox=[(13520759, 437326), (13522689, 438602)], crs=CRS.POP_WEB) + cls.image_field_name = 'Masks' + + cls.gpd_request = GeopediaImageRequest(layer=1749, bbox=bbox, image_field_name=cls.image_field_name, + image_format=MimeType.PNG, data_folder=cls.OUTPUT_FOLDER) + cls.image_list = cls.gpd_request.get_data(save_data=True) + + def test_return_type(self): + self.assertTrue(isinstance(self.image_list, list), 'Expected a list, got {}'.format(type(self.image_list))) + + expected_len = 5 + self.assertEqual(len(self.image_list), expected_len, + "Expected a list of length {}, got length {}".format(expected_len, len(self.image_list))) + + def test_stats(self): + self.test_numpy_stats(np.array(self.image_list), exp_min=0, exp_max=255, exp_mean=66.88769, exp_median=0) + + def test_names(self): + filenames = self.gpd_request.get_filename_list() + image_stats = list(self.gpd_request.get_items())[0]['properties'][self.image_field_name] + + for filename, image_stat in zip(filenames, image_stats): + self.assertEqual(filename, image_stat['niceName'].replace(' ', '_'), 'Filenames dont match') + + +class TestGeopediaFeatureIterator(TestSentinelHub): + + @classmethod + def setUpClass(cls): + cls.bbox = BBox(bbox=[(2947363, 4629723), (3007595, 4669471)], crs=CRS.POP_WEB) + cls.bbox.transform(CRS.WGS84) + + def test_item_count(self): + gpd_iter = GeopediaFeatureIterator(1749, bbox=self.bbox) + data = list(gpd_iter) + minimal_data_len = 21 + + self.assertTrue(len(data) >= minimal_data_len, 'Expected at least {} results, got {}'.format(minimal_data_len, + len(data))) + + def test_without_bbox(self): + gpd_iter = GeopediaFeatureIterator(1749) + + minimal_data_len = 1000 + + for idx, class_item in enumerate(gpd_iter): + self.assertTrue(isinstance(class_item, dict), 'Expected at dictionary, got {}'.format(type(class_item))) + + if idx >= minimal_data_len - 1: + break + + self.assertEqual(gpd_iter.index, minimal_data_len, 'Expected at least {} results, ' + 'got {}'.format(minimal_data_len, gpd_iter.index)) if __name__ == '__main__': diff --git a/tests/test_ogc.py b/tests/test_ogc.py index cfc85867..9b979982 100644 --- a/tests/test_ogc.py +++ b/tests/test_ogc.py @@ -240,28 +240,31 @@ def test_filter(self): def test_stats(self): for test_case in self.test_cases: - delta = 1e-1 if np.issubdtype(test_case.data[0].dtype, np.integer) else 1e-4 - - if test_case.img_min is not None: - min_val = np.amin(test_case.data[0]) - with self.subTest(msg='Test case {}'.format(test_case.name)): - self.assertAlmostEqual(test_case.img_min, min_val, delta=delta, - msg="Expected min {}, got {}".format(test_case.img_min, min_val)) - if test_case.img_max is not None: - max_val = np.amax(test_case.data[0]) - with self.subTest(msg='Test case {}'.format(test_case.name)): - self.assertAlmostEqual(test_case.img_max, max_val, delta=delta, - msg="Expected max {}, got {}".format(test_case.img_max, max_val)) - if test_case.img_mean is not None: - mean_val = np.mean(test_case.data[0]) - with self.subTest(msg='Test case {}'.format(test_case.name)): - self.assertAlmostEqual(test_case.img_mean, mean_val, delta=delta, - msg="Expected mean {}, got {}".format(test_case.img_mean, mean_val)) - if test_case.img_median is not None: - median_val = np.median(test_case.data[0]) - with self.subTest(msg='Test case {}'.format(test_case.name)): - self.assertAlmostEqual(test_case.img_median, median_val, delta=delta, - msg="Expected median {}, got {}".format(test_case.img_median, median_val)) + self.test_numpy_stats(test_case.data[0], exp_min=test_case.img_min, exp_max=test_case.img_max, + exp_mean=test_case.img_mean, exp_median=test_case.img_median, + test_name=test_case.name) + # delta = 1e-1 if np.issubdtype(test_case.data[0].dtype, np.integer) else 1e-4 + # + # if test_case.img_min is not None: + # min_val = np.amin(test_case.data[0]) + # with self.subTest(msg='Test case {}'.format(test_case.name)): + # self.assertAlmostEqual(test_case.img_min, min_val, delta=delta, + # msg="Expected min {}, got {}".format(test_case.img_min, min_val)) + # if test_case.img_max is not None: + # max_val = np.amax(test_case.data[0]) + # with self.subTest(msg='Test case {}'.format(test_case.name)): + # self.assertAlmostEqual(test_case.img_max, max_val, delta=delta, + # msg="Expected max {}, got {}".format(test_case.img_max, max_val)) + # if test_case.img_mean is not None: + # mean_val = np.mean(test_case.data[0]) + # with self.subTest(msg='Test case {}'.format(test_case.name)): + # self.assertAlmostEqual(test_case.img_mean, mean_val, delta=delta, + # msg="Expected mean {}, got {}".format(test_case.img_mean, mean_val)) + # if test_case.img_median is not None: + # median_val = np.median(test_case.data[0]) + # with self.subTest(msg='Test case {}'.format(test_case.name)): + # self.assertAlmostEqual(test_case.img_median, median_val, delta=delta, + # msg="Expected median {}, got {}".format(test_case.img_median, median_val)) if __name__ == '__main__': diff --git a/tests/tests_all.py b/tests/tests_all.py index 747634b8..bf8cabc3 100644 --- a/tests/tests_all.py +++ b/tests/tests_all.py @@ -3,6 +3,8 @@ import shutil import logging +import numpy as np + from sentinelhub import SHConfig @@ -29,6 +31,21 @@ class TestSentinelHub(unittest.TestCase): def tearDownClass(cls): shutil.rmtree(cls.OUTPUT_FOLDER, ignore_errors=True) + def test_numpy_stats(self, data=None, exp_min=None, exp_max=None, exp_mean=None, exp_median=None, test_name=''): + """ Validates data over basic 4 statistics + """ + if data is None: + return + delta = 1e-1 if np.issubdtype(data.dtype, np.integer) else 1e-4 + + for exp_stat, stat_func, stat_name in [(exp_min, np.amin, 'min'), (exp_max, np.amax, 'max'), + (exp_mean, np.mean, 'mean'), (exp_median, np.median, 'median')]: + if exp_stat is not None: + stat_val = stat_func(data) + with self.subTest(msg='Test case {}'.format(test_name)): + self.assertAlmostEqual(stat_val, exp_stat, delta=delta, + msg='Expected {} {}, got {}'.format(stat_name, exp_stat, stat_val)) + if __name__ == '__main__': loader = unittest.TestLoader()