diff --git a/docs/conf.py b/docs/conf.py
index a10136491..b4e125407 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -250,6 +250,8 @@
"shapely": ("https://shapely.readthedocs.io/en/stable/", None),
}
+suppress_warnings = ["misc.copy_overwrite"]
+
def _build_finished(app, exception):
"""Post-build pages edit"""
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 41c8a78fb..52abea5e2 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -261,5 +261,5 @@ Plugin configuration
:members:
:member-order: bysource
:undoc-members:
- :exclude-members: priority, products, product_type_config, yaml_loader, from_mapping, from_yaml, update, validate,
- yaml_dumper, yaml_tag
+ :exclude-members: priority, product_type_config, yaml_loader, from_mapping, from_yaml, update, validate, yaml_dumper,
+ yaml_tag
diff --git a/eodag/config.py b/eodag/config.py
index 4252c4c13..69c1316ef 100644
--- a/eodag/config.py
+++ b/eodag/config.py
@@ -258,6 +258,41 @@ class DiscoverMetadata(TypedDict):
#: Path to the metadata in search result
metadata_path: str
+ class DiscoverProductTypes(TypedDict, total=False):
+ """Configuration for product types discovery"""
+
+ #: URL from which the product types can be fetched
+ fetch_url: Optional[str]
+ #: Type of the provider result
+ result_type: str
+ #: JsonPath to the list of product types
+ results_entry: Union[JSONPath, str]
+ #: Mapping for the product type id
+ generic_product_type_id: str
+ #: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider
+ #: result
+ generic_product_type_parsable_metadata: Dict[str, str]
+ #: Mapping for product type properties which can be parsed from the result that are not product type metadata
+ generic_product_type_parsable_properties: Dict[str, str]
+ #: URL to fetch data for a single collection
+ single_collection_fetch_url: str
+ #: Query string to be added to the fetch_url to filter for a collection
+ single_collection_fetch_qs: str
+ #: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url
+ single_product_type_parsable_metadata: Dict[str, str]
+
+ class DiscoverQueryables(TypedDict, total=False):
+ """Configuration for queryables discovery"""
+
+ #: URL to fetch the queryables valid for all product types
+ fetch_url: Optional[str]
+ #: URL to fetch the queryables for a specific product type
+ product_type_fetch_url: Optional[str]
+ #: Type of the result
+ result_type: str
+ #: JsonPath to retrieve the queryables from the provider result
+ results_entry: str
+
class OrderOnResponse(TypedDict):
"""Configuration for order on-response during download"""
@@ -279,7 +314,7 @@ class OrderStatusSuccess(TypedDict):
#: Success value for status response HTTP code
http_code: int
- class OrderStatusOrdered(TypedDict):
+ class OrderStatusOrdered(TypedDict, total=False):
"""
Configuration to identify order status ordered during download
"""
@@ -287,7 +322,7 @@ class OrderStatusOrdered(TypedDict):
#: HTTP code of the order status response
http_code: int
- class OrderStatusRequest(TypedDict):
+ class OrderStatusRequest(TypedDict, total=False):
"""
Order status request configuration
"""
@@ -297,7 +332,7 @@ class OrderStatusRequest(TypedDict):
#: Request hearders
headers: Dict[str, Any]
- class OrderStatusOnSuccess(TypedDict):
+ class OrderStatusOnSuccess(TypedDict, total=False):
"""Configuration for order status on-success during download"""
#: Whether a new search is needed on success or not
@@ -309,7 +344,7 @@ class OrderStatusOnSuccess(TypedDict):
#: Metadata-mapping to apply to the success status result
metadata_mapping: Dict[str, Union[str, List[str]]]
- class OrderStatus(TypedDict):
+ class OrderStatus(TypedDict, total=False):
"""Configuration for order status during download"""
#: Order status request configuration
@@ -325,6 +360,16 @@ class OrderStatus(TypedDict):
#: Configuration for order status on-success during download
on_success: PluginConfig.OrderStatusOnSuccess
+ class MetadataPreMapping(TypedDict, total=False):
+ """Configuration which can be used to simplify further metadata extraction"""
+
+ #: JsonPath of the metadata entry
+ metadata_path: str
+ #: Key to get the metadata id
+ metadata_path_id: str
+ #: Key to get the metadata value
+ metadata_path_value: str
+
#: :class:`~eodag.plugins.base.PluginTopic` The name of the plugin class to use to instantiate the plugin object
name: str
#: :class:`~eodag.plugins.base.PluginTopic` Plugin type
@@ -335,12 +380,12 @@ class OrderStatus(TypedDict):
s3_bucket: str
#: :class:`~eodag.plugins.base.PluginTopic` Authentication error codes
auth_error_code: Union[int, List[int]]
+ #: :class:`~eodag.plugins.base.PluginTopic` Time to wait until request timeout in seconds
+ timeout: float
# search & api -----------------------------------------------------------------------------------------------------
# copied from ProviderConfig in PluginManager.get_search_plugins()
priority: int
- # copied from ProviderConfig in PluginManager.get_search_plugins()
- products: Dict[str, Any]
# per product type metadata-mapping, set in core._prepare_search
product_type_config: Dict[str, Any]
@@ -360,19 +405,35 @@ class OrderStatus(TypedDict):
#: :class:`~eodag.plugins.search.base.Search` Configuration for the metadata auto-discovery
discover_metadata: PluginConfig.DiscoverMetadata
#: :class:`~eodag.plugins.search.base.Search` Configuration for the product types auto-discovery
- discover_product_types: Dict[str, Any]
+ discover_product_types: PluginConfig.DiscoverProductTypes
#: :class:`~eodag.plugins.search.base.Search` Configuration for the queryables auto-discovery
- discover_queryables: Dict[str, Any]
+ discover_queryables: PluginConfig.DiscoverQueryables
#: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
metadata_mapping: Dict[str, Union[str, List[str]]]
#: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables
constraints_file_url: str
+ #: :class:`~eodag.plugins.search.base.Search`
+ #: Key which is used in the eodag configuration to map the eodag product type to the provider product type
+ constraints_file_dataset_key: str
+ #: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found
+ constraints_entry: str
+ #: :class:`~eodag.plugins.search.base.Search`
+ #: Whether only a provider result containing constraints_entry is accepted as valid and used to create constraints
+ #: or not
+ stop_without_constraints_entry_key: bool
#: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
remove_from_queryables: List[str]
+ #: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string
+ literal_search_params: Dict[str, str]
+ #: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params
+ dont_quote: List[str]
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build
free_text_search_operations: Dict[str, Any]
+ #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search
+ #: result and a two step search has to be performed
+ per_product_metadata_query: bool
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict used to simplify further metadata extraction
- metadata_pre_mapping: Dict[str, Any]
+ metadata_pre_mapping: PluginConfig.MetadataPreMapping
#: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to which the data request shall be sent
data_request_url: str
#: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to fetch the status of the data request
@@ -389,8 +450,6 @@ class OrderStatus(TypedDict):
#: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
#: Maximum number of connections for HTTP requests
max_connections: int
- #: :class:`~eodag.plugins.search.base.Search` Time to wait until request timeout in seconds
- timeout: float
#: :class:`~eodag.plugins.search.build_search_result.BuildSearchResult`
#: Whether end date should be excluded from search request or not
end_date_excluded: bool
@@ -410,6 +469,12 @@ class OrderStatus(TypedDict):
output_extension: str
#: :class:`~eodag.plugins.download.base.Download` Whether the directory structure should be flattened or not
flatten_top_dirs: bool
+ #: :class:`~eodag.plugins.download.base.Download` Level in extracted path tree where to find data
+ archive_depth: int
+ #: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``downloadLink`` or not
+ ignore_assets: bool
+ #: :class:`~eodag.plugins.download.base.Download` Product type specific configuration
+ products: Dict[str, Dict[str, Any]]
#: :class:`~eodag.plugins.download.http.HTTPDownload` Whether the product has to be ordered to download it or not
order_enabled: bool
#: :class:`~eodag.plugins.download.http.HTTPDownload` HTTP request method for the order request
@@ -425,6 +490,8 @@ class OrderStatus(TypedDict):
#: :class:`~eodag.plugins.download.http.HTTPDownload`
#: Do not authenticate the download request but only the order and order status ones
no_auth_download: bool
+ #: :class:`~eodag.plugins.download.http.HTTPDownload` Parameters to be added to the query params of the request
+ dl_url_params: Dict[str, str]
#: :class:`~eodag.plugins.download.s3rest.S3RestDownload`
#: At which level of the path part of the url the bucket can be found
bucket_path_level: int
diff --git a/eodag/plugins/apis/ecmwf.py b/eodag/plugins/apis/ecmwf.py
index 7ac0ea212..76d1d1567 100644
--- a/eodag/plugins/apis/ecmwf.py
+++ b/eodag/plugins/apis/ecmwf.py
@@ -68,10 +68,19 @@ class EcmwfApi(Api, BuildPostSearchResult):
is in query), or on MARS Operational Archive (if ``dataset`` parameter is not in
query).
- This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility,
- :class:`~eodag.plugins.download.base.Download` for download methods, and
- :class:`~eodag.plugins.search.qssearch.QueryStringSearch` for metadata-mapping and
- query build methods.
+ This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility and
+ :class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult` for the creation
+ of the search result.
+
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): EcmwfApi
+ * **api_endpoint** [str] (mandatory): url of the ecmwf api
+ * **metadata_mapping** [Dict[str, Union[str, list]]]: how parameters should be mapped between
+ the provider and eodag; If a string is given, this is the mapping parameter returned by
+ provider -> eodag parameter. If a list with 2 elements is given, the first one is the mapping
+ eodag parameter -> provider query parameters and the second one the mapping provider result
+ parameter -> eodag parameter
"""
def __init__(self, provider: str, config: PluginConfig) -> None:
diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py
index 4e7874107..d981ec017 100644
--- a/eodag/plugins/apis/usgs.py
+++ b/eodag/plugins/apis/usgs.py
@@ -68,7 +68,22 @@
class UsgsApi(Api):
- """A plugin that enables to query and download data on the USGS catalogues"""
+ """A plugin that enables to query and download data on the USGS catalogues
+
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): UsgsApi
+ * **pagination** [Dict[str, Any]] (mandatory): dict containing parameters for pagination; should contain the
+ key total_items_nb_key_path which is indicating the key for the number of total items in the provider result
+ * **ssl_verify** [bool]: if the ssl certificates should be verified in the download request; default: True
+ * **need_auth** [bool]: if authentication is required for search; default: False
+ * **extract** [bool]: if the content of the downloaded file should be extracted; default: True
+ * **order_enabled** [bool]: if the product has to be ordered to download it; default: False
+ * **metadata_mapping** [Dict[str, Union[str, list]]]: how parameters should be mapped between the provider and eodag
+ If a string is given, this is the mapping parameter returned by provider -> eodag parameter. If a list with
+ 2 elements is given, the first one is the mapping eodag parameter -> provider query parameters and the second one
+ the mapping provider result parameter -> eodag parameter
+ """
def __init__(self, provider: str, config: PluginConfig) -> None:
super(UsgsApi, self).__init__(provider, config)
diff --git a/eodag/plugins/authentication/aws_auth.py b/eodag/plugins/authentication/aws_auth.py
index 4ca0f9c72..02f52abf5 100644
--- a/eodag/plugins/authentication/aws_auth.py
+++ b/eodag/plugins/authentication/aws_auth.py
@@ -30,14 +30,22 @@
class AwsAuth(Authentication):
"""AWS authentication plugin
- Authentication will use the first valid method within the following ones:
+ Authentication will use the first valid method within the following ones depending on which
+ parameters are available in the configuration:
- - auth anonymously using no-sign-request
- - auth using ``aws_profile``
- - auth using ``aws_access_key_id`` and ``aws_secret_access_key``
+ * auth anonymously using no-sign-request
+ * auth using ``aws_profile``
+ * auth using ``aws_access_key_id`` and ``aws_secret_access_key``
(optionally ``aws_session_token``)
- - auth using current environment (AWS environment variables and/or ``~/aws/*``),
+ * auth using current environment (AWS environment variables and/or ``~/aws/*``),
will be skipped if AWS credentials are filled in eodag conf
+
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): AwsAuth
+ * **auth_error_code** [int] (mandatory for creodias_s3): which error code is returned
+ in case of an authentication error
+
"""
s3_client: S3Client
diff --git a/eodag/plugins/authentication/generic.py b/eodag/plugins/authentication/generic.py
index 9ec482280..b1da2f912 100644
--- a/eodag/plugins/authentication/generic.py
+++ b/eodag/plugins/authentication/generic.py
@@ -29,7 +29,16 @@
class GenericAuth(Authentication):
- """GenericAuth authentication plugin"""
+ """GenericAuth authentication plugin (authentication using username and password)
+
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): GenericAuth
+ * **method** [str]: specifies if digest authentication (digest) or basic authentication
+ (basic) should be used; default: basic
+
+ The mandatory parameters that have to be added in the eodag config are username and password.
+ """
def authenticate(self) -> AuthBase:
"""Authenticate"""
diff --git a/eodag/plugins/authentication/header.py b/eodag/plugins/authentication/header.py
index 2fed1502b..60bcf77eb 100644
--- a/eodag/plugins/authentication/header.py
+++ b/eodag/plugins/authentication/header.py
@@ -34,7 +34,12 @@ class HTTPHeaderAuth(Authentication):
This plugin enables implementation of custom HTTP authentication scheme (other than Basic, Digest, Token
negotiation et al.) using HTTP headers.
- The plugin is configured as follows in the providers config file::
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): HTTPHeaderAuth
+ * **headers** [Dict[str, str]]: dictionary containing all keys/value pairs that should be added to the headers
+
+ Below an example for the configuration in the providers config file is shown::
provider:
...
diff --git a/eodag/plugins/authentication/keycloak.py b/eodag/plugins/authentication/keycloak.py
index e0a7ebda8..ab7d4b448 100644
--- a/eodag/plugins/authentication/keycloak.py
+++ b/eodag/plugins/authentication/keycloak.py
@@ -41,7 +41,19 @@
class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
"""Authentication plugin using Keycloak and OpenId Connect.
- This plugin request a token and use it through a query-string or a header.
+ This plugin requests a token which is added to a query-string or a header for authentication.
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): KeycloakOIDCPasswordAuth
+ * **auth_base_uri** [str] (mandatory): base url used in the request to fetch the token
+ * **realm** [str] (mandatory): keycloak realm
+ * **client_id** [str] (mandatory): keycloak client id
+ * **client_secret** [str] (mandatory): keycloak client secret, set to null if no secret is used
+ * **token_provision** [str] (mandatory): if the token should be added to the query string (qs)
+ or to the header (header)
+ * **token_qs_key** [str] (mandatory if token_provision=qs): key of the param added to the query string
+ * **auth_error_code** [int]: which error code is returned in case of an authentication error
+ * **ssl_verify** [bool]: if the ssl certificates should be verified in the token request; default: True
Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link
`http://example.com?foo=bar` will become
diff --git a/eodag/plugins/authentication/oauth.py b/eodag/plugins/authentication/oauth.py
index 8f31a3e5b..24ddea304 100644
--- a/eodag/plugins/authentication/oauth.py
+++ b/eodag/plugins/authentication/oauth.py
@@ -26,7 +26,14 @@
class OAuth(Authentication):
- """OAuth authentication plugin"""
+ """OAuth authentication plugin
+
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): OAuth
+
+ The mandatory parameters that have to be added in the eodag config are aws_access_key_id and aws_secret_access_key.
+ """
def __init__(self, provider: str, config: PluginConfig) -> None:
super(OAuth, self).__init__(provider, config)
diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py
index 2c0cc1da9..4ffce2156 100644
--- a/eodag/plugins/authentication/openid_connect.py
+++ b/eodag/plugins/authentication/openid_connect.py
@@ -42,9 +42,12 @@
class OIDCRefreshTokenBase(Authentication):
- """OIDC refresh token base class, to be used through specific OIDC flows plugins.
+ """OIDC refresh token base class, to be used through specific OIDC flows plugins;
+ Common mechanism to handle refresh token from all OIDC auth plugins;
+ Plugins inheriting from this base class must implement the methods _request_new_token and
+ _get_token_with_refresh_token. Depending oh the implementation of these methods they can have
+ different configuration parameters.
- Common mechanism to handle refresh token from all OIDC auth plugins.
"""
class TokenInfo(TypedDict, total=False):
@@ -180,83 +183,53 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
The headless interaction is fully configurable, and rely on XPATH to retrieve all the necessary
information.
- The configuration keys of this plugin are as follows (they have no defaults)::
-
- # (mandatory) The authorization url of the server (where to query for grants)
- authorization_uri:
-
- # (mandatory) The callback url that will handle the code given by the OIDC provider
- redirect_uri:
-
- # (mandatory) The url to query to exchange the authorization code obtained from the OIDC provider
- # for an authorized token
- token_uri:
-
- # (mandatory) The OIDC provider's client ID of the eodag provider
- client_id:
-
- # (mandatory) Wether a user consent is needed during the authentication
- user_consent_needed:
-
- # (mandatory) One of: json, data or params. This is the way to pass the data to the POST request
- # that is made to the token server. They correspond to the recognised keywords arguments
- # of the Python `requests `_ library
- token_exchange_post_data_method:
-
- # (mandatory) The key pointing to the token in the json response to the POST request to the token server
- token_key:
-
- # (mandatory) One of qs or header. This is how the token obtained will be used to authenticate the user
- # on protected requests. If 'qs' is chosen, then 'token_qs_key' is mandatory
- token_provision:
-
- # (mandatory) The xpath to the HTML form element representing the user login form
- login_form_xpath:
-
- # (mandatory) Where to look for the authentication_uri. One of 'config' (in the configuration) or 'login-form'
- # (use the 'action' URL found in the login form retrieved with login_form_xpath). If the value is 'config',
- # authentication_uri config param is mandatory
- authentication_uri_source:
-
- # (optional) The URL of the authentication backend of the OIDC provider
- authentication_uri:
-
- # (optional) The xpath to the user consent form. The form is searched in the content of the response
- # to the authorization request
- user_consent_form_xpath:
-
- # (optional) The data that will be passed with the POST request on the form 'action' URL. The data are
- # given as a key value pairs, the keys representing the data key and the value being either
- # a 'constant' string value, or a string of the form 'xpath()'
- # and representing a value to be retrieved in the user consent form. The xpath must resolve
- # directly to a string value, not to an HTML element. Example:
- # `xpath(//input[@name="sessionDataKeyConsent"]/@value)`
- user_consent_form_data:
-
- # (optional) A mapping giving additional data to be passed to the login POST request. The value follows the
- # same rules as with user_consent_form_data
- additional_login_form_data:
-
- # (optional) Key/value pairs of patterns/messages. If exchange_url contains the given pattern, the associated
- message will be sent in an AuthenticationError
- exchange_url_error_pattern:
-
- # (optional) The OIDC provider's client secret of the eodag provider
- client_secret:
-
- # (optional) A mapping between OIDC url query string and token handler query string
- # params (only necessary if they are not the same as for OIDC). This is eodag provider
- # dependant
- token_exchange_params:
- redirect_uri:
- client_id:
-
- # (optional) Only necessary when 'token_provision' is 'qs'. Refers to the name of the query param to be
- # used in the query request
- token_qs_key:
-
- # (optional) The key pointing to the refresh_token in the json response to the POST request to the token server
- refresh_token_key:
+ The configuration keys of this plugin are as follows (they have no defaults):
+
+ * **type** [str] (mandatory): OIDCAuthorizationCodeFlowAuth
+ * **authorization_uri** [str] (mandatory): The authorization url of the server (where to query for grants)
+ * **redirect_uri** [str] (mandatory): The callback url that will handle the code given by the OIDC provider
+ * **token_uri** [str] (mandatory): The url to query to exchange the authorization
+ code obtained from the OIDC provider for an authorized token
+ * **client_id** [str] (mandatory): The OIDC provider's client ID of the eodag provider
+ * **user_consent_needed** [bool] (mandatory): Whether a user consent is needed
+ during the authentication
+ * **token_exchange_post_data_method** [str] (mandatory): One of: json, data or params.
+ This is the way to pass the data to the POST request that is made to the token server.
+ They correspond to the recognised keywords arguments of the Python
+ `requests `_ library
+ * **token_key** [str] (mandatory): The key pointing to the token in the json response
+ to the POST request to the token server
+ * **token_provision** [str] (mandatory): One of qs or header. This is how the token
+ obtained will be used to authenticate the user on protected requests. If 'qs' is
+ chosen, then 'token_qs_key' is mandatory
+ * **login_form_xpath** [str] (mandatory): The xpath to the HTML form element representing
+ the user login form
+ * **authentication_uri_source** [str] (mandatory): Where to look for the authentication_uri.
+ One of 'config' (in the configuration) or 'login-form' (use the 'action' URL found in the
+ login form retrieved with login_form_xpath). If the value is 'config',
+ authentication_uri config param is mandatory
+ * **authentication_uri** [str] (mandatory if authentication_uri_source=config):
+ The URL of the authentication backend of the OIDC provider
+ * **user_consent_form_xpath** [str]: The xpath to the user consent form. The form
+ is searched in the content of the response to the authorization request
+ * **user_consent_form_data** [Dict[str, str]]: The data that will be passed with the
+ POST request on the form 'action' URL. The data are given as a key value pairs, the
+ keys representing the data key and the value being either a 'constant' string value,
+ or a string of the form 'xpath()' and representing a
+ value to be retrieved in the user consent form. The xpath must resolve directly to a
+ string value, not to an HTML element. Example: `xpath(//input[@name="sessionDataKeyConsent"]/@value)`
+ * **additional_login_form_data** [Dict[str, str]]: A mapping giving additional data
+ to be passed to the login POST request. The value follows the same rules as with user_consent_form_data
+ * **exchange_url_error_pattern** [Dict[str, str]]: Key/value pairs of patterns/messages.
+ If exchange_url contains the given pattern, the associated message will be sent in an AuthenticationError
+ * **client_secret** [str]:The OIDC provider's client secret of the eodag provider
+ * **token_exchange_params** [Dict[str, str]: mandatory keys for the dict: redirect_uri, client_id;
+ A mapping between OIDC url query string and token handler query string params
+ (only necessary if they are not the same as for OIDC). This is eodag provider dependant
+ * **token_qs_key** [str] (mandatory when token_provision=qs): Refers to the name
+ of the query param to be used in the query request
+ * **refresh_token_key** [str]: The key pointing to the refresh_token in the json response
+ to the POST request to the token server
"""
SCOPE = "openid"
diff --git a/eodag/plugins/authentication/qsauth.py b/eodag/plugins/authentication/qsauth.py
index f85f6938f..6e532db8b 100644
--- a/eodag/plugins/authentication/qsauth.py
+++ b/eodag/plugins/authentication/qsauth.py
@@ -36,6 +36,11 @@ class HttpQueryStringAuth(Authentication):
"""An Authentication plugin using HTTP query string parameters.
This plugin sends credentials as query-string parameters.
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): HttpQueryStringAuth
+ * **auth_uri** [str]: used to check the credentials given in the configuration
+
Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link
`http://example.com?foo=bar` will become
`http://example.com?foo=bar&apikey=XXX&otherkey=YYY` if associated to the following
diff --git a/eodag/plugins/authentication/sas_auth.py b/eodag/plugins/authentication/sas_auth.py
index 3965a2632..e6040d238 100644
--- a/eodag/plugins/authentication/sas_auth.py
+++ b/eodag/plugins/authentication/sas_auth.py
@@ -85,7 +85,18 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest:
class SASAuth(Authentication):
- """SASAuth authentication plugin"""
+ """SASAuth authentication plugin
+
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): SASAuth
+ * **auth_uri** [str] (mandatory): url used to get the signed url
+ * **signed_url_key** [str] (mandatory): key to get the signed url
+ * **headers** [Dict[str, str]] (mandatory if apiKey is used): headers to be added to the requests
+ * **ssl_verify** [bool]: if the ssl certificates should be verified in the requests; default: True
+
+ An apiKey that is added in the headers can be given in the credentials in the config file.
+ """
def validate_config_credentials(self) -> None:
"""Validate configured credentials"""
diff --git a/eodag/plugins/authentication/token.py b/eodag/plugins/authentication/token.py
index 6cd89214b..55cc8e575 100644
--- a/eodag/plugins/authentication/token.py
+++ b/eodag/plugins/authentication/token.py
@@ -41,7 +41,22 @@
class TokenAuth(Authentication):
- """TokenAuth authentication plugin"""
+ """TokenAuth authentication plugin - fetches a token which is added to search/download requests
+
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): TokenAuth
+ * **auth_uri** [str] (mandatory): url used to fetch the access token with user/password
+ * **refresh_uri** [str] : url used to fetch the access token with a refresh token
+ * **token_type** [str]: type of the token (json or text); default: text
+ * **token_key** [str] (mandatory if token_type=json): key to get the access token in the
+ response to the token request
+ * **refresh_token_key** [str]: key to get the refresh token in the response to the token request
+ * **ssl_verify** [bool]: if the ssl certificates should be verified in the requests; default: True
+ * **auth_error_code** [int]: which error code is returned in case of an authentication error
+ * **req_data** [Dict[str, Any]]: if the credentials should be sent as data in the post request,
+ the json structure can be given in this parameter
+ """
def __init__(self, provider: str, config: PluginConfig) -> None:
super(TokenAuth, self).__init__(provider, config)
diff --git a/eodag/plugins/authentication/token_exchange.py b/eodag/plugins/authentication/token_exchange.py
index 36789e4ae..52dc24b68 100644
--- a/eodag/plugins/authentication/token_exchange.py
+++ b/eodag/plugins/authentication/token_exchange.py
@@ -38,25 +38,17 @@ class OIDCTokenExchangeAuth(Authentication):
"""Token exchange implementation using
:class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` token as subject.
- The configuration keys of this plugin are as follows (they have no defaults)::
+ The configuration keys of this plugin are as follows (they have no defaults):
- # (mandatory) The full OIDCAuthorizationCodeFlowAuth plugin configuration used to retrieve subject token
- subject:
+ * **subject** [Dict[str, Any]] (mandatory): The full OIDCAuthorizationCodeFlowAuth plugin
+ configuration used to retrieve subject token
+ * **subject_issuer** [str] (mandatory): Identifies the issuer of the subject_token
+ * **token_uri** [str] (mandatory): The url to query to get the authorized token
+ * **client_id** [str] (mandatory): The OIDC provider's client ID of the eodag provider
+ * **audience** [str] (mandatory): This parameter specifies the target client you want the new token minted for.
+ * **token_key** [str] (mandatory): The key pointing to the token in the json response to
+ the POST request to the token server
- # (mandatory) Identifies the issuer of the subject_token
- subject_issuer:
-
- # (mandatory) The url to query to get the authorized token
- token_uri:
-
- # (mandatory) The OIDC provider's client ID of the eodag provider
- client_id:
-
- # (mandatory) This parameter specifies the target client you want the new token minted for.
- audience:
-
- # (mandatory) The key pointing to the token in the json response to the POST request to the token server
- token_key:
"""
GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
diff --git a/eodag/plugins/crunch/base.py b/eodag/plugins/crunch/base.py
index db2f16265..5bb367c32 100644
--- a/eodag/plugins/crunch/base.py
+++ b/eodag/plugins/crunch/base.py
@@ -27,7 +27,10 @@
class Crunch(PluginTopic):
- """Base cruncher"""
+ """Base cruncher
+ :param config: Crunch configuration
+ :type config: Dict[str, Any]
+ """
def __init__(self, config: Optional[Dict[str, Any]]) -> None:
self.config = PluginConfig()
diff --git a/eodag/plugins/crunch/filter_date.py b/eodag/plugins/crunch/filter_date.py
index 6901fd855..e52535693 100644
--- a/eodag/plugins/crunch/filter_date.py
+++ b/eodag/plugins/crunch/filter_date.py
@@ -37,10 +37,10 @@
class FilterDate(Crunch):
"""FilterDate cruncher: filter products by date
- :param config: Crunch configuration, may contain :
+ The Crunch configuration, may contain :
- - `start`: (optional) start sensing time in iso format
- - `end`: (optional) end sensing time in iso format
+ * **start** [str]: start sensing time in iso format
+ * **end** [str]: end sensing time in iso format
"""
@staticmethod
diff --git a/eodag/plugins/crunch/filter_latest_intersect.py b/eodag/plugins/crunch/filter_latest_intersect.py
index 3b91989e9..aaa404f77 100644
--- a/eodag/plugins/crunch/filter_latest_intersect.py
+++ b/eodag/plugins/crunch/filter_latest_intersect.py
@@ -39,7 +39,8 @@
class FilterLatestIntersect(Crunch):
"""FilterLatestIntersect cruncher
- Filter latest products (the ones with a the highest start date) that intersect search extent
+ Filter latest products (the ones with a the highest start date) that intersect search extent;
+ The configuration for this plugin is an empty dict
"""
@staticmethod
diff --git a/eodag/plugins/crunch/filter_latest_tpl_name.py b/eodag/plugins/crunch/filter_latest_tpl_name.py
index 669943bf6..53dd9c1d2 100644
--- a/eodag/plugins/crunch/filter_latest_tpl_name.py
+++ b/eodag/plugins/crunch/filter_latest_tpl_name.py
@@ -35,9 +35,9 @@ class FilterLatestByName(Crunch):
Filter Search results to get only the latest product, based on the name of the product
- :param config: Crunch configuration, must contain :
+ The Crunch configuration must contain :
- - `name_pattern` : product name pattern
+ * **name_pattern** [str] (mandatory) : product name pattern
"""
NAME_PATTERN_CONSTRAINT = re.compile(r"\(\?P\\d\{6\}\)")
diff --git a/eodag/plugins/crunch/filter_overlap.py b/eodag/plugins/crunch/filter_overlap.py
index 43b7fbc0d..b5a7d25f6 100644
--- a/eodag/plugins/crunch/filter_overlap.py
+++ b/eodag/plugins/crunch/filter_overlap.py
@@ -40,14 +40,14 @@ class FilterOverlap(Crunch):
Filter products, retaining only those that are overlapping with the search_extent
- :param config: Crunch configuration, may contain :
+ The Crunch configuration may contain :
- - `minimum_overlap` : minimal overlap percentage
- - `contains` : True if product geometry contains the search area
- - `intersects` : True if product geometry intersects the search area
- - `within` : True if product geometry is within the search area
+ * **minimum_overlap** [Union[float, str]]: minimal overlap percentage; default: "0"
+ * **contains** [bool]: True if product geometry contains the search area; default: False
+ * **intersects** [bool]: True if product geometry intersects the search area; default: False
+ * **within** [bool]: True if product geometry is within the search area; default: False
- These configuration parameters are mutually exclusive.
+ These configuration parameters are mutually exclusive.
"""
def proceed(
diff --git a/eodag/plugins/crunch/filter_property.py b/eodag/plugins/crunch/filter_property.py
index 2bb064e79..4e70ffd93 100644
--- a/eodag/plugins/crunch/filter_property.py
+++ b/eodag/plugins/crunch/filter_property.py
@@ -34,10 +34,10 @@ class FilterProperty(Crunch):
Filter products, retaining only those whose property match criteria
- :param config: Crunch configuration, should contain :
+ The Crunch configuration should contain :
- - `property=value` : property key from product.properties, associated to its filter value
- - `operator` : (optional) Operator used for filtering (one of `lt,le,eq,ne,ge,gt`). Default is `eq`
+ * **** [Any] (mandatory): property key from product.properties, associated to its filter value
+ * **operator** [str]: Operator used for filtering (one of `lt,le,eq,ne,ge,gt`). Default is `eq`
"""
def proceed(
diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py
index 49a91fb64..6c06e454e 100644
--- a/eodag/plugins/download/aws.py
+++ b/eodag/plugins/download/aws.py
@@ -214,15 +214,25 @@
class AwsDownload(Download):
"""Download on AWS using S3 protocol.
- :param provider: provider name
- :param config: Download plugin configuration:
-
- * ``config.base_uri`` (str) - s3 endpoint url
- * ``config.requester_pays`` (bool) - (optional) whether download is done from a
- requester-pays bucket or not
- * ``config.flatten_top_dirs`` (bool) - (optional) flatten directory structure
- * ``config.products`` (dict) - (optional) product_type specific configuration
- * ``config.ignore_assets`` (bool) - (optional) ignore assets and download using downloadLink
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): AwsDownload
+ * **base_uri** [str] (mandatory): s3 endpoint url
+ * **requester_pays** [bool]: whether download is done from a requester-pays bucket or not; default: False
+ * **flatten_top_dirs** [bool]: if the directory structure should be flattened; default: True
+ * **ignore_assets** [bool]: ignore assets and download using downloadLink; default: False
+ * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True
+ * **bucket_path_level** [int]: at which level of the path part of the url the bucket can be found;
+ If no bucket_path_level is given, the bucket is taken from the first element of the netloc part.
+ * **products** [Dict[str, Dict[str, Any]]: product type specific config; the keys are the product types,
+ the values are dictionaries which can contain the keys:
+
+ * **default_bucket** [str]: bucket where the product type can be found
+ * **complementary_url_key** [str]: keys to add additional urls
+ * **build_safe** [bool]: if a SAFE (Standard Archive Format for Europe) product should
+ be created; used for Sentinel products; default: False
+ * **fetch_metadata** [Dict[str, Any]: config for metadata to be fetched for the SAFE product
+
"""
def __init__(self, provider: str, config: PluginConfig) -> None:
diff --git a/eodag/plugins/download/creodias_s3.py b/eodag/plugins/download/creodias_s3.py
index 89108878a..79d1609c2 100644
--- a/eodag/plugins/download/creodias_s3.py
+++ b/eodag/plugins/download/creodias_s3.py
@@ -25,7 +25,13 @@
class CreodiasS3Download(AwsDownload):
"""
- Download on creodias s3 from their VMs
+ Download on creodias s3 from their VMs (extension of AwsDownload)
+ The configuration parameters for this plugin are:
+
+ * **type** [str] (mandatory): CreodiasS3Download
+ * **base_uri** [str] (mandatory): s3 endpoint url
+ * **s3_bucket** [str] (mandatory): bucket where the products can be found
+ * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True
"""
def _get_authenticated_objects_unsigned(self, bucket_name, prefix, auth_dict):
diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
index 83926fba7..f16fb9cf9 100644
--- a/eodag/plugins/download/http.py
+++ b/eodag/plugins/download/http.py
@@ -100,18 +100,47 @@ class HTTPDownload(Download):
:param provider: provider name
:param config: Download plugin configuration:
- * ``config.base_uri`` (str) - (optional) default endpoint url
- * ``config.extract`` (bool) - (optional) extract downloaded archive or not
- * ``config.auth_error_code`` (int) - (optional) authentication error code
- * ``config.dl_url_params`` (dict) - (optional) attitional parameters to send in the request
- * ``config.archive_depth`` (int) - (optional) level in extracted path tree where to find data
- * ``config.flatten_top_dirs`` (bool) - (optional) flatten directory structure
- * ``config.ignore_assets`` (bool) - (optional) ignore assets and download using downloadLink
- * ``config.order_enabled`` (bool) - (optional) wether order is enabled or not if product is `OFFLINE`
- * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST
- * ``config.order_headers`` (dict) - (optional) order request headers
- * ``config.order_on_response`` (dict) - (optional) edit or add new product properties
- * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - (optional) Order status handling
+ * :attr:`~eodag.config.PluginConfig.type` (``str``): (mandatory) ``HTTPDownload``
+ * :attr:`~eodag.config.PluginConfig.base_uri` (``str``): default endpoint url
+ * :attr:`~eodag.config.PluginConfig.method` (``str``): HTTP request method for the download request (``GET`` or
+ ``POST``); default: ``GET``
+ * :attr:`~eodag.config.PluginConfig.extract` (``bool``): if the content of the downloaded file should be
+ extracted; default: ``True``
+ * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
+ authentication error
+ * :attr:`~eodag.config.PluginConfig.dl_url_params` [Dict[str, Any]]: parameters to be added to the query params
+ of the request
+ * :attr:`~eodag.config.PluginConfig.archive_depth` (``int``): level in extracted path tree where to find data;
+ default: ``1``
+ * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure should be
+ flattened; default: ``True``
+ * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download using downloadLink;
+ default: ``False``
+ * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
+ default: ``5``
+ * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
+ requests; default: ``True``
+ * :attr:`~eodag.config.PluginConfig.output_extension` (``str``): which extension should be used for the
+ downloaded file
+ * :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without
+ authentication; default: ``True``
+ * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it;
+ if this paramter is set to true, a mapping for the orderLink has to be added to the metadata mapping of
+ the search plugin used for the provider; default: False
+ * :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET``
+ or ``POST``); default: ``GET``
+ * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): headers to be added to the order
+ request
+ * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
+ a typed dictionary containing the key ``metadata_mapping`` which can be used to add new product properties
+ based on the data in response to the order request
+ * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
+ configuration to handle the order status; contains information which method to use, how the response data is
+ interpreted, which status corresponds to success, ordered and error and what should be done on success.
+ * :attr:`~eodag.config.PluginConfig.products` (``Dict[str, Dict[str, Any]``): product type specific config; the
+ keys are the product types, the values are dictionaries which can contain the keys
+ :attr:`~eodag.config.PluginConfig.output_extension` and :attr:`~eodag.config.PluginConfig.extract` to
+ overwrite the provider config for a specific product type
"""
@@ -130,12 +159,13 @@ def order_download(
and has `orderLink` in its properties.
Product ordering can be configured using the following download plugin parameters:
- - **order_enabled**: Wether order is enabled or not (may not use this method
+ - :attr:`~eodag.config.PluginConfig.order_enabled`: Wether order is enabled or not (may not use this method
if no `orderLink` exists)
- - **order_method**: (optional) HTTP request method, GET (default) or POST
+ - :attr:`~eodag.config.PluginConfig.order_method`: (optional) HTTP request method, GET (default) or POST
- - **order_on_response**: (optional) things to do with obtained order response:
+ - :attr:`~eodag.config.PluginConfig.order_on_response`: (optional) things to do with obtained order
+ response:
- *metadata_mapping*: edit or add new product propoerties properties
@@ -256,7 +286,7 @@ def order_download_status(
It will be executed before each download retry.
Product order status request can be configured using the following download plugin parameters:
- - **order_status**: :class:`~eodag.config.PluginConfig.OrderStatus`
+ - :attr:`~eodag.config.PluginConfig.order_status`: :class:`~eodag.config.PluginConfig.OrderStatus`
Product properties used for order status:
diff --git a/eodag/plugins/download/s3rest.py b/eodag/plugins/download/s3rest.py
index 79dc5517b..43de0e4e0 100644
--- a/eodag/plugins/download/s3rest.py
+++ b/eodag/plugins/download/s3rest.py
@@ -67,18 +67,19 @@ class S3RestDownload(Download):
Re-use AwsDownload bucket some handling methods
- :param provider: provider name
- :param config: Download plugin configuration:
-
- * ``config.base_uri`` (str) - default endpoint url
- * ``config.extract`` (bool) - (optional) extract downloaded archive or not
- * ``config.auth_error_code`` (int) - (optional) authentication error code
- * ``config.bucket_path_level`` (int) - (optional) bucket location index in path.split('/')
- * ``config.order_enabled`` (bool) - (optional) wether order is enabled or not if product is `OFFLINE`
- * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST
- * ``config.order_headers`` (dict) - (optional) order request headers
- * ``config.order_on_response`` (dict) - (optional) edit or add new product properties
- * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - Order status handling
+ Download plugin configuration:
+
+ * **config.base_uri** [str] (mandatory): default endpoint url
+ * **config.extract** [bool]: extract downloaded archive or not
+ * **config.auth_error_code** [int]: authentication error code
+ * **config.bucket_path_level** [int]: bucket location index in path.split('/')
+ * **config.order_enabled** [bool]: whether order is enabled or not if product is `OFFLINE`
+ * **config.order_method** [str]: HTTP request method, GET (default) or POST
+ * **config.order_headers** [dict]: order request headers
+ * **order_on_response** [:class:`~eodag.config.PluginConfig.OrderOnResponse`]: a typed dictionary
+ containing the key 'metadata_mapping' which can be used to add new product properties
+ based on the data in response to the order request
+ * **config.order_status** [:class:`~eodag.config.PluginConfig.OrderStatus`]: Order status handling
"""
def __init__(self, provider: str, config: PluginConfig) -> None:
diff --git a/eodag/plugins/search/base.py b/eodag/plugins/search/base.py
index 22007ecbb..f724ad877 100644
--- a/eodag/plugins/search/base.py
+++ b/eodag/plugins/search/base.py
@@ -59,7 +59,9 @@ class Search(PluginTopic):
"""Base Search Plugin.
:param provider: An EODAG provider name
+ :type provider: str
:param config: An EODAG plugin configuration
+ :type config: Dict[str, Any]
"""
auth: Union[AuthBase, Dict[str, str]]
diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py
index ab400773e..0be3f1b74 100644
--- a/eodag/plugins/search/build_search_result.py
+++ b/eodag/plugins/search/build_search_result.py
@@ -72,21 +72,12 @@ class BuildPostSearchResult(PostJsonSearch):
performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
object.
- The available configuration parameters inherits from parent classes, with particularly
- for this plugin:
+ The available configuration parameters inherits from parent classes (PostJsonSearch and
+ QueryStringSearch), with particularly for this plugin:
- - **api_endpoint**: (mandatory) The endpoint of the provider's search interface
+ * **remove_from_query** [List[str]]: List of parameters used to parse metadata but that must
+ not be included to the query
- - **pagination**: The configuration of how the pagination is done
- on the provider. It is a tree with the following nodes:
-
- - *next_page_query_obj*: (optional) The additional parameters needed to perform
- search. These paramaters won't be included in result. This must be a json dict
- formatted like `{{"foo":"bar"}}` because it will be passed to a `.format()`
- method before being loaded as json.
-
- :param provider: An eodag providers configuration dictionary
- :param config: Path to the user configuration file
"""
def count_hits(
@@ -179,7 +170,7 @@ def normalize_results(
result.update(results.product_type_def_params)
result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
- # parse porperties
+ # parse properties
parsed_properties = properties_from_json(
result,
self.config.metadata_mapping,
@@ -240,19 +231,13 @@ class BuildSearchResult(BuildPostSearchResult):
This plugin builds a single :class:`~eodag.api.search_result.SearchResult` object
using given query parameters as product properties.
- The available configuration parameters inherits from parent classes, with particularly
- for this plugin:
-
- - **end_date_excluded**: Set to `False` if provider does not include end date to
- search
-
- - **remove_from_query**: List of parameters used to parse metadata but that must
- not be included to the query
+ The available configuration parameters inherits from parent classes (BuildPostSearchResult,
+ PostJsonSearch and QueryStringSearch), with particularly for this plugin:
- - **constraints_file_url**: url of the constraint file used to build queryables
+ * **end_date_excluded** [bool]: Set to `False` if provider does not include end date in
+ the search request; In this case, if the end date is at midnight, the previous day will be
+ used. default: true
- :param provider: An eodag providers configuration dictionary
- :param config: Path to the user configuration file
"""
def __init__(self, provider: str, config: PluginConfig) -> None:
diff --git a/eodag/plugins/search/cop_marine.py b/eodag/plugins/search/cop_marine.py
index 99afaab09..a37c9493a 100644
--- a/eodag/plugins/search/cop_marine.py
+++ b/eodag/plugins/search/cop_marine.py
@@ -109,7 +109,16 @@ def _check_int_values_properties(properties: Dict[str, Any]):
class CopMarineSearch(StaticStacSearch):
- """class that implements search for the Copernicus Marine provider"""
+ """class that implements search for the Copernicus Marine provider
+ It calls discover_product_types inherited from StaticStacSearch but for the actual search
+ a special method which fetches the urls of the available products from an S3 storage and
+ filters them has been written.
+ The configuration parameters are inherited from the parent and grand-parent classes. The
+ `auto_discovery` parameter in the `discover_metadata` section has to be set to `false` and the
+ `fetch_url` in the `discover_queryables` queryables section has to be set to `null` to
+ overwrite the default config from the stac provider configuration because those functionalities
+ are not available.
+ """
def __init__(self, provider: str, config: PluginConfig):
original_metadata_mapping = copy.deepcopy(config.metadata_mapping)
@@ -122,12 +131,10 @@ def _get_product_type_info(
) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
"""Fetch product type and associated datasets info"""
- fetch_url = cast(
- str,
- self.config.discover_product_types["fetch_url"].format(
- **self.config.__dict__
- ),
+ fetch_url = cast(str, self.config.discover_product_types["fetch_url"]).format(
+ **self.config.__dict__
)
+
logger.debug("fetch data for collection %s", product_type)
provider_product_type = self.config.products.get(product_type, {}).get(
"productType", None
diff --git a/eodag/plugins/search/creodias_s3.py b/eodag/plugins/search/creodias_s3.py
index 62a224a66..52ff0adb6 100644
--- a/eodag/plugins/search/creodias_s3.py
+++ b/eodag/plugins/search/creodias_s3.py
@@ -117,7 +117,11 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
class CreodiasS3Search(ODataV4Search):
"""
- Search on creodias and adapt results to s3
+ CreodiasS3Search is an extension of ODataV4Search, it executes a Search on creodias and
+ adapts results so that the assets contain links to s3. It has the same configuration
+ parameters as ODataV4Search and one additional parameter:
+
+ * **s3_endpoint** [str] (mandatory): base url of the s3
"""
def __init__(self, provider, config):
diff --git a/eodag/plugins/search/csw.py b/eodag/plugins/search/csw.py
index e12fee62f..214da4f72 100644
--- a/eodag/plugins/search/csw.py
+++ b/eodag/plugins/search/csw.py
@@ -52,7 +52,43 @@
class CSWSearch(Search):
- """A plugin for implementing search based on OGC CSW"""
+ """A plugin for implementing search based on OGC CSW
+ It has the following configuration parameters:
+
+ * **api_endpoint** [str] (mandatory): The endpoint of the provider's search interface
+ * **version** [str]: OGC Catalogue Service version; default: 2.0.2
+ * **search_definition** [Dict[str, Any]] (mandatory):
+
+ * **product_type_tags** [List[Dict[str, Any]]: dict of product type tags
+ * **resource_location_filter** [str]: regex string
+ * **date_tags** [Dict[str, Any]]: tags for start and end
+
+ * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a
+ metadata mapping is "query-able", and get the semantics of how to format the query string
+ parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
+ just configure it in the metadata mapping to be a list of 2 items, the first one being the
+ specification of the query string search formatting. The later is a string following the
+ specification of Python string formatting, with a special behaviour added to it. For example,
+ an entry in the metadata mapping of this kind::
+
+ completionTimeFromAscendingNode:
+ - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
+ - '$.properties.acquisition.endViewingDate'
+
+ means that the search url will have a query string parameter named *"f"* with a value of
+ *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value
+ of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
+ ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
+ of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
+ semantics of the query string formatting introduced by this plugin: any eodag search parameter
+ can be referenced in the query string with an additional optional conversion function that
+ is separated from it by a ``#`` (see :func:`~eodag.utils.format_metadata` for further details
+ on the available converters). Note that for the values in the ``free_text_search_operations``
+ configuration parameter follow the same rule. If the metadata_mapping is not a list but
+ only a string, this means that the parameters is not queryable but it is included in the
+ result obtained from the provider. The string indicates how the provider result should be
+ mapped to the eodag parameter.
+ """
def __init__(self, provider: str, config: PluginConfig) -> None:
super(CSWSearch, self).__init__(provider, config)
diff --git a/eodag/plugins/search/data_request_search.py b/eodag/plugins/search/data_request_search.py
index 862abd4ec..1ea9d9e4b 100644
--- a/eodag/plugins/search/data_request_search.py
+++ b/eodag/plugins/search/data_request_search.py
@@ -58,9 +58,84 @@
class DataRequestSearch(Search):
"""
Plugin to execute search requests composed of several steps:
- - do a data request which defines which data shall be searched
- - check the status of the request job
- - if finished - fetch the result of the job
+
+ #. do a data request which defines which data shall be searched
+ #. check the status of the request job
+ #. if finished - fetch the result of the job
+
+ It has the following configuration parameters:
+
+ * **api_endpoint** [str] (mandatory): The endpoint of the provider's search interface
+ * **results_entry** [str] (mandatory): The name of the key in the provider search
+ result that gives access to the result entries
+ * **data_request_url** [str] (mandatory): url to which the data request shall be sent
+ * **status_url** [str] (mandatory): url to fetch the status of the data request
+ * **result_url** [str] (mandatory): url to fetch the search result when the data request is done
+ * **need_auth** [bool]: if authentication is needed for the search request; default: False
+ * **auth_error_code** [int]: which error code is returned in case of an authentication
+ error; only used if `need_auth=true`
+ * **ssl_verify** [bool]: if the ssl certificates should be verified in requests; default: True
+ * **timeout** [int]: time to wait until request timeout in seconds; default: 5
+ * **dates_required** [bool]: if date parameters are mandatory in the request; default: True
+ * **pagination** [:class:`~eodag.config.PluginConfig.Pagination`] (mandatory): The configuration of
+ how the pagination is done on the provider. It is a tree with the following nodes:
+
+ * **total_items_nb_key_path** [str]: An XPath or JsonPath leading to the total number of
+ results satisfying a request. This is used for providers which provides the total results
+ metadata along with the result of the query and don't have an endpoint for querying
+ the number of items satisfying a request, or for providers for which the count endpoint
+ returns a json or xml document
+ * **max_items_per_page** [int]: The maximum number of items per page that the provider can
+ handle; default: 50
+ * **start_page** [int]: number of the first page; default: 1
+ * **discover_product_types** [Dict[str, Any]]: configuration for product type discovery
+ based on information from the provider; It contains the keys:
+
+ * **fetch_url** [str] (mandatory): url from which the product types can be fetched
+ * **result_type** [str]: type of the provider result; currently only "json" is supported,
+ (other types could be used in an extension of this plugin)
+ * **results_entry** [str] (mandatory): json path to the list of product types
+ * **generic_product_type_id** [str]: mapping for the product type id
+ * **generic_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type
+ metadata (e.g. abstract, licence) which can be parsed from the provider result
+ * **generic_product_type_parsable_properties** [Dict[str, str]]: mapping for product type
+ properties which can be parsed from the result that are not product type metadata
+ * **single_collection_fetch_url** [str]: url to fetch data for a single collection;
+ used if product type metadata is not available from the endpoint given in `fetch_url`
+ * **single_collection_fetch_qs** [str]: query string to be added to the `fetch_url`
+ to filter for a collection
+ * **single_product_type_parsable_metadata** [Dict[str, str]]: mapping for product type
+ metadata returned by the endpoint given in `single_collection_fetch_url`.
+ * **constraints_file_url** [str]: url to fetch the constraints for a specific product type,
+ can be an http url or a path to a file; the constraints are used to build queryables
+ * **constraints_entry** [str]: key in the json result where the constraints can be found;
+ if not given, it is assumed that the constraints are on top level of the result, i.e.
+ the result is an array of constraints
+ * **metadata_mapping** [Dict[str, Any]]: The search plugins of this kind can detect when a
+ metadata mapping is "query-able", and get the semantics of how to format the query string
+ parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
+ just configure it in the metadata mapping to be a list of 2 items, the first one being the
+ specification of the query string search formatting. The later is a string following the
+ specification of Python string formatting, with a special behaviour added to it. For example,
+ an entry in the metadata mapping of this kind::
+
+ completionTimeFromAscendingNode:
+ - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
+ - '$.properties.acquisition.endViewingDate'
+
+ means that the search url will have a query string parameter named *"f"* with a value of
+ *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value
+ of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
+ ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
+ of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
+ semantics of the query string formatting introduced by this plugin: any eodag search parameter
+ can be referenced in the query string with an additional optional conversion function that
+ is separated from it by a ``#`` (see :func:`~eodag.utils.format_metadata` for further details
+ on the available converters). Note that for the values in the ``free_text_search_operations``
+ configuration parameter follow the same rule. If the metadata_mapping is not a list but
+ only a string, this means that the parameters is not queryable but it is included in the
+ result obtained from the provider. The string indicates how the provider result should be
+ mapped to the eodag parameter.
"""
data_request_id: Optional[str]
diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py
index e235b1218..953aa1fe0 100644
--- a/eodag/plugins/search/qssearch.py
+++ b/eodag/plugins/search/qssearch.py
@@ -110,98 +110,171 @@
class QueryStringSearch(Search):
"""A plugin that helps implementing any kind of search protocol that relies on
- query strings (e.g: opensearch).
-
- The available configuration parameters for this kind of plugin are:
-
- - **result_type**: (optional) One of "json" or "xml", depending on the
- representation of the provider's search results. The default is "json"
-
- - **results_entry**: (mandatory) The name of the key in the provider search
- result that gives access to the result entries
-
- - **api_endpoint**: (mandatory) The endpoint of the provider's search interface
-
- - **literal_search_params**: (optional) A mapping of (search_param =>
- search_value) pairs giving search parameters to be passed as is in the search
- url query string. This is useful for example in situations where the user wants
- to pass-in a search query as it is done on the provider interface. In such a case,
- the user can put in his configuration file the query he needs to pass to the provider.
-
- - **pagination**: (mandatory) The configuration of how the pagination is done
- on the provider. It is a tree with the following nodes:
-
- - *next_page_url_tpl*: The template for pagination requests. This is a simple
- Python format string which will be resolved using the following keywords:
- ``url`` (the base url of the search endpoint), ``search`` (the query string
- corresponding to the search request), ``items_per_page`` (the number of
- items to return per page), ``skip`` (the number of items to skip) or
- ``skip_base_1`` (the number of items to skip, starting from 1) and
- ``page`` (which page to return).
-
- - *total_items_nb_key_path*: (optional) An XPath or JsonPath leading to the
- total number of results satisfying a request. This is used for providers
- which provides the total results metadata along with the result of the
- query and don't have an endpoint for querying the number of items
- satisfying a request, or for providers for which the count endpoint returns
- a json or xml document
-
- - *count_endpoint*: (optional) The endpoint for counting the number of items
- satisfying a request
-
- - *next_page_url_key_path*: (optional) A JSONPATH expression used to retrieve
- the URL of the next page in the response of the current page.
-
- - **free_text_search_operations**: (optional) A tree structure of the form::
-
- # noqa: E800
- : # e.g: $search
- union: # how to join the operations below (e.g: ' AND ' -->
- # '(op1 AND op2) AND (op3 OR op4)')
- wrapper: # a pattern for how each operation will be wrapped
- # (e.g: '({})' --> '(op1 AND op2)')
- operations: # The operations to build
- : # e.g: AND
- - # e.g:
- # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]'
- - # e.g:
- # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]'
- ...
- ...
- ...
-
- With the structure above, each operation will become a string of the form:
- '( )', then the operations will be joined together using
- the union string and finally if the number of operations is greater than 1,
- they will be wrapped as specified by the wrapper config key.
-
- The search plugins of this kind can detect when a metadata mapping is "query-able",
- and get the semantics of how to format the query string parameter that enables to
- make a query on the corresponding metadata. To make a metadata query-able, just
- configure it in the metadata mapping to be a list of 2 items, the first one being
- the specification of the query string search formatting. The later is a string
- following the specification of Python string formatting, with a special behaviour
- added to it. For example, an entry in the metadata mapping of this kind::
-
- completionTimeFromAscendingNode:
- - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
- - '$.properties.acquisition.endViewingDate'
-
- means that the search url will have a query string parameter named *"f"* with a
- value of *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done
- with the value of ``completionTimeFromAscendingNode`` being
- ``2018-12-04T12:18:00``. What happened is that
- ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp
- of the value of ``completionTimeFromAscendingNode``. This example shows all there
- is to know about the semantics of the query string formatting introduced by this
- plugin: any eodag search parameter can be referenced in the query string
- with an additional optional conversion function that is separated from it by a
- ``#`` (see :func:`~eodag.utils.format_metadata` for further details on the
- available converters). Note that for the values in the
- ``free_text_search_operations`` configuration parameter follow the same rule.
-
- :param provider: An eodag providers configuration dictionary
- :param config: Path to the user configuration file
+ query strings (e.g: opensearch). Most of the other search plugins inherit from this plugin.
+
+ :param provider: provider name
+ :param config: Search plugin configuration:
+
+ * :attr:`~eodag.config.PluginConfig.result_type` (``str``): One of ``json`` or ``xml``, depending on the
+ representation of the provider's search results. The default is ``json``.
+ * :attr:`~eodag.config.PluginConfig.results_entry` (``str``) (**mandatory**): The name of the key in the
+ provider search result that gives access to the result entries
+ * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the provider's
+ search interface
+ * :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is needed for the search request;
+ default: ``False``
+ * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
+ authentication error; only used if ``need_auth=true``
+ * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
+ requests; default: ``True``
+ * :attr:`~eodag.config.PluginConfig.dont_quote` (``List[str]``): characters that should not be quoted in the
+ url params
+ * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
+ default: ``5``
+ * :attr:`~eodag.config.PluginConfig.literal_search_params` (``Dict[str, str]``): A mapping of (search_param =>
+ search_value) pairs giving search parameters to be passed as is in the search url query string. This is useful
+ for example in situations where the user wants to add a fixed search query parameter exactly
+ as it is done on the provider interface.
+ * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
+ (**mandatory**): The configuration of how the pagination is done on the provider. It is a tree with the
+ following nodes:
+
+ * :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_tpl` (``str``) (**mandatory**): The template for
+ pagination requests. This is a simple Python format string which will be resolved using the following
+ keywords: ``url`` (the base url of the search endpoint), ``search`` (the query string corresponding
+ to the search request), ``items_per_page`` (the number of items to return per page),
+ ``skip`` (the number of items to skip) or ``skip_base_1`` (the number of items to skip,
+ starting from 1) and ``page`` (which page to return).
+ * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``): An XPath or JsonPath
+ leading to the total number of results satisfying a request. This is used for providers which provides the
+ total results metadata along with the result of the query and don't have an endpoint for querying
+ the number of items satisfying a request, or for providers for which the count endpoint
+ returns a json or xml document
+ * :attr:`~eodag.config.PluginConfig.Pagination.count_endpoint` (``str``): The endpoint for counting the number
+ of items satisfying a request
+ * :attr:`~eodag.config.PluginConfig.Pagination.count_tpl` (``str``): template for the count parameter that
+ should be added to the search request
+ * :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_key_path` (``str``): A JsonPath expression used
+ to retrieve the URL of the next page in the response of the current page.
+ * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum number of items per
+ page that the provider can handle; default: ``50``
+ * :attr:`~eodag.config.PluginConfig.Pagination.start_page` (``int``): number of the first page; default: ``1``
+
+ * :attr:`~eodag.config.PluginConfig.discover_product_types`
+ (:class:`~eodag.config.PluginConfig.DiscoverProductTypes`): configuration for product type discovery based on
+ information from the provider; It contains the keys:
+
+ * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` (``str``) (**mandatory**): url from which
+ the product types can be fetched
+ * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.result_type` (``str``): type of the provider result;
+ currently only ``json`` is supported (other types could be used in an extension of this plugin)
+ * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.results_entry` (``str``) (**mandatory**): json path
+ to the list of product types
+ * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
+ product type id
+ * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
+ (``Dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
+ from the provider result
+ * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
+ (``Dict[str, str]``): mapping for product type properties which can be parsed from the result that are not
+ product type metadata
+ * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
+ data for a single collection; used if product type metadata is not available from the endpoint given in
+ :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url`
+ * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_qs` (``str``): query string
+ to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
+ collection
+ * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
+ (``Dict[str, str]``): mapping for product type metadata returned by the endpoint given in
+ :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
+
+ * :attr:`~eodag.config.PluginConfig.sort` (:class:`~eodag.config.PluginConfig.Sort`): configuration for sorting
+ the results. It contains the keys:
+
+ * :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``List[Tuple(str, Literal["ASC", "DESC"])]``):
+ parameter and sort order by which the result will be sorted by default (if the user does not enter a
+ ``sort_by`` parameter); if not given the result will use the default sorting of the provider; Attention:
+ for some providers sorting might cause a timeout if no filters are used. In that case no default
+ sort parameters should be given. The format is::
+
+ sort_by_default:
+ - !!python/tuple [, (ASC or DESC)]
+
+ * :attr:`~eodag.config.PluginConfig.Sort.sort_by_tpl` (``str``): template for the sort parameter that is added
+ to the request; It contains the parameters `sort_param` and `sort_order` which will be replaced by user
+ input or default value. If the parameters are added as query params to a GET request, the string
+ should start with ``&``, otherwise it should be a valid json string surrounded by ``{{ }}``.
+ * :attr:`~eodag.config.PluginConfig.Sort.sort_param_mapping` (``Dict [str, str]``): mapping for the parameters
+ available for sorting
+ * :attr:`~eodag.config.PluginConfig.Sort.sort_order_mapping`
+ (``Dict[Literal["ascending", "descending"], str]``): mapping for the sort order
+ * :attr:`~eodag.config.PluginConfig.Sort.max_sort_params` (``int``): maximum number of sort parameters
+ supported by the provider; used to validate the user input to avoid failed requests or unexpected behaviour
+ (not all parameters are used in the request)
+
+ * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
+ detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
+ parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
+ just configure it in the metadata mapping to be a list of 2 items, the first one being the
+ specification of the query string search formatting. The later is a string following the
+ specification of Python string formatting, with a special behaviour added to it. For example,
+ an entry in the metadata mapping of this kind::
+
+ completionTimeFromAscendingNode:
+ - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
+ - '$.properties.acquisition.endViewingDate'
+
+ means that the search url will have a query string parameter named *"f"* with a value of
+ *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done with the value
+ of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
+ ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
+ of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
+ semantics of the query string formatting introduced by this plugin: any eodag search parameter
+ can be referenced in the query string with an additional optional conversion function that
+ is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
+ details on the available converters). Note that for the values in the
+ :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule.
+ If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but
+ it is included in the result obtained from the provider. The string indicates how the provider result should
+ be mapped to the eodag parameter.
+ * :attr:`~eodag.config.PluginConfig.discover_metadata` (:class:`~eodag.config.PluginConfig.DiscoverMetadata`):
+ configuration for the auto-discovery of queryable parameters as well as parameters returned by the provider
+ which are not in the metadata mapping. It has the attributes:
+
+ * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.auto_discovery` (``bool``): if the automatic discovery of
+ metadata is activated; default: ``False``; if false, the other parameters are not used;
+ * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_pattern` (``str``): regex string a parameter in
+ the result should match so that is used
+ * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, Dict[str, Any]]``): format
+ to add a query param given by the user and not in the metadata mapping to the requets, 'metadata' will be
+ replaced by the search param; can be a string or a dict containing
+ :attr:`~eodag.config.PluginConfig.free_text_search_operations`
+ (see :class:`~eodag.plugins.search.qssearch.ODataV4Search`)
+ * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_path` (``str``): path where the queryable
+ properties can be found in the provider result
+
+ * :attr:`~eodag.config.PluginConfig.discover_queryables`
+ (:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a
+ provider queryables endpoint; It has the following keys:
+
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid
+ for all product types
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` (``str``): url to fetch the
+ queryables for a specific product type
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.result_type` (``str``): type of the result (currently
+ only ``json`` is used)
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.results_entry` (``str``): json path to retrieve the
+ queryables from the provider result
+
+ * :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific
+ product type, can be an http url or a path to a file; the constraints are used to build queryables
+ * :attr:`~eodag.config.PluginConfig.constraints_file_dataset_key` (``str``): key which is used in the eodag
+ configuration to map the eodag product type to the provider product type; default: ``dataset``
+ * :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints
+ can be found; if not given, it is assumed that the constraints are on top level of the result, i.e.
+ the result is an array of constraints
+ * :attr:`~eodag.config.PluginConfig.stop_without_constraints_entry_key` (``bool``): if true only a provider
+ result containing `constraints_entry` is accepted as valid and used to create constraints; default: ``False``
"""
extract_properties: Dict[str, Callable[..., Dict[str, Any]]] = {
@@ -369,12 +442,10 @@ def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]:
try:
prep = PreparedSearch()
- prep.url = cast(
- str,
- self.config.discover_product_types["fetch_url"].format(
- **self.config.__dict__
- ),
- )
+ fetch_url = self.config.discover_product_types.get("fetch_url")
+ if fetch_url is None:
+ return None
+ prep.url = fetch_url.format(**self.config.__dict__)
# get auth if available
if "auth" in kwargs:
@@ -416,12 +487,14 @@ def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]:
if self.config.discover_product_types["result_type"] == "json":
resp_as_json = response.json()
# extract results from response json
- result = [
- match.value
- for match in self.config.discover_product_types[
- "results_entry"
- ].find(resp_as_json)
- ]
+ results_entry = self.config.discover_product_types["results_entry"]
+ if not isinstance(results_entry, JSONPath):
+ logger.warning(
+ f"Could not parse {self.provider} discover_product_types.results_entry"
+ f" as JSONPath: {results_entry}"
+ )
+ return None
+ result = [match.value for match in results_entry.find(resp_as_json)]
if result and isinstance(result[0], list):
result = result[0]
@@ -1182,8 +1255,49 @@ def _request(
class ODataV4Search(QueryStringSearch):
- """A specialisation of a QueryStringSearch that does a two step search to retrieve
- all products metadata"""
+ """A specialisation of a :class:`~eodag.plugins.search.qssearch.QueryStringSearch` that does a two step search to
+ retrieve all products metadata. All configuration parameters of
+ :class:`~eodag.plugins.search.qssearch.QueryStringSearch` are also available for this plugin. In addition, the
+ following parameters can be configured:
+
+ :param provider: provider name
+ :param config: Search plugin configuration:
+
+ * :attr:`~eodag.config.PluginConfig.per_product_metadata_query` (``bool``): should be set to true if the metadata
+ is not given in the search result and a two step search has to be performed; default: false
+ * :attr:`~eodag.config.PluginConfig.metadata_pre_mapping` (:class:`~eodag.config.PluginConfig.MetadataPreMapping`)
+ : a dictionary which can be used to simplify further metadata extraction. For example, going from
+ ``$.Metadata[?(@.id="foo")].value`` to ``$.Metadata.foo.value``. It has the keys:
+
+ * :attr:`~eodag.config.PluginConfig.MetadataPreMapping.metadata_path` (``str``): json path of the metadata entry
+ * :attr:`~eodag.config.PluginConfig.MetadataPreMapping.metadata_path_id` (``str``): key to get the metadata id
+ * :attr:`~eodag.config.PluginConfig.MetadataPreMapping.metadata_path_value` (``str``): key to get the metadata
+ value
+
+ * :attr:`~eodag.config.PluginConfig.free_text_search_operations`: (optional) A tree structure of the form::
+
+ # noqa: E800
+ : # e.g: $search
+ union: # how to join the operations below (e.g: ' AND ' -->
+ # '(op1 AND op2) AND (op3 OR op4)')
+ wrapper: # a pattern for how each operation will be wrapped
+ # (e.g: '({})' --> '(op1 AND op2)')
+ operations: # The operations to build
+ : # e.g: AND
+ - # e.g:
+ # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]'
+ - # e.g:
+ # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]'
+ ...
+ ...
+ ...
+
+ With the structure above, each operation will become a string of the form:
+ ``( )``, then the operations will be joined together using
+ the union string and finally if the number of operations is greater than 1,
+ they will be wrapped as specified by the wrapper config key.
+
+ """
def __init__(self, provider: str, config: PluginConfig) -> None:
super(ODataV4Search, self).__init__(provider, config)
@@ -1274,7 +1388,26 @@ def normalize_results(
class PostJsonSearch(QueryStringSearch):
- """A specialisation of a QueryStringSearch that uses POST method"""
+ """A specialisation of a QueryStringSearch that uses POST method
+ All configuration parameters available for QueryStringSearch are also available for PostJsonSearch.
+ The mappings given in metadata_mapping are used to construct a (json) body for the
+ POST request that is sent to the provider. Due to the fact that we sent a POST request and
+ not a get request, the pagination configuration will look slightly different. It has the
+ following parameters:
+
+ * **next_page_query_obj**: The additional parameters needed to add pagination information to
+ the search request. These parameters won't be included in result. This must be a json dict
+ formatted like `{{"foo":"bar"}}` because it will be passed to a `.format()` method
+ before being loaded as json.
+ * **total_items_nb_key_path** [str]: An XPath or JsonPath leading to the total number of
+ results satisfying a request. This is used for providers which provides the total results
+ metadata along with the result of the query and don't have an endpoint for querying
+ the number of items satisfying a request, or for providers for which the count endpoint
+ returns a json or xml document
+ * **max_items_per_page** [int]: The maximum number of items per page that the provider can
+ handle; default: 50
+
+ """
def _get_default_end_date_from_start_date(
self, start_datetime: str, product_type: str
@@ -1658,7 +1791,15 @@ def _request(
class StacSearch(PostJsonSearch):
- """A specialisation of a QueryStringSearch that uses generic STAC configuration"""
+ """A specialisation of PostJsonSearch that uses generic STAC configuration, it therefore
+ has the same configuration parameters (those inherited from QueryStringSearch)
+ For providers using StacSearch default values are defined for most of the parameters
+ (see stac_provider.yml). If some parameters are different for a specific provider, they
+ have to be overwritten. If certain functionalities are not available, their configuration
+ parameters have to be overwritten with `null`. E.g. if there is no queryables endpoint,
+ the `fetch_url` and `product_type_fetch_url` in the `discover_queryables` config have
+ to be set to `null`.
+ """
def __init__(self, provider: str, config: PluginConfig) -> None:
# backup results_entry overwritten by init
@@ -1740,6 +1881,8 @@ def discover_queryables(
if provider_product_type
else self.config.discover_queryables["fetch_url"]
)
+ if unparsed_fetch_url is None:
+ return None
fetch_url = unparsed_fetch_url.format(
provider_product_type=provider_product_type, **self.config.__dict__
@@ -1767,11 +1910,15 @@ def discover_queryables(
resp_as_json = response.json()
# extract results from response json
- json_queryables = [
- match.value
- for match in self.config.discover_queryables["results_entry"].find(
- resp_as_json
+ results_entry = self.config.discover_queryables["results_entry"]
+ if not isinstance(results_entry, JSONPath):
+ logger.warning(
+ f"Could not parse {self.provider} discover_queryables.results_entry"
+ f" as JSONPath: {results_entry}"
)
+ return None
+ json_queryables = [
+ match.value for match in results_entry.find(resp_as_json)
][0]
except KeyError as e:
@@ -1809,7 +1956,7 @@ def discover_queryables(
class PostJsonSearchWithStacQueryables(StacSearch, PostJsonSearch):
"""A specialisation of a :class:`~eodag.plugins.search.qssearch.PostJsonSearch` that
- uses generic STAC configuration for queryables.
+ uses generic STAC configuration for queryables (inherited from StacSearch).
"""
def __init__(self, provider: str, config: PluginConfig) -> None:
diff --git a/eodag/plugins/search/static_stac_search.py b/eodag/plugins/search/static_stac_search.py
index 277e93e5c..f7a500ec5 100644
--- a/eodag/plugins/search/static_stac_search.py
+++ b/eodag/plugins/search/static_stac_search.py
@@ -18,7 +18,7 @@
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from unittest import mock
import geojson
@@ -43,24 +43,18 @@
class StaticStacSearch(StacSearch):
"""Static STAC Catalog search plugin
-
- The available configuration parameters for this plugin are
- (to be set in provider configuration):
-
- - **api_endpoint**: (mandatory) path to the catalog (url or local system path)
-
- - **max_connections**: (optional) Maximum number of connections for HTTP requests,
- defaut is 100.
-
- - **timeout**: (mandatory) Timeout in seconds for each internal HTTP request,
- default is 5.
-
This plugin first loads all STAC items found in the catalog, and converts them to
EOProducts using StacSearch.
Then it uses crunchers to only keep products matching query parameters.
- :param provider: An eodag providers configuration dictionary
- :param config: Path to the user configuration file
+ The plugin inherits the configuration parameters from PostJsonSearch (via the StacSearch
+ inheritance) with the following particularities:
+
+ * **api_endpoint** [str] (mandatory): path to the catalog; in contrast to the api_endpoint for
+ other plugin types this can be a url or local system path.
+ * **max_connections** [int]: Maximum number of connections for HTTP requests; default: 100.
+ * **timeout** [int]: Timeout in seconds for each internal HTTP request; default: 5
+
"""
def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -88,19 +82,18 @@ def __init__(self, provider: str, config: PluginConfig) -> None:
getattr(self.config, "discover_product_types", {}).get("fetch_url")
== "{api_endpoint}/../collections"
):
- self.config.discover_product_types = {"fetch_url": None}
+ self.config.discover_product_types = {}
def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]:
"""Fetch product types list from a static STAC Catalog provider using `discover_product_types` conf
:returns: configuration dict containing fetched product types information
"""
- fetch_url = cast(
- str,
- self.config.discover_product_types["fetch_url"].format(
- **self.config.__dict__
- ),
- )
+ unformatted_fetch_url = self.config.discover_product_types.get("fetch_url")
+ if unformatted_fetch_url is None:
+ return None
+ fetch_url = unformatted_fetch_url.format(**self.config.__dict__)
+
collections = fetch_stac_collections(
fetch_url,
collection=kwargs.get("q"),
diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml
index b5ac3467d..50cdfd188 100644
--- a/eodag/resources/providers.yml
+++ b/eodag/resources/providers.yml
@@ -1589,7 +1589,7 @@
startTimeFromAscendingNode: properties.datetime
creationDate: properties.created
metadata_mapping:
- # redefine the following mapppings as the provider does not support advanced queries/filtering,
+ # redefine the following mappings as the provider does not support advanced queries/filtering,
# these parameters will not be queryable
doi: '$.properties."sci:doi"'
processingLevel: '$.properties."processing:level"'