1286 document config parameters in plugins

docs/conf.py

@@ -250,6 +250,8 @@
     "shapely": ("https://shapely.readthedocs.io/en/stable/", None),
 }
 
+suppress_warnings = ["misc.copy_overwrite"]
+
 
 def _build_finished(app, exception):
     """Post-build pages edit"""

docs/plugins.rst

@@ -261,5 +261,5 @@ Plugin configuration
    :members:
    :member-order: bysource
    :undoc-members:
-   :exclude-members: priority, products, product_type_config, yaml_loader, from_mapping, from_yaml, update, validate,
-                     yaml_dumper, yaml_tag
+   :exclude-members: priority, product_type_config, yaml_loader, from_mapping, from_yaml, update, validate, yaml_dumper,
+                     yaml_tag

eodag/config.py

@@ -258,6 +258,41 @@ class DiscoverMetadata(TypedDict):
         #: Path to the metadata in search result
         metadata_path: str
 
+    class DiscoverProductTypes(TypedDict, total=False):
+        """Configuration for product types discovery"""
+
+        #: URL from which the product types can be fetched
+        fetch_url: Optional[str]
+        #: Type of the provider result
+        result_type: str
+        #: JsonPath to the list of product types
+        results_entry: Union[JSONPath, str]
+        #: Mapping for the product type id
+        generic_product_type_id: str
+        #: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider
+        #: result
+        generic_product_type_parsable_metadata: Dict[str, str]
+        #: Mapping for product type properties which can be parsed from the result that are not product type metadata
+        generic_product_type_parsable_properties: Dict[str, str]
+        #: URL to fetch data for a single collection
+        single_collection_fetch_url: str
+        #: Query string to be added to the fetch_url to filter for a collection
+        single_collection_fetch_qs: str
+        #: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url
+        single_product_type_parsable_metadata: Dict[str, str]
+
+    class DiscoverQueryables(TypedDict, total=False):
+        """Configuration for queryables discovery"""
+
+        #: URL to fetch the queryables valid for all product types
+        fetch_url: Optional[str]
+        #: URL to fetch the queryables for a specific product type
+        product_type_fetch_url: Optional[str]
+        #: Type of the result
+        result_type: str
+        #: JsonPath to retrieve the queryables from the provider result
+        results_entry: str
+
     class OrderOnResponse(TypedDict):
         """Configuration for order on-response during download"""
 
@@ -279,15 +314,15 @@ class OrderStatusSuccess(TypedDict):
         #: Success value for status response HTTP code
         http_code: int
 
-    class OrderStatusOrdered(TypedDict):
+    class OrderStatusOrdered(TypedDict, total=False):
         """
         Configuration to identify order status ordered during download
         """
 
         #: HTTP code of the order status response
         http_code: int
 
-    class OrderStatusRequest(TypedDict):
+    class OrderStatusRequest(TypedDict, total=False):
         """
         Order status request configuration
         """
@@ -297,7 +332,7 @@ class OrderStatusRequest(TypedDict):
         #: Request hearders
         headers: Dict[str, Any]
 
-    class OrderStatusOnSuccess(TypedDict):
+    class OrderStatusOnSuccess(TypedDict, total=False):
         """Configuration for order status on-success during download"""
 
         #: Whether a new search is needed on success or not
@@ -309,7 +344,7 @@ class OrderStatusOnSuccess(TypedDict):
         #: Metadata-mapping to apply to the success status result
         metadata_mapping: Dict[str, Union[str, List[str]]]
 
-    class OrderStatus(TypedDict):
+    class OrderStatus(TypedDict, total=False):
         """Configuration for order status during download"""
 
         #: Order status request configuration
@@ -325,6 +360,16 @@ class OrderStatus(TypedDict):
         #: Configuration for order status on-success during download
         on_success: PluginConfig.OrderStatusOnSuccess
 
+    class MetadataPreMapping(TypedDict, total=False):
+        """Configuration which can be used to simplify further metadata extraction"""
+
+        #: JsonPath of the metadata entry
+        metadata_path: str
+        #: Key to get the metadata id
+        metadata_path_id: str
+        #: Key to get the metadata value
+        metadata_path_value: str
+
     #: :class:`~eodag.plugins.base.PluginTopic` The name of the plugin class to use to instantiate the plugin object
     name: str
     #: :class:`~eodag.plugins.base.PluginTopic` Plugin type
@@ -335,12 +380,12 @@ class OrderStatus(TypedDict):
     s3_bucket: str
     #: :class:`~eodag.plugins.base.PluginTopic` Authentication error codes
     auth_error_code: Union[int, List[int]]
+    #: :class:`~eodag.plugins.base.PluginTopic` Time to wait until request timeout in seconds
+    timeout: float
 
     # search & api -----------------------------------------------------------------------------------------------------
     # copied from ProviderConfig in PluginManager.get_search_plugins()
     priority: int
-    # copied from ProviderConfig in PluginManager.get_search_plugins()
-    products: Dict[str, Any]
     # per product type metadata-mapping, set in core._prepare_search
     product_type_config: Dict[str, Any]
 
@@ -360,19 +405,35 @@ class OrderStatus(TypedDict):
     #: :class:`~eodag.plugins.search.base.Search` Configuration for the metadata auto-discovery
     discover_metadata: PluginConfig.DiscoverMetadata
     #: :class:`~eodag.plugins.search.base.Search` Configuration for the product types auto-discovery
-    discover_product_types: Dict[str, Any]
+    discover_product_types: PluginConfig.DiscoverProductTypes
     #: :class:`~eodag.plugins.search.base.Search` Configuration for the queryables auto-discovery
-    discover_queryables: Dict[str, Any]
+    discover_queryables: PluginConfig.DiscoverQueryables
     #: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
     metadata_mapping: Dict[str, Union[str, List[str]]]
     #: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables
     constraints_file_url: str
+    #: :class:`~eodag.plugins.search.base.Search`
+    #: Key which is used in the eodag configuration to map the eodag product type to the provider product type
+    constraints_file_dataset_key: str
+    #: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found
+    constraints_entry: str
+    #: :class:`~eodag.plugins.search.base.Search`
+    #: Whether only a provider result containing constraints_entry is accepted as valid and used to create constraints
+    #: or not
+    stop_without_constraints_entry_key: bool
     #: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
     remove_from_queryables: List[str]
+    #: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string
+    literal_search_params: Dict[str, str]
+    #: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params
+    dont_quote: List[str]
     #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build
     free_text_search_operations: Dict[str, Any]
+    #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search
+    #: result and a two step search has to be performed
+    per_product_metadata_query: bool
     #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict used to simplify further metadata extraction
-    metadata_pre_mapping: Dict[str, Any]
+    metadata_pre_mapping: PluginConfig.MetadataPreMapping
     #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to which the data request shall be sent
     data_request_url: str
     #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to fetch the status of the data request
@@ -389,8 +450,6 @@ class OrderStatus(TypedDict):
     #: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
     #: Maximum number of connections for HTTP requests
     max_connections: int
-    #: :class:`~eodag.plugins.search.base.Search` Time to wait until request timeout in seconds
-    timeout: float
     #: :class:`~eodag.plugins.search.build_search_result.BuildSearchResult`
     #: Whether end date should be excluded from search request or not
     end_date_excluded: bool
@@ -410,6 +469,12 @@ class OrderStatus(TypedDict):
     output_extension: str
     #: :class:`~eodag.plugins.download.base.Download` Whether the directory structure should be flattened or not
     flatten_top_dirs: bool
+    #: :class:`~eodag.plugins.download.base.Download` Level in extracted path tree where to find data
+    archive_depth: int
+    #: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``downloadLink`` or not
+    ignore_assets: bool
+    #: :class:`~eodag.plugins.download.base.Download` Product type specific configuration
+    products: Dict[str, Dict[str, Any]]
     #: :class:`~eodag.plugins.download.http.HTTPDownload` Whether the product has to be ordered to download it or not
     order_enabled: bool
     #: :class:`~eodag.plugins.download.http.HTTPDownload` HTTP request method for the order request
@@ -425,6 +490,8 @@ class OrderStatus(TypedDict):
     #: :class:`~eodag.plugins.download.http.HTTPDownload`
     #: Do not authenticate the download request but only the order and order status ones
     no_auth_download: bool
+    #: :class:`~eodag.plugins.download.http.HTTPDownload` Parameters to be added to the query params of the request
+    dl_url_params: Dict[str, str]
     #: :class:`~eodag.plugins.download.s3rest.S3RestDownload`
     #: At which level of the path part of the url the bucket can be found
     bucket_path_level: int

eodag/plugins/apis/ecmwf.py

@@ -68,10 +68,19 @@ class EcmwfApi(Api, BuildPostSearchResult):
     is in query), or on MARS Operational Archive (if ``dataset`` parameter is not in
     query).
 
-    This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility,
-    :class:`~eodag.plugins.download.base.Download` for download methods, and
-    :class:`~eodag.plugins.search.qssearch.QueryStringSearch` for metadata-mapping and
-    query build methods.
+    This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility and
+    :class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult` for the creation
+    of the search result.
+
+    The configuration parameters for this plugin are:
+
+    * **type** [str] (mandatory): EcmwfApi
+    * **api_endpoint** [str] (mandatory): url of the ecmwf api
+    * **metadata_mapping** [Dict[str, Union[str, list]]]: how parameters should be mapped between
+      the provider and eodag; If a string is given, this is the mapping parameter returned by
+      provider -> eodag parameter. If a list with 2 elements is given, the first one is the mapping
+      eodag parameter -> provider query parameters and the second one the mapping provider result
+      parameter -> eodag parameter
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:

eodag/plugins/apis/usgs.py

@@ -68,7 +68,22 @@
 
 
 class UsgsApi(Api):
-    """A plugin that enables to query and download data on the USGS catalogues"""
+    """A plugin that enables to query and download data on the USGS catalogues
+
+    The configuration parameters for this plugin are:
+
+    * **type** [str] (mandatory): UsgsApi
+    * **pagination** [Dict[str, Any]] (mandatory): dict containing parameters for pagination; should contain the
+      key total_items_nb_key_path which is indicating the key for the number of total items in the provider result
+    * **ssl_verify** [bool]: if the ssl certificates should be verified in the download request; default: True
+    * **need_auth** [bool]: if authentication is required for search; default: False
+    * **extract** [bool]: if the content of the downloaded file should be extracted; default: True
+    * **order_enabled** [bool]: if the product has to be ordered to download it; default: False
+    * **metadata_mapping** [Dict[str, Union[str, list]]]: how parameters should be mapped between the provider and eodag
+      If a string is given, this is the mapping parameter returned by provider -> eodag parameter. If a list with
+      2 elements is given, the first one is the mapping eodag parameter -> provider query parameters and the second one
+      the mapping provider result parameter -> eodag parameter
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(UsgsApi, self).__init__(provider, config)

eodag/plugins/authentication/aws_auth.py

@@ -30,14 +30,22 @@
 class AwsAuth(Authentication):
     """AWS authentication plugin
 
-    Authentication will use the first valid method within the following ones:
+    Authentication will use the first valid method within the following ones depending on which
+    parameters are available in the configuration:
 
-    - auth anonymously using no-sign-request
-    - auth using ``aws_profile``
-    - auth using ``aws_access_key_id`` and ``aws_secret_access_key``
+    * auth anonymously using no-sign-request
+    * auth using ``aws_profile``
+    * auth using ``aws_access_key_id`` and ``aws_secret_access_key``
       (optionally ``aws_session_token``)
-    - auth using current environment (AWS environment variables and/or ``~/aws/*``),
+    * auth using current environment (AWS environment variables and/or ``~/aws/*``),
       will be skipped if AWS credentials are filled in eodag conf
+
+    The configuration parameters for this plugin are:
+
+    * **type** [str] (mandatory): AwsAuth
+    * **auth_error_code** [int] (mandatory for creodias_s3): which error code is returned
+      in case of an authentication error
+
     """
 
     s3_client: S3Client

eodag/plugins/authentication/generic.py

@@ -29,7 +29,16 @@
 
 
 class GenericAuth(Authentication):
-    """GenericAuth authentication plugin"""
+    """GenericAuth authentication plugin (authentication using username and password)
+
+    The configuration parameters for this plugin are:
+
+    * **type** [str] (mandatory): GenericAuth
+    * **method** [str]: specifies if digest authentication (digest) or basic authentication
+      (basic) should be used; default: basic
+
+    The mandatory parameters that have to be added in the eodag config are username and password.
+    """
 
     def authenticate(self) -> AuthBase:
         """Authenticate"""

eodag/plugins/authentication/header.py

@@ -34,7 +34,12 @@ class HTTPHeaderAuth(Authentication):
     This plugin enables implementation of custom HTTP authentication scheme (other than Basic, Digest, Token
     negotiation et al.) using HTTP headers.
 
-    The plugin is configured as follows in the providers config file::
+    The configuration parameters for this plugin are:
+
+    * **type** [str] (mandatory): HTTPHeaderAuth
+    * **headers** [Dict[str, str]]: dictionary containing all keys/value pairs that should be added to the headers
+
+    Below an example for the configuration in the providers config file is shown::
 
         provider:
             ...

eodag/plugins/authentication/keycloak.py

@@ -41,7 +41,19 @@
 class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
     """Authentication plugin using Keycloak and OpenId Connect.
 
-    This plugin request a token and use it through a query-string or a header.
+    This plugin requests a token which is added to a query-string or a header for authentication.
+    The configuration parameters for this plugin are:
+
+    * **type** [str] (mandatory): KeycloakOIDCPasswordAuth
+    * **auth_base_uri** [str] (mandatory): base url used in the request to fetch the token
+    * **realm** [str] (mandatory): keycloak realm
+    * **client_id** [str] (mandatory): keycloak client id
+    * **client_secret** [str] (mandatory): keycloak client secret, set to null if no secret is used
+    * **token_provision** [str] (mandatory): if the token should be added to the query string (qs)
+      or to the header (header)
+    * **token_qs_key** [str] (mandatory if token_provision=qs): key of the param added to the query string
+    * **auth_error_code** [int]: which error code is returned in case of an authentication error
+    * **ssl_verify** [bool]: if the ssl certificates should be verified in the token request; default: True
 
     Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link
     `http://example.com?foo=bar` will become

eodag/plugins/authentication/oauth.py

@@ -26,7 +26,14 @@
 
 
 class OAuth(Authentication):
-    """OAuth authentication plugin"""
+    """OAuth authentication plugin
+
+    The configuration parameters for this plugin are:
+
+    * **type** [str] (mandatory): OAuth
+
+    The mandatory parameters that have to be added in the eodag config are aws_access_key_id and aws_secret_access_key.
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(OAuth, self).__init__(provider, config)