docs: docstrings jaml (#1915)

Co-authored-by: Nan Wang <[email protected]>

jina/jaml/__init__.py

@@ -6,6 +6,7 @@
 from typing import Dict, Any, Union, TextIO, Optional
 
 import yaml
+from yaml.constructor import FullConstructor
 
 from .helper import JinaResolver, JinaLoader, parse_config_source, load_py_modules
 
@@ -73,14 +74,17 @@ def load(stream,
              context: Dict[str, Any] = None):
         """Parse the first YAML document in a stream and produce the corresponding Python object.
 
-        :param substitute: substitute environment, internal reference and context variables.
-        :param context: context replacement variables in a dict, the value of the dict is the replacement.
-
         .. note::
             :class:`BaseFlow`, :class:`BaseExecutor`, :class:`BaseDriver`
             and all their subclasses have already implemented JAML interfaces,
             to load YAML config into objects, please use :meth:`Flow.load_config`,
             :meth:`BaseExecutor.load_config`, etc.
+
+        :param substitute: substitute environment, internal reference and context variables.
+        :param context: context replacement variables in a dict, the value of the dict is the replacement.
+        :param stream: the stream to load
+        :return: the Python object
+
         """
         r = yaml.load(stream, Loader=JinaLoader)
         if substitute:
@@ -90,6 +94,9 @@ def load(stream,
     @staticmethod
     def load_no_tags(stream, **kwargs):
         """Load yaml object but ignore all customized tags, e.g. !Executor, !Driver, !Flow
+        :param stream: the output stream
+        :param **kwargs: other kwargs
+        :return: the Python object
         """
         safe_yml = '\n'.join(v if not re.match(r'^[\s-]*?!\b', v) else v.replace('!', '__cls: ') for v in stream)
         return JAML.load(safe_yml, **kwargs)
@@ -228,6 +235,11 @@ def dump(data, stream=None, **kwargs):
         """
         Serialize a Python object into a YAML stream.
         If stream is None, return the produced string instead.
+
+        :param data: the data to serialize
+        :param stream: the output stream
+        :param **kwargs: other kwargs
+        :return: the yaml output
         """
         return yaml.dump(data, stream=stream, default_flow_style=False, sort_keys=False, **kwargs)
 
@@ -237,6 +249,9 @@ def register(cls):
             - if it has attribute yaml_tag use that to register, else use class name
             - if it has methods to_yaml/from_yaml use those to dump/load else dump attributes
               as mapping
+
+        :param cls: the class to register
+        :return: the registered class
         """
 
         tag = getattr(cls, 'yaml_tag', '!' + cls.__name__)
@@ -289,18 +304,24 @@ def _to_yaml(cls, representer, data):
         .. warning::
             This function should not be used directly, please use :meth:`save_config`.
 
+        :param representer: the class that will serialize
+        :param data: the data to serialize
+        :return: the node's representation
         """
         from .parsers import get_parser
         tmp = get_parser(cls, version=data._version).dump(data)
         return representer.represent_mapping('!' + cls.__name__, tmp)
 
     @classmethod
-    def _from_yaml(cls, constructor, node):
+    def _from_yaml(cls, constructor: FullConstructor, node):
         """A low-level interface required by :mod:`pyyaml` load interface
 
         .. warning::
             This function should not be used directly, please use :meth:`load_config`.
 
+        :param constructor: the class that will construct
+        :param node: the node to traverse
+        :return: the parser associated with the class
         """
         data = constructor.construct_mapping(node, deep=True)
         from .parsers import get_parser
@@ -310,7 +331,6 @@ def save_config(self, filename: Optional[str] = None):
         """Save the object's config into a YAML file
 
         :param filename: file path of the yaml file, if not given then :attr:`config_abspath` is used
-        :return: successfully dumped or not
         """
         f = filename or getattr(self, 'config_abspath', None)
         if not f:
@@ -331,12 +351,6 @@ def load_config(cls,
         implements :class:`JAMLCompatible` mixin can enjoy this feature, e.g. :class:`BaseFlow`,
         :class:`BaseExecutor`, :class:`BaseDriver` and all their subclasses.
 
-        :param source: the multi-kind source of the configs.
-        :param allow_py_modules: allow importing plugins specified by ``py_modules`` in YAML at any levels
-        :param substitute: substitute environment, internal reference and context variables.
-        :param context: context replacement variables in a dict, the value of the dict is the replacement.
-        :return: :class:`JAMLCompatible` object
-
         Support substitutions in YAML:
             - Environment variables: `${{ENV.VAR}}` (recommended), ``${{VAR}}``, ``$VAR``.
             - Context dict (``context``): ``${{VAR}}``(recommended), ``$VAR``.
@@ -370,6 +384,13 @@ def load_config(cls,
             # disable substitute
             b = BaseExecutor.load_config('a.yml', substitute=False)
 
+        # noqa: DAR401
+        :param source: the multi-kind source of the configs.
+        :param allow_py_modules: allow importing plugins specified by ``py_modules`` in YAML at any levels
+        :param substitute: substitute environment, internal reference and context variables.
+        :param context: context replacement variables in a dict, the value of the dict is the replacement.
+        :param **kwargs: **kwargs for parse_config_source
+        :return: :class:`JAMLCompatible` object
         """
         stream, s_path = parse_config_source(source, **kwargs)
         with stream as fp:
@@ -396,11 +417,12 @@ def load_config(cls,
     def inject_config(cls, raw_config: Dict, *args, **kwargs) -> Dict:
         """Inject/modify the config before loading it into an object.
 
-        :param raw_config: raw config to work on
-
-
          .. note::
             This function is most likely to be overridden by its subclass.
 
+        :param raw_config: raw config to work on
+        :param *args: *args
+        :param **kwargs: **kwargs
+        :return: the config
         """
         return raw_config

jina/jaml/helper.py

@@ -87,8 +87,20 @@ def parse_config_source(path: Union[str, TextIO, Dict],
                         allow_dict: bool = True,
                         allow_json: bool = True,
                         *args, **kwargs) -> Tuple[TextIO, Optional[str]]:
-    """ Check if the text or text stream is valid
-
+    """Check if the text or text stream is valid.
+
+    # noqa: DAR401
+    :param path: the multi-kind source of the configs.
+    :param allow_stream: flag
+    :param allow_yaml_file: flag
+    :param allow_builtin_resource: flag
+    :param allow_raw_yaml_content: flag
+    :param allow_raw_driver_yaml_content: flag
+    :param allow_class_type: flag
+    :param allow_dict: flag
+    :param allow_json: flag
+    :param *args: *args
+    :param **kwargs: **kwargs
     :return: a tuple, the first element is the text stream, the second element is the file path associate to it
             if available.
     """
@@ -155,6 +167,10 @@ def complete_path(path: str, extra_search_paths: Optional[Tuple[str]] = None) ->
 
 def _search_file_in_paths(path, extra_search_paths: Optional[Tuple[str]] = None):
     """searches in all dirs of the PATH environment variable and all dirs of files used in the call stack.
+
+    :param path: the path to search for
+    :param extra_search_paths: any extra locations to search for
+    :return: the path (if found)
     """
     import inspect
     search_paths = []
@@ -177,7 +193,11 @@ def _search_file_in_paths(path, extra_search_paths: Optional[Tuple[str]] = None)
 
 
 def load_py_modules(d: Dict, extra_search_paths: Optional[Tuple[str]] = None) -> None:
-    """Find 'py_modules' in the dict recursively and then load them """
+    """Find 'py_modules' in the dict recursively and then load them
+
+    :param d: the dictionary to traverse
+    :param extra_search_paths: any extra paths to search
+    """
     mod = []
 
     def _finditem(obj, key='py_modules'):

jina/jaml/parsers/__init__.py

@@ -47,11 +47,11 @@ def _get_default_parser():
 
 
 def get_parser(cls: Type['JAMLCompatible'], version: Optional[str]) -> 'VersionedYAMLParser':
-    """ Get parser given the YAML version
-
+    """
+    # noqa: DAR401
     :param cls: the target class to parse
     :param version: yaml version number in "MAJOR[.MINOR]" format
-    :return:
+    :return: parser given the YAML version
     """
     all_parsers, legacy_parser = _get_all_parser(cls)
     if version:
@@ -78,6 +78,7 @@ def get_parser(cls: Type['JAMLCompatible'], version: Optional[str]) -> 'Versione
 def get_supported_versions(cls) -> List[str]:
     """List all supported versions
 
+    :param cls: the class to check
     :return: supported versions sorted alphabetically
     """
     all_parsers, _ = _get_all_parser(cls)

jina/jaml/parsers/base.py

@@ -20,14 +20,15 @@ class VersionedYAMLParser:
     def parse(self, cls: type, data: Dict) -> Union['BaseFlow', 'BaseExecutor', 'BaseDriver']:
         """Return the Flow YAML parser given the syntax version number
 
+        # noqa: DAR401
         :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type
         :param data: flow yaml file loaded as python dict
         """
         raise NotImplementedError
 
     def dump(self, data: Union['BaseFlow', 'BaseExecutor', 'BaseDriver']) -> Dict:
         """Return the dictionary given a versioned flow object
-
+        # noqa: DAR401
         :param data: versioned flow object
         """
         raise NotImplementedError

jina/jaml/parsers/default/v1.py

@@ -8,10 +8,10 @@ class V1Parser(VersionedYAMLParser):
     version = '1'  # the version number this parser designed for
 
     def parse(self, cls: Type['JAMLCompatible'], data: Dict) -> 'JAMLCompatible':
-        """Return the YAML parser given the syntax version number
-
+        """
         :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type
         :param data: flow yaml file loaded as python dict
+        :return: the YAML parser given the syntax version number
         """
         expanded_data = JAML.expand_dict(data, None)
         if 'with' in data:
@@ -21,9 +21,9 @@ def parse(self, cls: Type['JAMLCompatible'], data: Dict) -> 'JAMLCompatible':
         return obj
 
     def dump(self, data: 'JAMLCompatible') -> Dict:
-        """Return the dictionary given a versioned flow object
-
+        """
         :param data: versioned flow object
+        :return: the dictionary given a versioned flow object
         """
         a = V1Parser._dump_instance_to_yaml(data)
         r = {}

jina/jaml/parsers/driver/legacy.py

@@ -8,19 +8,19 @@ class LegacyParser(VersionedYAMLParser):
     version = 'legacy'  # the version number this parser designed for
 
     def parse(self, cls: Type['BaseDriver'], data: Dict) -> 'BaseDriver':
-        """Return the Flow YAML parser given the syntax version number
-
+        """
         :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type
         :param data: flow yaml file loaded as python dict
+        :return: the Flow YAML parser given the syntax version number
         """
 
         obj = cls(**data.get('with', {}))
         return obj
 
     def dump(self, data: 'BaseDriver') -> Dict:
-        """Return the dictionary given a versioned flow object
-
+        """
         :param data: versioned flow object
+        :return: dictionary given a versioned flow object
         """
         a = {k: v for k, v in data._init_kwargs_dict.items()}
         r = {}

jina/jaml/parsers/executor/legacy.py

@@ -1,6 +1,6 @@
 import os
 import inspect
-from typing import Dict, Any, Type
+from typing import Dict, Any, Type, Set
 from functools import reduce
 
 from ..base import VersionedYAMLParser
@@ -14,22 +14,23 @@ class LegacyParser(VersionedYAMLParser):
     @staticmethod
     def _get_all_arguments(class_):
         """
-        Returns a list of all the arguments of all the classess from which `class_`
-        inherits
-            
+
         :param class_: target class from which we want to retrieve arguments
+        :return: all the arguments of all the classes from which `class_` inherits
         """
         def get_class_arguments(class_):
             """
-            Retrieves a list containing the arguments from `class_`
+            :param class_: the class to check
+            :return: a list containing the arguments from `class_`
             """
             signature = inspect.signature(class_.__init__)
             class_arguments = [p.name for p in signature.parameters.values()]
             return class_arguments
 
-        def accumulate_classes(cls):
+        def accumulate_classes(cls) -> Set[Type]:
             """
-            Retrieves all classes from which cls inherits from
+            :param cls: the class to check
+            :return: all classes from which cls inherits from
             """
             def _accumulate_classes(c, cs):
                 cs.append(c)
@@ -74,10 +75,10 @@ def _get_dump_path_from_config(meta_config: Dict):
                     return bin_dump_path
 
     def parse(self, cls: Type['BaseExecutor'], data: Dict) -> 'BaseExecutor':
-        """Return the Flow YAML parser given the syntax version number
-
+        """
         :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type
         :param data: flow yaml file loaded as python dict
+        :return: the Flow YAML parser given the syntax version number
         """
         _meta_config = get_default_metas()
         _meta_config.update(data.get('metas', {}))
@@ -141,9 +142,9 @@ def parse(self, cls: Type['BaseExecutor'], data: Dict) -> 'BaseExecutor':
         return obj
 
     def dump(self, data: 'BaseExecutor') -> Dict:
-        """Return the dictionary given a versioned flow object
-
+        """
         :param data: versioned executor object
+        :return: the dictionary given a versioned flow object
         """
         # note: we only save non-default property for the sake of clarity
         _defaults = get_default_metas()

jina/jaml/parsers/flow/legacy.py

@@ -11,10 +11,10 @@ class LegacyParser(VersionedYAMLParser):
     version = 'legacy'  # the version number this parser designed for
 
     def parse(self, cls: Type['BaseFlow'], data: Dict) -> 'BaseFlow':
-        """Return the Flow YAML parser given the syntax version number
-
+        """
         :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type
         :param data: flow yaml file loaded as python dict
+        :return: the Flow YAML parser given the syntax version number
         """
         p = data.get('with', {})  # type: Dict[str, Any]
         a = p.pop('args') if 'args' in p else ()
@@ -34,9 +34,9 @@ def parse(self, cls: Type['BaseFlow'], data: Dict) -> 'BaseFlow':
         return obj
 
     def dump(self, data: 'BaseFlow') -> Dict:
-        """Return the dictionary given a versioned flow object
-
+        """
         :param data: versioned flow object
+        :return: the dictionary given a versioned flow object
         """
         r = {}
         if data._version:

jina/jaml/parsers/flow/v1.py

@@ -9,7 +9,9 @@
 
 
 def _get_taboo():
-    """Get a set of keys that should not be dumped"""
+    """
+    :return: set of keys that should not be dumped
+    """
     return {k.dest for k in set_pod_parser()._actions if k.help == argparse.SUPPRESS}
 
 
@@ -48,9 +50,10 @@ class V1Parser(VersionedYAMLParser):
     version = '1'  # the version number this parser designed for
 
     def parse(self, cls: type, data: Dict) -> 'Flow':
-        """Return the Flow YAML parser given the syntax version number
-
+        """
+        :param cls: the class registered for dumping/loading
         :param data: flow yaml file loaded as python dict
+        :return: the Flow YAML parser given the syntax version number
         """
         envs = data.get('env', {})  # type: Dict[str, str]
         p = data.get('with', {})  # type: Dict[str, Any]
@@ -73,9 +76,9 @@ def parse(self, cls: type, data: Dict) -> 'Flow':
         return obj
 
     def dump(self, data: 'Flow') -> Dict:
-        """Return the dictionary given a versioned flow object
-
+        """
         :param data: versioned flow object
+        :return: the dictionary given a versioned flow object
         """
         r = {}
         if data._version: