diff --git a/jina/jaml/__init__.py b/jina/jaml/__init__.py index 50a4e26f8422a..8ccaa43c9f82c 100644 --- a/jina/jaml/__init__.py +++ b/jina/jaml/__init__.py @@ -6,6 +6,7 @@ from typing import Dict, Any, Union, TextIO, Optional import yaml +from yaml.constructor import FullConstructor from .helper import JinaResolver, JinaLoader, parse_config_source, load_py_modules @@ -73,14 +74,17 @@ def load(stream, context: Dict[str, Any] = None): """Parse the first YAML document in a stream and produce the corresponding Python object. - :param substitute: substitute environment, internal reference and context variables. - :param context: context replacement variables in a dict, the value of the dict is the replacement. - .. note:: :class:`BaseFlow`, :class:`BaseExecutor`, :class:`BaseDriver` and all their subclasses have already implemented JAML interfaces, to load YAML config into objects, please use :meth:`Flow.load_config`, :meth:`BaseExecutor.load_config`, etc. + + :param substitute: substitute environment, internal reference and context variables. + :param context: context replacement variables in a dict, the value of the dict is the replacement. + :param stream: the stream to load + :return: the Python object + """ r = yaml.load(stream, Loader=JinaLoader) if substitute: @@ -90,6 +94,9 @@ def load(stream, @staticmethod def load_no_tags(stream, **kwargs): """Load yaml object but ignore all customized tags, e.g. !Executor, !Driver, !Flow + :param stream: the output stream + :param **kwargs: other kwargs + :return: the Python object """ safe_yml = '\n'.join(v if not re.match(r'^[\s-]*?!\b', v) else v.replace('!', '__cls: ') for v in stream) return JAML.load(safe_yml, **kwargs) @@ -228,6 +235,11 @@ def dump(data, stream=None, **kwargs): """ Serialize a Python object into a YAML stream. If stream is None, return the produced string instead. + + :param data: the data to serialize + :param stream: the output stream + :param **kwargs: other kwargs + :return: the yaml output """ return yaml.dump(data, stream=stream, default_flow_style=False, sort_keys=False, **kwargs) @@ -237,6 +249,9 @@ def register(cls): - if it has attribute yaml_tag use that to register, else use class name - if it has methods to_yaml/from_yaml use those to dump/load else dump attributes as mapping + + :param cls: the class to register + :return: the registered class """ tag = getattr(cls, 'yaml_tag', '!' + cls.__name__) @@ -289,18 +304,24 @@ def _to_yaml(cls, representer, data): .. warning:: This function should not be used directly, please use :meth:`save_config`. + :param representer: the class that will serialize + :param data: the data to serialize + :return: the node's representation """ from .parsers import get_parser tmp = get_parser(cls, version=data._version).dump(data) return representer.represent_mapping('!' + cls.__name__, tmp) @classmethod - def _from_yaml(cls, constructor, node): + def _from_yaml(cls, constructor: FullConstructor, node): """A low-level interface required by :mod:`pyyaml` load interface .. warning:: This function should not be used directly, please use :meth:`load_config`. + :param constructor: the class that will construct + :param node: the node to traverse + :return: the parser associated with the class """ data = constructor.construct_mapping(node, deep=True) from .parsers import get_parser @@ -310,7 +331,6 @@ def save_config(self, filename: Optional[str] = None): """Save the object's config into a YAML file :param filename: file path of the yaml file, if not given then :attr:`config_abspath` is used - :return: successfully dumped or not """ f = filename or getattr(self, 'config_abspath', None) if not f: @@ -331,12 +351,6 @@ def load_config(cls, implements :class:`JAMLCompatible` mixin can enjoy this feature, e.g. :class:`BaseFlow`, :class:`BaseExecutor`, :class:`BaseDriver` and all their subclasses. - :param source: the multi-kind source of the configs. - :param allow_py_modules: allow importing plugins specified by ``py_modules`` in YAML at any levels - :param substitute: substitute environment, internal reference and context variables. - :param context: context replacement variables in a dict, the value of the dict is the replacement. - :return: :class:`JAMLCompatible` object - Support substitutions in YAML: - Environment variables: `${{ENV.VAR}}` (recommended), ``${{VAR}}``, ``$VAR``. - Context dict (``context``): ``${{VAR}}``(recommended), ``$VAR``. @@ -370,6 +384,13 @@ def load_config(cls, # disable substitute b = BaseExecutor.load_config('a.yml', substitute=False) + # noqa: DAR401 + :param source: the multi-kind source of the configs. + :param allow_py_modules: allow importing plugins specified by ``py_modules`` in YAML at any levels + :param substitute: substitute environment, internal reference and context variables. + :param context: context replacement variables in a dict, the value of the dict is the replacement. + :param **kwargs: **kwargs for parse_config_source + :return: :class:`JAMLCompatible` object """ stream, s_path = parse_config_source(source, **kwargs) with stream as fp: @@ -396,11 +417,12 @@ def load_config(cls, def inject_config(cls, raw_config: Dict, *args, **kwargs) -> Dict: """Inject/modify the config before loading it into an object. - :param raw_config: raw config to work on - - .. note:: This function is most likely to be overridden by its subclass. + :param raw_config: raw config to work on + :param *args: *args + :param **kwargs: **kwargs + :return: the config """ return raw_config diff --git a/jina/jaml/helper.py b/jina/jaml/helper.py index a879b8b2c2cc8..5927b02061478 100644 --- a/jina/jaml/helper.py +++ b/jina/jaml/helper.py @@ -87,8 +87,20 @@ def parse_config_source(path: Union[str, TextIO, Dict], allow_dict: bool = True, allow_json: bool = True, *args, **kwargs) -> Tuple[TextIO, Optional[str]]: - """ Check if the text or text stream is valid - + """Check if the text or text stream is valid. + + # noqa: DAR401 + :param path: the multi-kind source of the configs. + :param allow_stream: flag + :param allow_yaml_file: flag + :param allow_builtin_resource: flag + :param allow_raw_yaml_content: flag + :param allow_raw_driver_yaml_content: flag + :param allow_class_type: flag + :param allow_dict: flag + :param allow_json: flag + :param *args: *args + :param **kwargs: **kwargs :return: a tuple, the first element is the text stream, the second element is the file path associate to it if available. """ @@ -155,6 +167,10 @@ def complete_path(path: str, extra_search_paths: Optional[Tuple[str]] = None) -> def _search_file_in_paths(path, extra_search_paths: Optional[Tuple[str]] = None): """searches in all dirs of the PATH environment variable and all dirs of files used in the call stack. + + :param path: the path to search for + :param extra_search_paths: any extra locations to search for + :return: the path (if found) """ import inspect search_paths = [] @@ -177,7 +193,11 @@ def _search_file_in_paths(path, extra_search_paths: Optional[Tuple[str]] = None) def load_py_modules(d: Dict, extra_search_paths: Optional[Tuple[str]] = None) -> None: - """Find 'py_modules' in the dict recursively and then load them """ + """Find 'py_modules' in the dict recursively and then load them + + :param d: the dictionary to traverse + :param extra_search_paths: any extra paths to search + """ mod = [] def _finditem(obj, key='py_modules'): diff --git a/jina/jaml/parsers/__init__.py b/jina/jaml/parsers/__init__.py index 7d04d177e52cb..504769af3a7c9 100644 --- a/jina/jaml/parsers/__init__.py +++ b/jina/jaml/parsers/__init__.py @@ -47,11 +47,11 @@ def _get_default_parser(): def get_parser(cls: Type['JAMLCompatible'], version: Optional[str]) -> 'VersionedYAMLParser': - """ Get parser given the YAML version - + """ + # noqa: DAR401 :param cls: the target class to parse :param version: yaml version number in "MAJOR[.MINOR]" format - :return: + :return: parser given the YAML version """ all_parsers, legacy_parser = _get_all_parser(cls) if version: @@ -78,6 +78,7 @@ def get_parser(cls: Type['JAMLCompatible'], version: Optional[str]) -> 'Versione def get_supported_versions(cls) -> List[str]: """List all supported versions + :param cls: the class to check :return: supported versions sorted alphabetically """ all_parsers, _ = _get_all_parser(cls) diff --git a/jina/jaml/parsers/base.py b/jina/jaml/parsers/base.py index 424111d433da7..14725285251eb 100644 --- a/jina/jaml/parsers/base.py +++ b/jina/jaml/parsers/base.py @@ -20,6 +20,7 @@ class VersionedYAMLParser: def parse(self, cls: type, data: Dict) -> Union['BaseFlow', 'BaseExecutor', 'BaseDriver']: """Return the Flow YAML parser given the syntax version number + # noqa: DAR401 :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type :param data: flow yaml file loaded as python dict """ @@ -27,7 +28,7 @@ def parse(self, cls: type, data: Dict) -> Union['BaseFlow', 'BaseExecutor', 'Bas def dump(self, data: Union['BaseFlow', 'BaseExecutor', 'BaseDriver']) -> Dict: """Return the dictionary given a versioned flow object - + # noqa: DAR401 :param data: versioned flow object """ raise NotImplementedError diff --git a/jina/jaml/parsers/default/v1.py b/jina/jaml/parsers/default/v1.py index 68b9a28b3398c..b0cdf113215c5 100644 --- a/jina/jaml/parsers/default/v1.py +++ b/jina/jaml/parsers/default/v1.py @@ -8,10 +8,10 @@ class V1Parser(VersionedYAMLParser): version = '1' # the version number this parser designed for def parse(self, cls: Type['JAMLCompatible'], data: Dict) -> 'JAMLCompatible': - """Return the YAML parser given the syntax version number - + """ :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type :param data: flow yaml file loaded as python dict + :return: the YAML parser given the syntax version number """ expanded_data = JAML.expand_dict(data, None) if 'with' in data: @@ -21,9 +21,9 @@ def parse(self, cls: Type['JAMLCompatible'], data: Dict) -> 'JAMLCompatible': return obj def dump(self, data: 'JAMLCompatible') -> Dict: - """Return the dictionary given a versioned flow object - + """ :param data: versioned flow object + :return: the dictionary given a versioned flow object """ a = V1Parser._dump_instance_to_yaml(data) r = {} diff --git a/jina/jaml/parsers/driver/legacy.py b/jina/jaml/parsers/driver/legacy.py index 58ff8d7553cc6..cf53a7e169d9b 100644 --- a/jina/jaml/parsers/driver/legacy.py +++ b/jina/jaml/parsers/driver/legacy.py @@ -8,19 +8,19 @@ class LegacyParser(VersionedYAMLParser): version = 'legacy' # the version number this parser designed for def parse(self, cls: Type['BaseDriver'], data: Dict) -> 'BaseDriver': - """Return the Flow YAML parser given the syntax version number - + """ :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type :param data: flow yaml file loaded as python dict + :return: the Flow YAML parser given the syntax version number """ obj = cls(**data.get('with', {})) return obj def dump(self, data: 'BaseDriver') -> Dict: - """Return the dictionary given a versioned flow object - + """ :param data: versioned flow object + :return: dictionary given a versioned flow object """ a = {k: v for k, v in data._init_kwargs_dict.items()} r = {} diff --git a/jina/jaml/parsers/executor/legacy.py b/jina/jaml/parsers/executor/legacy.py index 6c1c50d27f8e3..b1e8f06e9c43e 100644 --- a/jina/jaml/parsers/executor/legacy.py +++ b/jina/jaml/parsers/executor/legacy.py @@ -1,6 +1,6 @@ import os import inspect -from typing import Dict, Any, Type +from typing import Dict, Any, Type, Set from functools import reduce from ..base import VersionedYAMLParser @@ -14,22 +14,23 @@ class LegacyParser(VersionedYAMLParser): @staticmethod def _get_all_arguments(class_): """ - Returns a list of all the arguments of all the classess from which `class_` - inherits - + :param class_: target class from which we want to retrieve arguments + :return: all the arguments of all the classes from which `class_` inherits """ def get_class_arguments(class_): """ - Retrieves a list containing the arguments from `class_` + :param class_: the class to check + :return: a list containing the arguments from `class_` """ signature = inspect.signature(class_.__init__) class_arguments = [p.name for p in signature.parameters.values()] return class_arguments - def accumulate_classes(cls): + def accumulate_classes(cls) -> Set[Type]: """ - Retrieves all classes from which cls inherits from + :param cls: the class to check + :return: all classes from which cls inherits from """ def _accumulate_classes(c, cs): cs.append(c) @@ -74,10 +75,10 @@ def _get_dump_path_from_config(meta_config: Dict): return bin_dump_path def parse(self, cls: Type['BaseExecutor'], data: Dict) -> 'BaseExecutor': - """Return the Flow YAML parser given the syntax version number - + """ :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type :param data: flow yaml file loaded as python dict + :return: the Flow YAML parser given the syntax version number """ _meta_config = get_default_metas() _meta_config.update(data.get('metas', {})) @@ -141,9 +142,9 @@ def parse(self, cls: Type['BaseExecutor'], data: Dict) -> 'BaseExecutor': return obj def dump(self, data: 'BaseExecutor') -> Dict: - """Return the dictionary given a versioned flow object - + """ :param data: versioned executor object + :return: the dictionary given a versioned flow object """ # note: we only save non-default property for the sake of clarity _defaults = get_default_metas() diff --git a/jina/jaml/parsers/flow/legacy.py b/jina/jaml/parsers/flow/legacy.py index a355c97ab4287..4a0f392d5da81 100644 --- a/jina/jaml/parsers/flow/legacy.py +++ b/jina/jaml/parsers/flow/legacy.py @@ -11,10 +11,10 @@ class LegacyParser(VersionedYAMLParser): version = 'legacy' # the version number this parser designed for def parse(self, cls: Type['BaseFlow'], data: Dict) -> 'BaseFlow': - """Return the Flow YAML parser given the syntax version number - + """ :param cls: target class type to parse into, must be a :class:`JAMLCompatible` type :param data: flow yaml file loaded as python dict + :return: the Flow YAML parser given the syntax version number """ p = data.get('with', {}) # type: Dict[str, Any] a = p.pop('args') if 'args' in p else () @@ -34,9 +34,9 @@ def parse(self, cls: Type['BaseFlow'], data: Dict) -> 'BaseFlow': return obj def dump(self, data: 'BaseFlow') -> Dict: - """Return the dictionary given a versioned flow object - + """ :param data: versioned flow object + :return: the dictionary given a versioned flow object """ r = {} if data._version: diff --git a/jina/jaml/parsers/flow/v1.py b/jina/jaml/parsers/flow/v1.py index f365a022eed15..f86f7eabc0853 100644 --- a/jina/jaml/parsers/flow/v1.py +++ b/jina/jaml/parsers/flow/v1.py @@ -9,7 +9,9 @@ def _get_taboo(): - """Get a set of keys that should not be dumped""" + """ + :return: set of keys that should not be dumped + """ return {k.dest for k in set_pod_parser()._actions if k.help == argparse.SUPPRESS} @@ -48,9 +50,10 @@ class V1Parser(VersionedYAMLParser): version = '1' # the version number this parser designed for def parse(self, cls: type, data: Dict) -> 'Flow': - """Return the Flow YAML parser given the syntax version number - + """ + :param cls: the class registered for dumping/loading :param data: flow yaml file loaded as python dict + :return: the Flow YAML parser given the syntax version number """ envs = data.get('env', {}) # type: Dict[str, str] p = data.get('with', {}) # type: Dict[str, Any] @@ -73,9 +76,9 @@ def parse(self, cls: type, data: Dict) -> 'Flow': return obj def dump(self, data: 'Flow') -> Dict: - """Return the dictionary given a versioned flow object - + """ :param data: versioned flow object + :return: the dictionary given a versioned flow object """ r = {} if data._version: