From 31dec0e163b86b331f6fb15e3a23d9ea5925c4a1 Mon Sep 17 00:00:00 2001 From: Tom Harris Date: Fri, 15 Feb 2019 19:26:58 -0500 Subject: [PATCH 1/7] add yaml provider --- src/pubsub/utils/yamltopicdefnprovider.py | 313 ++++++++++++++++++++++ tests/suite/test6_yamlprovider.py | 87 ++++++ tests/suite/yamlprovider_topics.yaml | 18 ++ 3 files changed, 418 insertions(+) create mode 100644 src/pubsub/utils/yamltopicdefnprovider.py create mode 100644 tests/suite/test6_yamlprovider.py create mode 100644 tests/suite/yamlprovider_topics.yaml diff --git a/src/pubsub/utils/yamltopicdefnprovider.py b/src/pubsub/utils/yamltopicdefnprovider.py new file mode 100644 index 0000000..a5bce0e --- /dev/null +++ b/src/pubsub/utils/yamltopicdefnprovider.py @@ -0,0 +1,313 @@ +""" +Contributed by Tom Harris, adapted by Oliver Schoenborn to be +consistent with pubsub API. + +An extension for pubsub (http://pubsub.sourceforge.net) so topic tree +specification can be encoded in YAML format rather than pubsub's default +Python nested class format. + +To use: + + yaml = ''' + topicdefntree: + - description: Test Topics showing hierarchy and topic inheritance + - topics: + - id: parent + description: Parent with a parameter and subtopics + listenerspec: + - id: lastname + description: surname + optional: False + - id: lastname + description: given name + optional: True + topics: + - id: child + description: This is the first child + listenerspec: + - id: nick + description: A nickname + optional: False + ''' + +These topic definitions are loaded through an YamlTopicDefnProvider: + + pub.addTopicDefnProvider( YamlTopicDefnProvider(yaml) ) + +The YamlTopicDefnProvider also accepts a filename instead of yaml string: + + provider = YamlTopicDefnProvider("path/to/yamlfile.yaml", TOPIC_TREE_FROM_FILE) + pub.addTopicDefnProvider( provider ) + +Topics can be exported to a yaml file using the exportTopicTreeSpecYaml function. +This will create a text file for the yaml and return the string representation +of the yaml tree. + +:copyright: Copyright since 2013 by Oliver Schoenborn, all rights reserved. +:license: BSD, see LICENSE_BSD_Simple.txt for details. +""" + +__author__ = 'Tom Harris' +__revision__ = 0 +__date__ = '2019-02-14' + +from collections.abc import Iterable +from typing import Any + +import voluptuous as vol +import yaml + +from ..core.topictreetraverser import ITopicTreeVisitor +from ..core.topicdefnprovider import ( + ITopicDefnProvider, + ArgSpecGiven, + TOPIC_TREE_FROM_STRING, +) + + +__all__ = [ + 'YamlTopicDefnProvider', + 'exportTopicTreeSpecYaml', + 'TOPIC_TREE_FROM_FILE' +] + + +TOPIC_TREE_FROM_FILE = 'file' +TOPIC = 'topic' +TOPICS = 'topics' +TOPIC_DESCRIPTION = 'description' +TOPIC_LISTENER_SPEC = 'listenerspec' +TOPIC_LISTENER_ARG = 'arg' +TOPIC_LISTENR_ARG_OPTIONAL = 'optional' +ALL_TOPICS_NAME = 'ALL_TOPICS' + + +def string(value: Any) -> str: + """Coerce value to string, except for None.""" + if value is None: + raise vol.Invalid('string value is None') + if isinstance(value, (list, dict)): + raise vol.Invalid('value should be a string') + + return str(value) + + +def string_no_space(value: Any) -> str: + """Coerce value to string with no spaces.""" + return string(value).strip().replace(' ', '_') + + +def boolean(value: Any) -> bool: + """Validate and coerce a boolean value.""" + if isinstance(value, str): + value = value.lower() + if value in ('1', 'true', 'yes', 'on', 'enable'): + return True + if value in ('0', 'false', 'no', 'off', 'disable'): + return False + raise vol.Invalid('invalid boolean value {}'.format(value)) + return bool(value) + + +def print_topic(topic_tree: dict, level=0): + if topic_tree: + topic_id = topic_tree.get(TOPIC) + topic_desc = topic_tree.get(TOPIC_DESCRIPTION) + print(' ' * level, topic_id, ': ', topic_desc) + for topic in topic_tree.get(TOPICS, []): + print_topic(topic, level + 1) + + +LISTENER_SPEC_SCHEMA = vol.Schema({ + vol.Required(TOPIC_LISTENER_ARG): string_no_space, + vol.Required(TOPIC_DESCRIPTION): string, + vol.Optional(TOPIC_LISTENR_ARG_OPTIONAL, True): boolean +}) + + +TOPIC_SCHEMA = vol.Schema({ + vol.Required(TOPIC): string_no_space, + vol.Required(TOPIC_DESCRIPTION): string, + vol.Optional(TOPIC_LISTENER_SPEC): [LISTENER_SPEC_SCHEMA], + vol.Optional(TOPICS, []): [vol.Self] +}) + +SCHEMA = vol.Schema({ + vol.Required(string_no_space): vol.Schema({ + vol.Required(TOPIC_DESCRIPTION): string, + vol.Optional(TOPICS, []): [TOPIC_SCHEMA] + }) +}) + + +class YamlTopicDefnProvider(ITopicDefnProvider): + class YamlParserError(RuntimeError): + pass + + class UnrecognizedSourceFormatError(ValueError): + pass + + def __init__(self, yaml_str, format=TOPIC_TREE_FROM_STRING): + self._topics = {} + self._treeDoc = '' + if format == TOPIC_TREE_FROM_FILE: + with open(yaml_str, 'r') as stream: + try: + yaml_topics = SCHEMA(yaml.load(stream)) + except yaml.YAMLError as err: + raise self.YamlParserError( + 'YAML file format is incorrect, or file missing: %s', + err) + except vol.error.MultipleInvalid as err: + raise self.YamlParserError( + 'YAML file content is incorrect: %s', err) + elif format == TOPIC_TREE_FROM_STRING: + try: + yaml_topics = SCHEMA(yaml.load(yaml_str)) + except vol.error.MultipleInvalid as err: + raise self.YamlParserError( + 'YAML string content is incorrect: %s', err) + else: + raise self.UnrecognizedSourceFormatError() + + #for topic_tree in yaml_topics: + for tree_name, _ in yaml_topics.items(): + self._treeDoc = yaml_topics[tree_name][TOPIC_DESCRIPTION] + self._parse_tree(yaml_topics[tree_name]) + + def _parse_tree(self, tree: dict): + self._treeDoc = tree.get(TOPIC_DESCRIPTION) + + for topic in tree.get(TOPICS, []): + self._parse_topic(topic) + + def _parse_topic(self, topic, parents=None, specs=None, reqlist=None): + parents = parents or [] + specs = specs or {} + reqlist = reqlist or [] + + topic_desc = topic.get(TOPIC_DESCRIPTION) + + topic_id = topic.get(TOPIC) + + for spec in topic.get(TOPIC_LISTENER_SPEC, []): + arg = spec.get(TOPIC_LISTENER_ARG) + arg_desc = spec.get(TOPIC_DESCRIPTION) + arg_optional = spec.get(TOPIC_LISTENR_ARG_OPTIONAL) + + if not arg_optional: + reqlist.append(arg) + + specs[arg] = arg_desc + + defn = ArgSpecGiven(specs, tuple(reqlist)) + + parents.append(topic_id) + + self._topics[tuple(parents)] = topic_desc, defn + + for subtopic in topic.get(TOPICS, []): + self._parse_topic(subtopic, parents[:], specs.copy(), reqlist[:]) + + def getDefn(self, topicNameTuple): + return self._topics.get(topicNameTuple, (None, None)) + + def topicNames(self): + return self._topics.keys() # dict_keys iter in 3, list in 2 + + def getTreeDoc(self): + return self._treeDoc + + +class YamlVisitor(ITopicTreeVisitor): + def __init__(self, rootTopic): + self._rootTopic = rootTopic + self.tree = [] + + def _startTraversal(self): + root_dict = {TOPIC_DESCRIPTION: self._rootTopic.getDescription()} + lsnr_spec = self._get_listenerspec(self._rootTopic) + if lsnr_spec: + root_dict[TOPIC_LISTENER_SPEC] = lsnr_spec + subtopics = [] + for subtopic in self._rootTopic.subtopics: + topic_dict = self._topic_to_dict(subtopic) + subtopics.append(topic_dict) + if subtopic: + root_dict[TOPICS] = subtopics + self.tree = {self._rootTopic.getName(): root_dict} + + def _topic_to_dict(self, topicObj): + if topicObj.isAll(): + topic_id = ALL_TOPICS_NAME + else: + topic_id = topicObj.getNodeName() + topic_desc = topicObj.description + subtopics = [] + for subtopic in topicObj.subtopics: + subtopics.append(self._topic_to_dict(subtopic)) + + topic_dict = {TOPIC: topic_id, + TOPIC_DESCRIPTION: topic_desc} + + listener_spec = self._get_listenerspec(topicObj) + if listener_spec: + topic_dict[TOPIC_LISTENER_SPEC] = listener_spec + if subtopics: + topic_dict[TOPICS] = subtopics + return topic_dict + + def _get_listenerspec(self, topicObj): + req_args, opt_args = topicObj.getArgs() + listener_spec = [] + if req_args: + for arg in req_args: + desc = topicObj.getArgDescriptions()[arg] + spec = {TOPIC_LISTENER_ARG: arg, + TOPIC_DESCRIPTION: desc} + listener_spec.append(spec) + if opt_args: + for arg in opt_args: + desc = topicObj.getArgDescriptions()[arg] + spec = {TOPIC_LISTENER_ARG: arg, + TOPIC_DESCRIPTION: desc, + TOPIC_LISTENR_ARG_OPTIONAL: "True"} + listener_spec.append(spec) + return listener_spec + + +def exportTopicTreeSpecYaml(rootTopic=ALL_TOPICS_NAME, filename=None, bak='bak'): + """ + Export the topic tree to a YAML file. + + rootTopic: Topic or str - Optional - Top level topic to export, including + subtopics. If rootTopic is empty ALL_TOPICS is used. + + filename: str - Optional - file name to export to. File extention will be + '.yaml' + + bak: - str - Optional - file extention to use for backing up existing file. + """ + + if rootTopic is None: + from .. import pub + topicMgr = pub.getDefaultTopicMgr() + rootTopic = topicMgr.getRootAllTopics() + elif isinstance(rootTopic, str): + from .. import pub + topicMgr = pub.getDefaultTopicMgr() + rootTopic = topicMgr.getTopic(rootTopic) + + visitor = YamlVisitor(rootTopic) + traverser = pub.TopicTreeTraverser(visitor) + traverser.traverse(rootTopic) + + tree = SCHEMA(visitor.tree) + print(tree) + if filename: + filename = '%s.yaml' % filename + with open(filename, 'w') as fulltree: + yaml.dump(tree, fulltree, default_flow_style=False) + print(yaml.dump(tree, default_flow_style=False)) + + return yaml.dump(tree, default_flow_style=False) diff --git a/tests/suite/test6_yamlprovider.py b/tests/suite/test6_yamlprovider.py new file mode 100644 index 0000000..107cc4b --- /dev/null +++ b/tests/suite/test6_yamlprovider.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +from pubsub import pub +from pubsub.utils.yamltopicdefnprovider import ( + YamlTopicDefnProvider, + TOPIC_TREE_FROM_FILE, + exportTopicTreeSpecYaml + ) +from pubsub.utils.topictreeprinter import printTreeDocs + +topicMgr = pub.getDefaultTopicMgr() + +def test_yaml_from_file(): + pub.clearTopicDefnProviders() + + provider = YamlTopicDefnProvider('yamlprovider_topics.yaml', TOPIC_TREE_FROM_FILE) + assert topicMgr.getTopic('parent', True) is None + assert topicMgr.getTopic('parent.child', True) is None + assert topicMgr.getOrCreateTopic('parent') is not None + assert topicMgr.getOrCreateTopic('parent.child') is not None + +def test_yaml_import(): + pub.clearTopicDefnProviders() + topicMgr.delTopic('parent') + # verify pre: + assert topicMgr.getTopic('parent', True) is None + assert topicMgr.getTopic('parent.child', True) is None + + provider = YamlTopicDefnProvider('yamlprovider_topics.yaml', TOPIC_TREE_FROM_FILE) + pub.addTopicDefnProvider( provider ) + # force instantiation of two topic definitions that were defined in yaml: + pub.sendMessage('parent', lastname='') + pub.sendMessage('parent.child', lastname='', nick='') + + # verify post: + assert topicMgr.getTopic('parent') is not None + assert topicMgr.getTopic('parent.child') is not None + +def test_yaml_string_import(): + str_yaml="""ALL_TOPICS: + description: Root of all topics + topics: + - topic: parent + description: Parent with a parameter and subtopics + listenerspec: + - arg: lastname + description: surname + - arg: name + description: given name + optional: true + topics: + - topic: child + description: This is the first child + listenerspec: + - arg: nick + description: A nickname + """ + + topicMgr.delTopic('parent') + pub.clearTopicDefnProviders() + assert topicMgr.getTopic('parent', True) is None + assert topicMgr.getTopic('parent.child', True) is None + + provider = YamlTopicDefnProvider(str_yaml) + pub.addTopicDefnProvider( provider ) + # to force instantiation of two topic definitions that were defined in yaml, + # this time we just instantiate all of them: + pub.instantiateAllDefinedTopics(provider) + + printTreeDocs() + + assert topicMgr.getTopic('parent') is not None + assert topicMgr.getTopic('parent.child') is not None + +# def test_yaml_topics(): +# # validate that topic specs were properly parsed +# def isValid(topicName, listener): +# topic = topicMgr.getTopic(topicName) +# assert topic.getDescription() is not None +# assert topic.hasMDS() +# return topic.isValid(listener) + +# def hello(lastname, name=None): pass +# def friend(lastname, nick, name=None): pass + +# assert isValid('parent', hello) +# assert isValid('parent.child', friend) diff --git a/tests/suite/yamlprovider_topics.yaml b/tests/suite/yamlprovider_topics.yaml new file mode 100644 index 0000000..6d21402 --- /dev/null +++ b/tests/suite/yamlprovider_topics.yaml @@ -0,0 +1,18 @@ +ALL_TOPICS: + description: Root of all topics + topics: + - topic: parent + description: Parent with a parameter and subtopics + listenerspec: + - arg: lastname + description: surname + - arg: name + description: given name + optional: true + topics: + - topic: child + description: This is the first child + listenerspec: + - arg: nick + description: A nickname + \ No newline at end of file From 776e1ff3b9ee3f0e25fda05a3c6b38694b67e12d Mon Sep 17 00:00:00 2001 From: Tom Harris Date: Fri, 15 Feb 2019 21:14:01 -0500 Subject: [PATCH 2/7] add voluptuous to requirements --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d026496..7d47c14 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,8 @@ def getPubsubVersion(): def getInstallRequires(): import sys if sys.version_info < (3,5): - return ['typing'] - return [] + return ['typing', 'voluptuous'] + return ['voluptuous'] setup( From 11b9fd7b5ae35312d3a846ee52e42d12dc2fd2dd Mon Sep 17 00:00:00 2001 From: Tom Harris Date: Fri, 15 Feb 2019 21:16:34 -0500 Subject: [PATCH 3/7] add yaml module to requirements --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7d47c14..77272da 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,8 @@ def getPubsubVersion(): def getInstallRequires(): import sys if sys.version_info < (3,5): - return ['typing', 'voluptuous'] - return ['voluptuous'] + return ['typing', 'voluptuous', 'yaml'] + return ['voluptuous', 'yaml'] setup( From 1e39b30016e5d40389341059fd31c5bcc5a8ebf4 Mon Sep 17 00:00:00 2001 From: Tom Harris Date: Fri, 15 Feb 2019 21:20:17 -0500 Subject: [PATCH 4/7] Add pyyaml to requirements --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 77272da..d2c8edf 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,8 @@ def getPubsubVersion(): def getInstallRequires(): import sys if sys.version_info < (3,5): - return ['typing', 'voluptuous', 'yaml'] - return ['voluptuous', 'yaml'] + return ['typing', 'voluptuous', 'pyyaml'] + return ['voluptuous', 'pyyaml'] setup( From 2dee5edbc5bf932a970d498050f0c227662e8b7d Mon Sep 17 00:00:00 2001 From: Tom Harris Date: Fri, 15 Feb 2019 21:29:36 -0500 Subject: [PATCH 5/7] add test_yaml_topics test --- tests/suite/test6_yamlprovider.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/suite/test6_yamlprovider.py b/tests/suite/test6_yamlprovider.py index 107cc4b..d0e03b8 100644 --- a/tests/suite/test6_yamlprovider.py +++ b/tests/suite/test6_yamlprovider.py @@ -72,16 +72,16 @@ def test_yaml_string_import(): assert topicMgr.getTopic('parent') is not None assert topicMgr.getTopic('parent.child') is not None -# def test_yaml_topics(): -# # validate that topic specs were properly parsed -# def isValid(topicName, listener): -# topic = topicMgr.getTopic(topicName) -# assert topic.getDescription() is not None -# assert topic.hasMDS() -# return topic.isValid(listener) +def test_yaml_topics(): + # validate that topic specs were properly parsed + def isValid(topicName, listener): + topic = topicMgr.getTopic(topicName) + assert topic.getDescription() is not None + assert topic.hasMDS() + return topic.isValid(listener) -# def hello(lastname, name=None): pass -# def friend(lastname, nick, name=None): pass + def hello(lastname, name=None): pass + def friend(lastname, nick, name=None): pass -# assert isValid('parent', hello) -# assert isValid('parent.child', friend) + assert isValid('parent', hello) + assert isValid('parent.child', friend) From d0defae618fa0eba806577e8b11d91e32752829b Mon Sep 17 00:00:00 2001 From: Tom Harris Date: Fri, 15 Feb 2019 21:42:07 -0500 Subject: [PATCH 6/7] Debug test_yaml_from_file --- tests/suite/test6_yamlprovider.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/suite/test6_yamlprovider.py b/tests/suite/test6_yamlprovider.py index d0e03b8..ed82ecc 100644 --- a/tests/suite/test6_yamlprovider.py +++ b/tests/suite/test6_yamlprovider.py @@ -14,6 +14,8 @@ def test_yaml_from_file(): pub.clearTopicDefnProviders() provider = YamlTopicDefnProvider('yamlprovider_topics.yaml', TOPIC_TREE_FROM_FILE) + printTreeDocs() + print(topicMgr.getTopic('parent', True)) assert topicMgr.getTopic('parent', True) is None assert topicMgr.getTopic('parent.child', True) is None assert topicMgr.getOrCreateTopic('parent') is not None From 357f719e9f571886284b1d306317b70f2c42df75 Mon Sep 17 00:00:00 2001 From: Tom Harris Date: Fri, 15 Feb 2019 22:04:52 -0500 Subject: [PATCH 7/7] Clean up topics before tests --- tests/suite/test6_yamlprovider.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/suite/test6_yamlprovider.py b/tests/suite/test6_yamlprovider.py index ed82ecc..59d6d83 100644 --- a/tests/suite/test6_yamlprovider.py +++ b/tests/suite/test6_yamlprovider.py @@ -10,7 +10,15 @@ topicMgr = pub.getDefaultTopicMgr() +def remove_all_topics(): + names = [] + for topic in topicMgr.getRootAllTopics().subtopics: + names.append(topic.getName()) + for name in names: + topicMgr.delTopic(name) + def test_yaml_from_file(): + remove_all_topics() pub.clearTopicDefnProviders() provider = YamlTopicDefnProvider('yamlprovider_topics.yaml', TOPIC_TREE_FROM_FILE) @@ -22,6 +30,7 @@ def test_yaml_from_file(): assert topicMgr.getOrCreateTopic('parent.child') is not None def test_yaml_import(): + remove_all_topics() pub.clearTopicDefnProviders() topicMgr.delTopic('parent') # verify pre: @@ -39,6 +48,7 @@ def test_yaml_import(): assert topicMgr.getTopic('parent.child') is not None def test_yaml_string_import(): + remove_all_topics() str_yaml="""ALL_TOPICS: description: Root of all topics topics: