From 767f50d14cced0cd20820ff21522efbe6840f1be Mon Sep 17 00:00:00 2001 From: naman108 Date: Wed, 16 Nov 2022 23:30:22 -0400 Subject: [PATCH 01/11] added registration and manifest support to digitalpy --- digitalpy/component/impl/default_facade.py | 10 ++ .../config/impl/inifile_configuration.py | 18 ++- .../registration/registration_handler.py | 135 ++++++++++++++++++ setup.py | 2 +- 4 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 digitalpy/registration/registration_handler.py diff --git a/digitalpy/component/impl/default_facade.py b/digitalpy/component/impl/default_facade.py index 2de99169..3280d4ff 100644 --- a/digitalpy/component/impl/default_facade.py +++ b/digitalpy/component/impl/default_facade.py @@ -28,6 +28,7 @@ def __init__( configuration=None, configuration_path_template=None, tracing_provider_instance=None, + manifest_path=None, **kwargs, ): super().__init__( @@ -54,6 +55,11 @@ def __init__( component_name ) + # load the manifest file as a configuration + if manifest_path is not None: + self.manifest = InifileConfiguration("") + self.manifest.add_configuration(manifest_path) + # define the logging self.log_manager = LogManager() DefaultFileLogger.set_base_logging_path(log_file_path) @@ -106,6 +112,10 @@ def register(self, config: InifileConfiguration): ) self._register_type_mapping() + def get_manifest(self): + """returns the current manifest configuration""" + return self.manifest + def _register_type_mapping(self): """any component may or may not have a type mapping defined, if it does then it should be registered""" diff --git a/digitalpy/config/impl/inifile_configuration.py b/digitalpy/config/impl/inifile_configuration.py index 4eeae5b0..9c96bed3 100644 --- a/digitalpy/config/impl/inifile_configuration.py +++ b/digitalpy/config/impl/inifile_configuration.py @@ -159,7 +159,19 @@ def get_key(self, value, section): raise Exception(f"Value {value} not found in section {section}") return map[value] - def get_section(self, section, include_meta=False): + def get_section(self, section: str, include_meta=False) -> dict: + """get a section from the configuration as a dictionary + + Args: + section (str): the name of the section to be retrieved + include_meta (bool, optional): whether or not to include values not prefixed by '__'. Defaults to False. + + Raises: + ValueError: raised if the section is not found + + Returns: + dict: a dictionary representing the contents of the section + """ lookup_entry = self._lookup(section) if lookup_entry is None: raise ValueError(f"section {section} not found") @@ -187,7 +199,7 @@ def build_lookup_table(self) -> Any: lookup_section_key = section.lower() + ":" self.lookup_table[lookup_section_key] = [section] for key, _ in entry.items(): - lookup_key = lookup_section_key.lower()+key + lookup_key = lookup_section_key.lower() + key self.lookup_table[lookup_key] = {section: key} def check_file_date(self, file_list: Any, reference_file: Any) -> Any: @@ -245,7 +257,7 @@ def lookup(self, section: Any, key: Any = "") -> Any: @return Array with section as first entry and key as second or None if not found """ - lookup_key = section.lower()+":"+key.lower() + lookup_key = section.lower() + ":" + key.lower() if lookup_key in self.lookup_table: return self.lookup_table[lookup_key] else: diff --git a/digitalpy/registration/registration_handler.py b/digitalpy/registration/registration_handler.py new file mode 100644 index 00000000..78c58a9f --- /dev/null +++ b/digitalpy/registration/registration_handler.py @@ -0,0 +1,135 @@ +import importlib +import os +from pathlib import PurePath +from typing import List + +import pkg_resources +from digitalpy.component.impl.default_facade import DefaultFacade + +from digitalpy.config.configuration import Configuration +from digitalpy.config.impl.inifile_configuration import InifileConfiguration + +MANIFEST = "manifest" +DIGITALPY = "digitalpy" +REQUIRED_ALFA_VERSION = "requiredAlfaVersion" +NAME = "name" +VERSION = "version" + + +class RegistrationHandler: + """this class is used to manage component registration""" + + registered_components = {} + + @staticmethod + def discover_components(component_folder_path: PurePath) -> List[str]: + """this method is used to discover all available components + + Args: + component_folder_path (str): the path in which to search for components. the searchable folder should be in the following format:\n + component_folder_path \n + |-- some_component \n + | `-- some_component_facade.py\n + `-- another_component\n + `-- another_component_facade.py\n + Returns: + List[str]: a list of available components in the given path + """ + potential_components = os.scandir(component_folder_path) + components = [] + for potential_component in potential_components: + facade_path = PurePath( + potential_component.path, potential_component.name + "_facade.py" + ) + if os.path.exists(facade_path): + components.append(PurePath(potential_component.path)) + return components + + @staticmethod + def register_component( + component_path: PurePath, import_root: str, config: InifileConfiguration + ) -> bool: + """this method is used to register a given component + + Args: + component_path (PurePath): the path to the directory of the component to be registered. + import_root (str): the import root from which to import the components facade. + config (InifileConfiguration): the main configuration through which the components actions should be exposed. + + Returns: + bool: whether or not the component was registered successfully + """ + try: + facade_path = PurePath(component_path, component_path.name + "_facade.py") + if os.path.exists(str(facade_path)): + component_name = component_path.name.replace("_component", "") + + component_facade = getattr( + importlib.import_module( + f"{import_root}.{component_path.name}.{component_name}_facade" + ), + f"{''.join([name.capitalize() if name[0].isupper()==False else name for name in component_name.split('_')])}", + ) + facade_instance: DefaultFacade = component_facade( + None, None, None, None + ) + + if RegistrationHandler.validate_manifest( + facade_instance.get_manifest(), component_name + ): + facade_instance.register(config) + else: + return False + else: + return False + return True + except Exception as e: + # must use a print because logger may not be available + print(f"failed to register component: {component_path}, with error: {e}") + return False + + @staticmethod + def validate_manifest(manifest: Configuration, component_name: str) -> bool: + """validate that the component is compatible with the current digitalpy version + + Args: + manifest (Configuration): the manifest of a component to be validated for this digitalpy installation + component_name (str): the name of the component to be validated + + Raises: + ValueError: raised if the manifest section is missing from the manifest configuration + + Returns: + bool: whether the component is compatible with the current digitalpy installation + """ + # retrieve the current digitalpy version based on the setup.py + digitalpy_version = pkg_resources.require(DIGITALPY)[0].version + + try: + # get the manifest section from the configuration + section = manifest.get_section(component_name + MANIFEST, include_meta=True) + except ValueError: + + raise ValueError( + f"manifest section missing, requires name {component_name+MANIFEST} please add the \ + following section to the manifest [{component_name+MANIFEST}], for more information on component\ + manifests please refer to the digitalpy documentation" + ) + + # validate the component name matches the name specified in the manifest + if component_name != section[NAME]: + return False + + # validate the manifest required version against the current version + if section[REQUIRED_ALFA_VERSION] != digitalpy_version: + return False + + # dont approve the manifest if the component has already been registered + if ( + component_name in RegistrationHandler.registered_components + and section[VERSION] + != RegistrationHandler.registered_components[component_name][VERSION] + ): + return False + + return True diff --git a/setup.py b/setup.py index ec366067..cc8c2dcb 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='digitalpy', - version='0.2.4.3', + version='0.2.5', description="A python implementation of the aphrodite's specification, heavily based on WCMF", author='Natha Paquette', author_email='natha.paquette@gmail.com', From 334fce9d8e47a6c33071c8005409c55b2b8c8c21 Mon Sep 17 00:00:00 2001 From: naman108 Date: Thu, 17 Nov 2022 09:30:11 -0400 Subject: [PATCH 02/11] added support for manifest ID validation --- .../registration/registration_handler.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/digitalpy/registration/registration_handler.py b/digitalpy/registration/registration_handler.py index 78c58a9f..8f3a0752 100644 --- a/digitalpy/registration/registration_handler.py +++ b/digitalpy/registration/registration_handler.py @@ -2,10 +2,9 @@ import os from pathlib import PurePath from typing import List - import pkg_resources -from digitalpy.component.impl.default_facade import DefaultFacade +from digitalpy.component.impl.default_facade import DefaultFacade from digitalpy.config.configuration import Configuration from digitalpy.config.impl.inifile_configuration import InifileConfiguration @@ -14,6 +13,7 @@ REQUIRED_ALFA_VERSION = "requiredAlfaVersion" NAME = "name" VERSION = "version" +ID = "id" class RegistrationHandler: @@ -82,14 +82,21 @@ def register_component( return False else: return False + RegistrationHandler.save_component(facade_instance.get_manifest(), component_name) return True except Exception as e: # must use a print because logger may not be available print(f"failed to register component: {component_path}, with error: {e}") return False + @staticmethod + def save_component(manifest: Configuration, component_name: str): + section = manifest.get_section(component_name + MANIFEST, include_meta=True) + RegistrationHandler.registered_components[section[NAME]] = section + @staticmethod def validate_manifest(manifest: Configuration, component_name: str) -> bool: + #TODO: determine better way to inform the caller that the manifest is invalid """validate that the component is compatible with the current digitalpy version Args: @@ -132,4 +139,12 @@ def validate_manifest(manifest: Configuration, component_name: str) -> bool: ): return False + # dont approve the manifest if a component with the same name but a different ID already exists + if ( + component_name in RegistrationHandler.registered_components + and RegistrationHandler.registered_components[component_name][ID] + != section[ID] + ): + return False + return True From f24f61081707d93a7caf58a24c1e930f4a5e2c6b Mon Sep 17 00:00:00 2001 From: naman108 Date: Thu, 17 Nov 2022 11:17:42 -0400 Subject: [PATCH 03/11] updated id constant value --- digitalpy/registration/registration_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/digitalpy/registration/registration_handler.py b/digitalpy/registration/registration_handler.py index 8f3a0752..47c0a9e0 100644 --- a/digitalpy/registration/registration_handler.py +++ b/digitalpy/registration/registration_handler.py @@ -13,7 +13,7 @@ REQUIRED_ALFA_VERSION = "requiredAlfaVersion" NAME = "name" VERSION = "version" -ID = "id" +ID = "UUID" class RegistrationHandler: From eca24286dd787122a0241607e1965ad414dc4b42 Mon Sep 17 00:00:00 2001 From: naman108 Date: Thu, 17 Nov 2022 23:20:02 -0400 Subject: [PATCH 04/11] fixed bug which broke registration in components without type mappings --- digitalpy/component/impl/default_facade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/digitalpy/component/impl/default_facade.py b/digitalpy/component/impl/default_facade.py index 3280d4ff..cbcc1019 100644 --- a/digitalpy/component/impl/default_facade.py +++ b/digitalpy/component/impl/default_facade.py @@ -119,7 +119,7 @@ def get_manifest(self): def _register_type_mapping(self): """any component may or may not have a type mapping defined, if it does then it should be registered""" - if self.type_mapping is not None: + if self.type_mapping: request = ObjectFactory.get_new_instance("request") request.set_action("RegisterMachineToHumanMapping") request.set_value("machine_to_human_mapping", self.type_mapping) From 4f15e31f1411a827eaf7e5f0c34ab4f26216997f Mon Sep 17 00:00:00 2001 From: naman108 Date: Thu, 17 Nov 2022 23:20:31 -0400 Subject: [PATCH 05/11] fixed bug which broke table building --- digitalpy/config/impl/inifile_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/digitalpy/config/impl/inifile_configuration.py b/digitalpy/config/impl/inifile_configuration.py index 9c96bed3..fb5de34d 100644 --- a/digitalpy/config/impl/inifile_configuration.py +++ b/digitalpy/config/impl/inifile_configuration.py @@ -327,7 +327,7 @@ def set_value( else: final_key_name = key self.config_array[final_section_name][final_key_name] = value - self.build_lookup_table() + self._build_lookup_table() def unserialize(self, parsed_files: Any) -> Any: """Retrieve parsed ini data from the file system and update the current instance. From 45bf8533cddc008c95d92ae38d6e30018601304e Mon Sep 17 00:00:00 2001 From: naman108 Date: Thu, 17 Nov 2022 23:20:59 -0400 Subject: [PATCH 06/11] added return in the case of an error so that the client isn't left blocking --- digitalpy/routing/impl/default_routing_worker.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/digitalpy/routing/impl/default_routing_worker.py b/digitalpy/routing/impl/default_routing_worker.py index a93a8bcb..073b420c 100644 --- a/digitalpy/routing/impl/default_routing_worker.py +++ b/digitalpy/routing/impl/default_routing_worker.py @@ -63,6 +63,7 @@ def start(self): self.initialize_tracing() while True: try: + print("listening") message = self.sock.recv_multipart() topic = message[0] @@ -196,4 +197,12 @@ def start(self): ] ) except Exception as e: - pass + try: + self.sock.send_multipart( + [ + response_topic.encode("utf-8"), + f"error thrown {str(e)}".encode("utf-8"), + ] + ) + except Exception as e: + print(str(e)) From 8b7205ad775cc873666807ab9e8715f402621399 Mon Sep 17 00:00:00 2001 From: naman108 Date: Thu, 17 Nov 2022 23:23:18 -0400 Subject: [PATCH 07/11] added optional pytest requirement --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cc8c2dcb..ae3bdd5e 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,9 @@ install_requires=[ "rule-engine", "pyzmq", - ], + ], + extras_require={ + "DEV": ["pytest"], + }, packages=find_packages(include=["digitalpy", "digitalpy.*"]) ) From 661d9a53adf9913f40ac7625b0e5c17a7177ec1e Mon Sep 17 00:00:00 2001 From: naman108 Date: Fri, 18 Nov 2022 12:24:01 -0400 Subject: [PATCH 08/11] added tracing support to action mapper --- digitalpy/component/impl/default_facade.py | 3 ++- digitalpy/core/object_factory.py | 1 + .../routing/impl/default_action_mapper.py | 21 ++++++++++++++++++- setup.py | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/digitalpy/component/impl/default_facade.py b/digitalpy/component/impl/default_facade.py index cbcc1019..bcd92ba3 100644 --- a/digitalpy/component/impl/default_facade.py +++ b/digitalpy/component/impl/default_facade.py @@ -107,7 +107,8 @@ def register(self, config: InifileConfiguration): ObjectFactory.register_instance( f"{self.component_name.lower()}actionmapper", self.base.ActionMapper( - ObjectFactory.get_instance("event_manager"), internal_config + ObjectFactory.get_instance("event_manager"), + internal_config, ), ) self._register_type_mapping() diff --git a/digitalpy/core/object_factory.py b/digitalpy/core/object_factory.py index 3f998e7b..d644a1d5 100644 --- a/digitalpy/core/object_factory.py +++ b/digitalpy/core/object_factory.py @@ -48,6 +48,7 @@ def add_interfaces(self, interfaces): @staticmethod def clear(): + """clear the configured factory instance""" if ObjectFactory.__factory != None: ObjectFactory.__factory.clear() ObjectFactory.__factory = None diff --git a/digitalpy/routing/impl/default_action_mapper.py b/digitalpy/routing/impl/default_action_mapper.py index d372005f..005c7d4f 100644 --- a/digitalpy/routing/impl/default_action_mapper.py +++ b/digitalpy/routing/impl/default_action_mapper.py @@ -19,15 +19,34 @@ class DefaultActionMapper(ActionMapper): # @param formatter # @param configuration # - def __init__(self, event_manager: EventManager, configuration: Configuration): + def __init__( + self, + event_manager: EventManager, + configuration: Configuration, + ): self.eventManager = event_manager self.configuration = configuration self.is_finished = False + self.tracing_provider = None # # @see ActionMapper.processAction() # + def initialize_tracing(self): + try: + self.tracing_provider = ObjectFactory.get_instance("tracingprovider") + ObjectFactory.register_instance( + "tracingproviderinstance", self.tracing_provider + ) + except Exception as e: + pass + def process_action(self, request: Request, response: Response): + + # this is added for the sake of the latter use of multiprocessing + if not self.tracing_provider: + self.initialize_tracing() + self.eventManager.dispatch( ApplicationEvent.NAME, ApplicationEvent(ApplicationEvent.BEFORE_ROUTE_ACTION, request), diff --git a/setup.py b/setup.py index ae3bdd5e..1179cdf6 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='digitalpy', - version='0.2.5', + version='0.2.5.1', description="A python implementation of the aphrodite's specification, heavily based on WCMF", author='Natha Paquette', author_email='natha.paquette@gmail.com', From 366da629db7340ed186ab9b91f3e72c8dee53bfd Mon Sep 17 00:00:00 2001 From: naman108 Date: Fri, 18 Nov 2022 12:40:02 -0400 Subject: [PATCH 09/11] added missing init file --- digitalpy/registration/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 digitalpy/registration/__init__.py diff --git a/digitalpy/registration/__init__.py b/digitalpy/registration/__init__.py new file mode 100644 index 00000000..e69de29b From 7abc5d2a52d984efa1640c25877d95efcb13cbae Mon Sep 17 00:00:00 2001 From: naman108 Date: Fri, 18 Nov 2022 12:41:13 -0400 Subject: [PATCH 10/11] updated version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1179cdf6..6718eb47 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='digitalpy', - version='0.2.5.1', + version='0.2.5.2', description="A python implementation of the aphrodite's specification, heavily based on WCMF", author='Natha Paquette', author_email='natha.paquette@gmail.com', From 9816dc0e70cf7b90fc283c6a01632869f2c9138e Mon Sep 17 00:00:00 2001 From: naman108 Date: Fri, 18 Nov 2022 15:41:35 -0400 Subject: [PATCH 11/11] added support for null tracer and added support for manifest specifying min version --- digitalpy/component/impl/default_facade.py | 4 +++- digitalpy/registration/registration_handler.py | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/digitalpy/component/impl/default_facade.py b/digitalpy/component/impl/default_facade.py index bcd92ba3..b995a110 100644 --- a/digitalpy/component/impl/default_facade.py +++ b/digitalpy/component/impl/default_facade.py @@ -50,11 +50,13 @@ def __init__( self.component_name = self.__class__.__name__ # get a tacer from the tracer provider + if tracing_provider_instance is not None: self.tracer: Tracer = tracing_provider_instance.create_tracer( component_name ) - + else: + self.tracer = None # load the manifest file as a configuration if manifest_path is not None: self.manifest = InifileConfiguration("") diff --git a/digitalpy/registration/registration_handler.py b/digitalpy/registration/registration_handler.py index 47c0a9e0..50e49d9b 100644 --- a/digitalpy/registration/registration_handler.py +++ b/digitalpy/registration/registration_handler.py @@ -14,6 +14,7 @@ NAME = "name" VERSION = "version" ID = "UUID" +VERSION_DELIMITER = "." class RegistrationHandler: @@ -127,7 +128,14 @@ def validate_manifest(manifest: Configuration, component_name: str) -> bool: if component_name != section[NAME]: return False - # validate the manifest required version against the current version + # iterate the delimited version number and compare it to the digitalpy version + for i in range(len(section[REQUIRED_ALFA_VERSION].split(VERSION_DELIMITER))): + #check if the version matches + digitalpy_version_number = digitalpy_version.split(VERSION_DELIMITER)[i] if len(digitalpy_version.split(VERSION_DELIMITER))>i else 0 + if int(digitalpy_version_number)>=int(section[REQUIRED_ALFA_VERSION].split(VERSION_DELIMITER)[i]): + continue + else: + return False if section[REQUIRED_ALFA_VERSION] != digitalpy_version: return False