From 235bb92bf833aed534f4ee56a6b0862dca667643 Mon Sep 17 00:00:00 2001 From: SRBuilds Date: Fri, 1 Nov 2024 12:32:52 -0700 Subject: [PATCH] 24.10.1 released --- docs/source/conf.py | 13 +- docs/source/ehs.rst | 5 +- docs/source/esm.rst | 59 +++++ docs/source/features/22.10.rst | 16 +- docs/source/features/23.10.rst | 8 + docs/source/features/24.07.rst | 8 + docs/source/features/24.10.rst | 21 ++ docs/source/index.rst | 9 +- docs/source/introduction.rst | 25 +- docs/source/pysros.rst | 8 + examples/convert_example.py | 2 +- examples/show_router_bgp_asn.py | 5 +- pysros/__init__.py | 2 + pysros/errors.py | 10 +- pysros/exceptions.py | 4 + pysros/identifier.py | 26 +- pysros/management.py | 267 +++++++++++++++---- pysros/model.py | 95 ++++++- pysros/model_builder.py | 268 +++++++++++-------- pysros/model_path.py | 7 +- pysros/model_walker.py | 168 +++++++++--- pysros/path_utils.py | 448 ++++++++++++++++++++++++++++++++ pysros/request_data.py | 237 ++++++++++------- pysros/wrappers.py | 38 ++- pysros/yang_type.py | 94 ++++++- setup.py | 3 +- 26 files changed, 1498 insertions(+), 348 deletions(-) create mode 100644 docs/source/esm.rst create mode 100644 docs/source/features/24.10.rst create mode 100644 pysros/path_utils.py diff --git a/docs/source/conf.py b/docs/source/conf.py index b9f2349..2e9ba1d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,7 +1,7 @@ import os import sys sys.path.insert(0, os.path.relpath('../')) - +from pysros import __version__ # -- Project information ----------------------------------------------------- @@ -10,9 +10,14 @@ author = 'Nokia' # The full version, including alpha/beta/rc tags -version = '24.7.1' -release = '24.7.1' - +version = __version__ +release = __version__ + +rst_epilog = """ +.. |pySROSProjectVersion| replace:: {versionnum} +""".format( +versionnum = version, +) # -- General configuration --------------------------------------------------- diff --git a/docs/source/ehs.rst b/docs/source/ehs.rst index 123c3a4..45a3fdd 100644 --- a/docs/source/ehs.rst +++ b/docs/source/ehs.rst @@ -138,14 +138,13 @@ the event handling system (EHS). .. Reviewed by PLM 20220118 .. Reviewed by TechComms 20220124 -.. class:: EventParams +.. class:: pysros.ehs.EventParams The additional parameters of the specific :py:class:`pysros.ehs.Event`. This class is *read-only*. Specific additional parameters may be accessed using standard Python subscript syntax. - .. Reviewed by PLM 20220118 - .. Reviewed by TechComms 20220124 + .. Reviewed by PLM 20240828 .. py:method:: keys diff --git a/docs/source/esm.rst b/docs/source/esm.rst new file mode 100644 index 0000000..b25a3a5 --- /dev/null +++ b/docs/source/esm.rst @@ -0,0 +1,59 @@ + +The :mod:`pysros.esm` module provides functionality obtain data from the +specific event that triggered the execution of a Python application from +the SR OS subscriber management system. + +.. Reviewed by PLM 20240828 + +.. note:: This module is available when executing on SR OS only. On a remote + machine, subscriber management functionality is not supported. + +.. Reviewed by PLM 20240828 + +.. py:function:: pysros.esm.get_event + + The subscriber management event that triggered the execution of the Python application. + + :return: The Event object or None. + :rtype: :py:class:`pysros.esm.Event` or ``None`` + + .. Reviewed by PLM 20240828 + +.. class:: pysros.esm.Event + + The ESM :py:class:`pysros.esm.Event` Class for the event that triggered the execution of the + Python application. + + .. py:attribute:: eventparameters + + The additional parameters specific to the event that caused the + Python application to execute. + + :type: :py:class:`pysros.ehs.EventParams` + + .. Reviewed by PLM 20240828 + +.. class:: pysros.esm.EventParams + + The additional parameters of the specific :py:class:`pysros.esm.Event`. + This class is *read-only*. Specific additional parameters may be + accessed using standard Python subscript syntax. + + .. Reviewed by PLM 20240828 + + .. py:method:: keys + + Obtain the additional parameters names. + + :return: Additional parameters names for the Event. + :rtype: tuple(str) + + .. Reviewed by PLM 20240828 + + .. describe:: params[key] + + Return the value of the parameter *key*. If the parameter does not exist, + a :exc:`KeyError` is raised. + + .. Reviewed by PLM 20240828 + diff --git a/docs/source/features/22.10.rst b/docs/source/features/22.10.rst index 14ae3d4..bddf8e1 100644 --- a/docs/source/features/22.10.rst +++ b/docs/source/features/22.10.rst @@ -1,6 +1,18 @@ Release 22.10 ************* +22.10.9 +####### + +* No additional features + +.. Reviewed by PLM 20240627 + +22.10.8 +####### + +* No additional features + 22.10.7 ####### @@ -44,7 +56,7 @@ Release 22.10 ####### * :py:meth:`pysros.management.Connection.convert` method adds support for the - conversion of the input/output data of YANG modeled operations (actions). + conversion of the input/output data of YANG-modeled operations (actions). @@ -53,7 +65,7 @@ Release 22.10 * :py:meth:`pysros.management.Datastore.compare` method provides the ability to compare the uncommitted candidate configuration with the baseline configuration prior to a commit. -* :py:meth:`pysros.management.Connection.action` method provides the ability to execute a YANG modeled +* :py:meth:`pysros.management.Connection.action` method provides the ability to execute a YANG-modeled action (operation) on SR OS using modeled, structured data as both input and output. * Updates to the :py:class:`pysros.wrappers.Schema` and :py:class:`pysros.wrappers.SchemaType` to include additional YANG schema information, diff --git a/docs/source/features/23.10.rst b/docs/source/features/23.10.rst index 8a5ad98..00843f2 100644 --- a/docs/source/features/23.10.rst +++ b/docs/source/features/23.10.rst @@ -1,6 +1,14 @@ Release 23.10 ************* +23.10.6 +####### + +* No additional features + +.. Reviewed by PLM 20240718 +.. Reviewed by TechComms 20240718 + 23.10.5 ####### diff --git a/docs/source/features/24.07.rst b/docs/source/features/24.07.rst index cba3789..52b05cd 100644 --- a/docs/source/features/24.07.rst +++ b/docs/source/features/24.07.rst @@ -1,6 +1,14 @@ Release 24.7 ************ +24.7.2 +###### + +* No additional features + +.. Reviewed by PLM 20240814 + + 24.7.1 ###### diff --git a/docs/source/features/24.10.rst b/docs/source/features/24.10.rst new file mode 100644 index 0000000..dfd1bd4 --- /dev/null +++ b/docs/source/features/24.10.rst @@ -0,0 +1,21 @@ +Release 24.10 +************* + +24.10.1 +####### + +* Provides the :py:meth:`.Connection.list_paths` method to describe supported JSON instance + paths for the given :py:class:`.Connection` object. Returns an iterator that can be used + to analyze or print the supported paths. +* Provides :py:mod:`pysros` support from remote servers for alternative operating systems + using standards-based NETCONF implementations, specifically, SR Linux. +* Introduces the :py:mod:`pysros.esm` module to provide integration with Enhanced Subscriber + Management (ESM) of SR OS. This module is available on SR OS only. It is not supported + on a remote machine. + + * Provides the :py:meth:`pysros.esm.get_event` method to obtain the data provided from the ESM + system that called the Python application. + +.. Reviewed by PLM 20240828 +.. Reviewed by TechComms 20240927 + diff --git a/docs/source/index.rst b/docs/source/index.rst index 44992b2..37933e3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -17,12 +17,11 @@ documentation will be updated accordingly. .. list-table:: :header-rows: 0 - * - pySROS release: 24.7.1 - * - Document Number: 3HE 20087 AAAD TQZZA - -.. Reviewed by PLM 20240506 -.. Reviewed by TechComms 20240529 + * - pySROS release: |pySROSProjectVersion| + * - Document Number: 3HE 20087 AAAF TQZZA +.. Reviewed by PLM 20240926 +.. Reviewed by TechComms 20240927 .. toctree:: diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index d007ee4..73d27a8 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -51,8 +51,13 @@ To use the pySROS libraries, the following pre-requisites must be met: All the required software is included and installed automatically on the SR OS node, including the Python 3 interpreter and all supported Python libraries. -.. Reviewed by PLM 20210902 -.. Reviewed by TechComms 20210902 +The pySROS libraries have been tested to work with other Network Operating Systems (NOS) using +standards-based NETCONF implementations, specifically, SR Linux. When using alternative Network +Operating Systems, the use of the :py:meth:`pysros.management.Datastore.lock` and +:py:meth:`pysros.management.Datastore.unlock` methods is strongly encouraged. + +.. Reviewed by PLM 20240926 +.. Reviewed by TechComms 20240927 YANG modeling @@ -126,6 +131,10 @@ See the following for examples: /openconfig-interfaces:interfaces/interface[name="1/1/c2/1"]/subinterfaces/subinterface[index=0]/openconfig-if-ip:ipv4/addresses +Once a connection to the pySROS interface on the node has been established (see :ref:`making-a-connection`) the +:py:meth:`.Connection.list_paths` method can used to obtain a list of the supported YANG-modeled paths for that specific +router in the JSON instance path format. + .. Reviewed by PLM 20220901 .. Reviewed by TechComms 20221012 @@ -555,6 +564,9 @@ See the :py:class:`pysros.wrappers.Annotations` class for more details. Getting Started ############### + +.. _making-a-connection: + Making a connection ******************* @@ -601,6 +613,7 @@ Example: .. Reviewed by TechComms 20221012 + Obtaining data ************** @@ -724,12 +737,12 @@ Performing operations An operation refers to the execution of an activity on the SR OS node that is not that of obtaining data or configuring the device. The method of performing operations on the SR OS -node through the pySROS libraries is using YANG modeled actions. +node through the pySROS libraries is using YANG-modeled actions. -This approach allows for YANG modeled and structured data to be used on both input and +This approach allows for YANG-modeled and structured data to be used on both input and output. Both input and output are represented as pySROS data structures. -To execute a YANG modeled operation, the :py:meth:`pysros.management.Connection.action` +To execute a YANG-modeled operation, the :py:meth:`pysros.management.Connection.action` method should be used. The :py:meth:`pysros.management.Connection.action` method uses the YANG schema obtained as @@ -779,7 +792,7 @@ If the ``do-something`` action was called with the input variables ``myinput-str look like this: .. code-block:: python - :caption: Example calling a YANG modeled action (operation) + :caption: Example calling a YANG-modeled action (operation) :name: calling-yang-action-example >>> from pysros.management import connect diff --git a/docs/source/pysros.rst b/docs/source/pysros.rst index 8195d5d..b3f5fab 100644 --- a/docs/source/pysros.rst +++ b/docs/source/pysros.rst @@ -57,6 +57,14 @@ .. include:: ehs.rst +:mod:`pysros.esm` - Functions for the enhanced subscriber management system (ESM) +--------------------------------------------------------------------------------- + +.. module:: pysros.esm + :synopsis: Functions for the SR OS enhanced subscriber management system (ESM) + +.. include:: esm.rst + :mod:`pysros.syslog` - Functions for syslog event handling ---------------------------------------------------------- diff --git a/examples/convert_example.py b/examples/convert_example.py index a136bfd..b698307 100755 --- a/examples/convert_example.py +++ b/examples/convert_example.py @@ -195,7 +195,7 @@ def converting(input_format, output_format, data, connection_object): """ print("-" * 79) print("Converting", input_format, "to", output_format, "\n") - print("The path used as the YANG modeled root for the data is:") + print("The path used as the YANG-modeled root for the data is:") print(data.path, "\n") print("The payload is:") print(data.payload, "\n") diff --git a/examples/show_router_bgp_asn.py b/examples/show_router_bgp_asn.py index 0506953..54e2355 100755 --- a/examples/show_router_bgp_asn.py +++ b/examples/show_router_bgp_asn.py @@ -7,7 +7,7 @@ # pylint: disable=import-error, import-outside-toplevel, line-too-long, too-many-branches, too-many-locals, too-many-statements """ -Tested on: SR OS 24.3.R1 +Tested on: SR OS 24.3.R2 Show all BGP peers for an ASN. @@ -221,6 +221,8 @@ def show_router_bgp_asn_output(connection_object, asn): num_down_neighbors = 0 num_disabled_neighbors = 0 for neighbor in sorted(bgp_config): + num_families = 0 + if asn == 0 or int(asn) == bgp_config[neighbor]["peer-as"].data: # Print line 1 print(bright_cyan, end="") @@ -270,7 +272,6 @@ def show_router_bgp_asn_output(connection_object, asn): == "Established" ): num_up_neighbors += 1 - num_families = 0 for family in sorted( bgp_stats[neighbor]["statistics"]["negotiated-family"] ): diff --git a/pysros/__init__.py b/pysros/__init__.py index 00d21cc..c91aaaf 100644 --- a/pysros/__init__.py +++ b/pysros/__init__.py @@ -3,3 +3,5 @@ __all__ = ("management", "exceptions", "wrappers", "pprint", ) __doc__ = """Library for management of Nokia SR OS nodes.""" + +__version__ = "24.10.1" diff --git a/pysros/errors.py b/pysros/errors.py index c2d8c85..608100a 100644 --- a/pysros/errors.py +++ b/pysros/errors.py @@ -3,7 +3,7 @@ from .exceptions import (ActionTerminatedIncompleteError, InternalError, InvalidPathError, JsonDecodeError, ModelProcessingError, SrosConfigConflictError, - SrosMgmtError, XmlDecodeError, make_exception) + SrosMgmtError, XmlDecodeError, NotSupportedNodeMethodError, make_exception) __doc__ = """This module contains error definitions for pySROS. @@ -22,7 +22,7 @@ pysros_err_can_not_find_yang = (ModelProcessingError, "Cannot find yang '{yang_name}'") pysros_err_cannot_call_go_to_parent = (InvalidPathError, "Cannot call go_to_parent on root") pysros_err_cannot_delete_from_state = (SrosMgmtError, "Cannot delete from state tree") -pysros_err_cannot_lock_and_unlock_running = (SrosMgmtError, "Cannot lock and unlock running or intended config") +pysros_err_cannot_lock_and_unlock_readonly_ds = (SrosMgmtError, "Cannot lock and unlock read-only datastore") pysros_err_cannot_modify_config = (SrosMgmtError, "Cannot modify running or intended config") pysros_err_cannot_modify_state = (SrosMgmtError, "Cannot modify state tree") pysros_err_cannot_pars_path = (ModelProcessingError, "Cannot parse path {path!r}") @@ -44,6 +44,7 @@ pysros_err_filter_not_supported_on_leaves = (InvalidPathError, "Filter is not supported for leaves") pysros_err_filter_should_be_dict = (TypeError, "Filter argument should be a dict") pysros_err_filter_wrong_leaf_value = (TypeError, "Unsupported leaf filter for '{leaf_name}'") +pysros_err_invalid_parse_error = (InvalidPathError, "Invalid character while parsing string") pysros_err_incorrect_leaf_value = (SrosMgmtError, "Invalid value for leaf {leaf_name}") pysros_err_invalid_align = (ValueError, "Invalid align: '{align}'") pysros_err_invalid_col_description = (TypeError, "Invalid column description") @@ -55,6 +56,7 @@ pysros_err_invalid_module_set_id_or_content_id = (RuntimeError, "Invalid module-set-id") pysros_err_invalid_operation_on_key = (InvalidPathError, "Operation cannot be performed on key") pysros_err_invalid_operation_on_leaflist = (InvalidPathError, "Operation cannot be performed on leaflist") +pysros_err_invalid_path_error = (InvalidPathError, "Path does not exist in schema") pysros_err_invalid_path_operation_missing_keys = (InvalidPathError, "Cannot perform operation on list without specifying keys") pysros_err_invalid_rd_state = (InternalError, "Invalid database state") pysros_err_invalid_target = (ValueError, "Invalid target") @@ -73,7 +75,7 @@ pysros_err_no_data_found = (LookupError, "No data found") pysros_err_not_connected = (RuntimeError, "Not connected") pysros_err_not_found_slash_before_name = (InvalidPathError, "'/' not found before element name") -pysros_err_path_should_be_string = (TypeError, "path argument should be a string") +pysros_err_path_should_be_string = (TypeError, "Path argument should be a string") pysros_err_prefix_does_not_have_ns = (LookupError, "prefix '{prefix}' of '{name}' does not have corresponding namespace") pysros_err_root_path = (InvalidPathError, "Operation cannot be performed on root") pysros_err_target_should_be_list = (InvalidPathError, "Target should be a list") @@ -121,3 +123,5 @@ pysros_err_too_many_leaflist_annotations = (SrosMgmtError, "Too many annotations in leaflist") pysros_err_expected_type_but_got_another = (TypeError, "expected {expected} object but got {type_name}") pysros_err_annotation_invalid_module = (SrosMgmtError, "Invalid annotation module") +pysros_err_ambiguous_model_node = (SrosMgmtError, "Ambiguous model node") +pysros_err_unsupported_node_method = (NotSupportedNodeMethodError, "Method not supported by the node") diff --git a/pysros/exceptions.py b/pysros/exceptions.py index 088ec3b..d69a39f 100644 --- a/pysros/exceptions.py +++ b/pysros/exceptions.py @@ -98,6 +98,10 @@ class XmlDecodeError(Exception): pass +class NotSupportedNodeMethodError(Exception): + """Exception raised when method which is not supported by box is called.""" + + def make_exception(arg, **kwarg): """Create an exception. diff --git a/pysros/identifier.py b/pysros/identifier.py index 63eb702..3638551 100644 --- a/pysros/identifier.py +++ b/pysros/identifier.py @@ -1,6 +1,7 @@ # Copyright 2021-2024 Nokia import re +import functools from .errors import * from .errors import make_exception @@ -29,6 +30,18 @@ def __eq__(self, other): return self is other +class Wildcard(metaclass=_Singleton): + def __str__(self): + return f"{self.__class__.__name__}()" + + def __hash__(self): + return id(self) + + def __eq__(self, other): + return self is other + + +@functools.total_ordering class Identifier: """Class to hold prefix-name pair for single YANG entry""" __slots__ = "_name", "_prefix" @@ -92,13 +105,16 @@ def __repr__(self): def __eq__(self, other): if type(other) is Identifier: - return self._name == other._name and self._prefix == other._prefix + return self._name == other._name and (self._prefix == other._prefix or Wildcard() in (self._prefix, other._prefix, )) elif type(other) == str: if ":" in other: return self == Identifier.from_model_string(other) return self.name == other return False + def __lt__(self, other): + return self.debug_string < other.debug_string + def __ne__(self, other): return not (self == other) @@ -123,3 +139,11 @@ def name(self): @property def model_string(self): return f"{self.prefix+':' if self.prefix else ''}{self.name}" + + @property + def debug_string(self): + if self.is_lazy_bound(): + return f"LazyBound():{self._name}" + if self.is_builtin(): + return f"Builtin():{self._name}" + return self.model_string diff --git a/pysros/management.py b/pysros/management.py index 2b68159..d26ef4d 100644 --- a/pysros/management.py +++ b/pysros/management.py @@ -28,7 +28,11 @@ from .model_builder import ModelBuilder from .model_walker import (ActionInputFilteredDataModelWalker, ActionOutputFilteredDataModelWalker, - FilteredDataModelWalker) + FilteredDataModelWalker, + JsonInstanceDataModelWalker, + JsonInstanceModelWalkerWithActionInput, + JsonInstanceModelWalkerWithActionOutput, + JsonInstanceModelWalkerActionOnly) from .request_data import COMMON_NAMESPACES, RequestData from .singleton import Empty from .wrappers import Container, LeafList, Annotation @@ -208,6 +212,8 @@ def __init__(self, *args, yang_directory, rebuild, **kwargs): pysros_err_could_not_create_conn, reason=e ) from None + self.debug = False + self.running = Datastore(self, 'running') self.intended = Datastore(self, 'intended') self.candidate = Datastore(self, 'candidate') @@ -218,11 +224,12 @@ def __init__(self, *args, yang_directory, rebuild, **kwargs): self._ns_map = types.MappingProxyType({model.name: model.namespace for model in self._models}) self._ns_map_rev = {v: k for k, v in self._ns_map.items()} self._mod_revs = {model.name: model.revision for model in self._models} + self._mod_revs.update((i.name, i.revision) for j in self._models for i in j.submodules) + self._sros = any(map(lambda x:x.startswith("urn:nokia.com:sros:ns:yang:sr:major-release"), self._nc.server_capabilities)) self.root, self.metadata = self._get_root(self._models) self._metadata_map = {i.name.model_string: i for i in self.metadata} self._metadata_map_no_module = {} - self.debug = False - self.intensive_checks = False + self._intensive_checks = False for v in self.metadata: self._metadata_map_no_module.setdefault(v.name.name, []).append(v) @@ -233,6 +240,7 @@ def _get_root(self, modules): hasher.update(b"Schema ver 5\n") hasher.update(b"Schema features\nBEGIN\n") hasher.update(b"ANNOTATIONS\n") + hasher.update(b"SRL\n") hasher.update(b"END\n") for m in yangs: hasher.update(f"mod:{m.name};rev:{m.revision};".encode()) @@ -249,7 +257,7 @@ def _get_root(self, modules): return (Model(storage, 0, None), metadata) # attempt to pickle. If load fails, we need to create a new tree - model_builder = ModelBuilder(self._yang_getter, self._ns_map) + model_builder = ModelBuilder(self._yang_getter, self._ns_map, self._sros) for mod in yangs: model_builder.register_yang(mod.name) model_builder.process_all_yangs() @@ -268,21 +276,21 @@ def _get_root(self, modules): return (model_builder.root, model_builder.metadata) - def _yang_getter(self, yang_name, *, debug=False): + def _yang_getter(self, yang_name): if self.yang_directory: - if debug: + if self.debug: print(f"open local file {yang_name}") with open(self._find_module(yang_name), "r", encoding="utf8") as f: return f.read() # download using netconf protocol - if debug: + if self.debug: start = datetime.datetime.now() print(f"GET SCHEMA {yang_name}") - response = self._nc.get_schema(yang_name) + response = self._nc.get_schema(yang_name, self._mod_revs[yang_name]) data = response.xpath("/ncbase:rpc-reply/monitoring:data", COMMON_NAMESPACES)[0].text - if debug: + if self.debug: print(f" * {len(data)/1024:.1f} kB downloaded in {(datetime.datetime.now()-start).total_seconds():.3f} sec") return data @@ -308,13 +316,13 @@ def _get_yang_models(self): with self._process_connected(): yangs_resp = self._nc.get(filter=("subtree", subtree)) - result = [] modules = yangs_resp.xpath( "/ncbase:rpc-reply/ncbase:data/library:modules-state/library:module", COMMON_NAMESPACES ) get_text = lambda e, path: e.findtext(path, namespaces=COMMON_NAMESPACES) + rev_map = {} for m in modules: submodules = [] for sm in m.xpath("./library:submodule", namespaces=COMMON_NAMESPACES): @@ -323,25 +331,30 @@ def _get_yang_models(self): revision=get_text(sm, "./library:revision"), )) submodules.sort(key=lambda sm: sm.name) - result.append(YangModule( - name=get_text(m, "./library:name"), + name = get_text(m, "./library:name") + rev_map.setdefault(name, []).append(YangModule( + name=name, namespace=get_text(m, "./library:namespace"), revision=get_text(m, "./library:revision"), submodules=tuple(submodules) )) - return tuple(result) + return tuple(sorted(i, key=(lambda m: m.revision))[-1] for i in rev_map.values()) def _request_data(self, **kwargs): return RequestData( self.root, self._metadata_map, self._metadata_map_no_module, self._ns_map, - self._ns_map_rev, **kwargs + self._ns_map_rev, self._sros, **kwargs ) + def _check_accessibility(self): + if not self._sros: + raise make_exception(pysros_err_unsupported_node_method) + def _action(self, path, value): rd = self._request_data(walker=ActionInputFilteredDataModelWalker) current = rd.process_path(path) - if self.intensive_checks: + if self._intensive_checks: rd.sanity_check() if not current.is_action(): raise make_exception(pysros_err_unsupported_action_path) @@ -371,7 +384,7 @@ def _action(self, path, value): # data should be wrapped in dummy root. RPC reply can be dummy root if does not have attributes response.xpath('.')[0].attrib.clear() rd.set_action_as_xml(path, response) - if self.intensive_checks: + if self._intensive_checks: rd.sanity_check() current = rd.process_path(path, strict=True) model = current.to_model() @@ -417,7 +430,7 @@ def convert_walker(action_io): else: raise NotImplementedError() - if self.intensive_checks: + if self._intensive_checks: rd.sanity_check() if dst_fmt == "pysros": @@ -493,6 +506,7 @@ def cli(self, command): .. Reviewed by PLM 20220901 """ + self._check_accessibility() with self._process_connected(): output = self._nc.md_cli_raw_command(command) status = output.xpath( @@ -511,20 +525,20 @@ def cli(self, command): return output[0].text if output else "" def action(self, path, value={}): - """Perform a YANG modeled action on SR OS by providing the *json-instance-path* to the + """Perform a YANG-modeled action on SR OS by providing the *json-instance-path* to the action statement in the chosen operations YANG model, and the pySROS data structure to - match the YANG modeled input for that action. + match the YANG-modeled input for that action. This method provides structured data input and output for available operations. :param path: *json-instance-path* to the YANG action. :type path: str :param value: pySROS data structure providing the input data for the chosen action. :type value: pySROS data structure - :returns: YANG modeled, structured data representing the output of the modeled action (operation) + :returns: YANG-modeled, structured data representing the output of the modeled action (operation) :rtype: pySROS data structure .. code-block:: python - :caption: Example calling the **ping** YANG modeled action (operation) + :caption: Example calling the **ping** YANG-modeled action (operation) :name: pysros-action-example >>> from pysros.management import connect @@ -616,6 +630,7 @@ def action(self, path, value={}): .. Reviewed by PLM 20220908 """ + self._check_accessibility() with self._process_connected(): return self._action(path, value) @@ -626,7 +641,7 @@ def convert(self, path, payload, *, source_format, destination_format, pretty_pr object that :py:meth:`convert` is being called against. :param path: *json-instance-path* to the location in the YANG schema that the - payload uses as its YANG modeled root. + payload uses as its YANG-modeled root. :type path: str :param payload: Input data for conversion. The payload must be valid data according to the YANG schema associated with the :py:class:`Connection` object and @@ -641,7 +656,7 @@ def convert(self, path, payload, *, source_format, destination_format, pretty_pr :type destination_format: str :param pretty_print: Format the output for human consumption. :type pretty_print: bool, optional - :param action_io: When converting the input/output of a YANG modeled operation (action), it is possible + :param action_io: When converting the input/output of a YANG-modeled operation (action), it is possible for there to be conflicting fields in the input and output sections of the YANG. This parameter selects whether to consider the payload against the ``input`` or ``output`` section of the YANG. Default: ``output``. @@ -667,6 +682,143 @@ def convert(self, path, payload, *, source_format, destination_format, pretty_pr destination_format, pretty_print, action_io ) + def list_paths(self, path="/", *, action_io=None): + """Obtain the JSON instance paths supported by a specific + :py:class:`.Connection` object. The :ref:`modeled-paths` + section describes more details about the JSON instance + path format. The method returns the supported schema + nodes and does not provide any instance data. + + YANG-modeled operations (called ``action`` in YANG) are not + returned by default. Use the ``action_io`` parameter to + output YANG-modeled operations. The path to the action + statement, or the paths to the actions input or output + parameters may be chosen. + + .. note:: + + List keys are shown in the JSON instance path but do not contain the list key's value + as this is instance data. + + :param path: Path to start the list of the supported paths from. The path + is an instance-identifier based on + `RFC 6020 `_ + and `RFC 7951 `_. + The path can be obtained from an SR OS device using the + ``pwc json-instance-path`` MD-CLI command. + The path may point to a YANG Container, List, Leaf or a Leaf-List. + The default is to provide all supported JSON instance paths from the root. + :type path: str + :param action_io: Obtain YANG action paths. Supported values are: ``action_only``, + ``input`` or ``output``. + :type action_io: str + + :return: A :py:class:`generator` object containing the supported JSON instance paths. + :rtype: :class:`generator` + + :raises KeyError: Unsupported ``action_io`` option. + + .. code-block:: python + :caption: Example printing all supported paths + :name: pysros-management-connection-list-paths-all-example + :emphasize-lines: 5-6 + + from pysros.management import connect + + connection_object = connect() + + for path in connection_object.list_paths(): + print(path) + + .. code-block:: python + :caption: Example printing all supported paths that contain the word "openconfig" + :name: pysros-management-connection-list-paths-all-openconfig-example + :emphasize-lines: 5-7 + + from pysros.management import connect + + connection_object = connect() + + for path in connection_object.list_paths(): + if "openconfig" in path: + print(path) + + .. code-block:: python + :caption: Example printing all supported paths in a specific path + :name: pysros-management-connection-list-paths-specific-path-example + :emphasize-lines: 5-6 + + from pysros.management import connect + + connection_object = connect() + + for path in connection_object.list_paths("/nokia-conf:configure/router[router-name]/static-routes"): + print(path) + + .. code-block:: python + :caption: Example printing all input and output parameters to all ``perform card`` operations + :name: pysros-management-connection-list-paths-specific-path-actions-example + :emphasize-lines: 5-10 + + from pysros.management import connect + + connection_object = connect() + + for direction in ["input", "output"]: + print("=" * 79) + print("%s paramaters" % direction) + print("-" * 79) + for path in c.list_paths("/nokia-oper-perform:perform/card", action_io=direction): + print(path) + + + .. Reviewed by PLM 20241003 + + + """ + with self._process_connected(): + walkers = { + None : JsonInstanceDataModelWalker, + "input" : JsonInstanceModelWalkerWithActionInput, + "output": JsonInstanceModelWalkerWithActionOutput, + "action_only": JsonInstanceModelWalkerActionOnly, + } + try: + walker = walkers[action_io](self.root, self._sros) + validation_walker = walkers[action_io](self.root, self._sros) + except KeyError: + raise make_exception(pysros_err_unsupported_action_io) + if path == "/": + return walker.export_paths() + exception_with_key_value = None + exception_without_key_value = None + try: + walker = walker.user_path_parse(self.root, path, self._sros, verify_keys=True) + except Exception as e: + exception_with_key_value = e + for node, keys in zip(walker.path, walker.keys): + validation_walker.go_to_child(node.name) + if not keys: continue + for key, value in keys.items(): + if all((key != child.name.name and str(child.data_def_stm) == "leaf" + for child in validation_walker.current.children)): + raise make_exception(pysros_err_invalid_key_in_path) + with validation_walker.visit_child(key): + type_adjusted_value = validation_walker.get_type().to_value(value) + if not validation_walker.check_field_value(value=type_adjusted_value): + raise make_exception(pysros_err_invalid_key_in_path) + if exception_with_key_value is None: + return walker.export_paths() + try: + walker = walker.user_path_parse(self.root, path, self._sros, verify_keys=False) + return walker.export_paths() + except Exception as e: + exception_without_key_value = e + if exception_with_key_value is None and exception_without_key_value is not None: + raise exception_without_key_value + else: + raise exception_with_key_value + def session_id(self): """Returns the current connections session-id. @@ -696,13 +848,16 @@ class _ExistReason(Enum): delete = auto() def __init__(self, connection, target): - if target not in ('running', 'candidate', 'intended'): + if target not in ('running', 'candidate', 'intended', ): raise make_exception(pysros_err_invalid_target) self.connection = connection - self.nc = connection._nc self.target = target self.transaction = None - self.debug = False + self.debug = self.connection.debug + + @property + def _nc(self): + return self.connection._nc def _get_defaults(self, defaults): return "report-all" if defaults else None @@ -723,16 +878,16 @@ def _prepare_root_ele(self, subtree, path, filter_tag = "filter"): def _operation_get(self, subtree, defaults, path): if subtree: root = self._prepare_root_ele(subtree, path) - return self.nc.get(filter=root, with_defaults=self._get_defaults(defaults)) + return self._nc.get(filter=root, with_defaults=self._get_defaults(defaults)) else: - return self.nc.get(with_defaults=self._get_defaults(defaults)) + return self._nc.get(with_defaults=self._get_defaults(defaults)) def _operation_get_config(self, subtree, defaults, path): if subtree: root = self._prepare_root_ele(subtree, path) - return self.nc.get_config(source=self.target, filter=root, with_defaults=self._get_defaults(defaults)) + return self._nc.get_config(source=self.target, filter=root, with_defaults=self._get_defaults(defaults)) else: - return self.nc.get_config(source=self.target, with_defaults=self._get_defaults(defaults)) + return self._nc.get_config(source=self.target, with_defaults=self._get_defaults(defaults)) def _operation_get_data(self, subtree, defaults, path): ds_pfx = "ds" @@ -749,10 +904,10 @@ def _operation_get_data(self, subtree, defaults, path): xml_defaults = etree.SubElement(xml_get, "with-defaults") xml_defaults.text = "report-all" - return self.nc.rpc(xml_get) + return self._nc.rpc(xml_get) def _get(self, path, *, defaults=False, custom_walker=None, config_only=False, filter=None): - model_walker = custom_walker if custom_walker else FilteredDataModelWalker.user_path_parse(self.connection.root, path) + model_walker = custom_walker if custom_walker else FilteredDataModelWalker.user_path_parse(self.connection.root, path, self.connection._sros) if config_only and model_walker.is_state: raise make_exception(pysros_err_no_data_found) @@ -789,7 +944,7 @@ def _get(self, path, *, defaults=False, custom_walker=None, config_only=False, f rd = self.connection._request_data() rd.set_as_xml(response) - if self.connection.intensive_checks: + if self.connection._intensive_checks: rd.sanity_check() try: @@ -853,7 +1008,8 @@ def _set(self, path, value, action, method="default", annotations_only=False): raise make_exception(pysros_err_annotation_invalid_type) model_walker = FilteredDataModelWalker.user_path_parse( self.connection.root, - path + path, + self.connection._sros ) if model_walker.current.config == False: raise make_exception(pysros_err_cannot_modify_state) @@ -879,7 +1035,7 @@ def _set(self, path, value, action, method="default", annotations_only=False): elif method == "merge": current.merge() - if self.connection.intensive_checks: + if self.connection._intensive_checks: rd.sanity_check() children = rd.to_xml() @@ -890,7 +1046,7 @@ def _set(self, path, value, action, method="default", annotations_only=False): print("SET request") print(f"path: '{path}', value: '{value}'") print(etree.dump(config)) - self.nc.edit_config( + self._nc.edit_config( target=self.target, default_operation=default_operation, config=config @@ -900,7 +1056,7 @@ def _delete(self, path, annotations_only): self._set(path, None, Datastore._SetAction.delete, annotations_only=annotations_only) def _exists(self, path, exist_reason): - model_walker = FilteredDataModelWalker.user_path_parse(self.connection.root, path) + model_walker = FilteredDataModelWalker.user_path_parse(self.connection.root, path, self.connection._sros) # if exists is called as a check before deletion, check for path to avoid # incorrect errors such as pysros_err_can_check_state_from_running_only # as we want to handle state delete related errors first @@ -933,7 +1089,8 @@ def _exists(self, path, exist_reason): def _get_list_keys(self, path, defaults): model_walker = FilteredDataModelWalker.user_path_parse( self.connection.root, - path + path, + self.connection._sros ) self._check_empty_string(model_walker) @@ -947,7 +1104,7 @@ def _get_list_keys(self, path, defaults): current.entry_get_keys() - if self.connection.intensive_checks: + if self.connection._intensive_checks: rd.sanity_check() config = rd.to_xml() @@ -965,7 +1122,7 @@ def _get_list_keys(self, path, defaults): rd = self.connection._request_data() rd.set_as_xml(response) - if self.connection.intensive_checks: + if self.connection._intensive_checks: rd.sanity_check() try: @@ -997,14 +1154,15 @@ def _compare(self, output_format, user_path): if path: model_walker = FilteredDataModelWalker.user_path_parse( self.connection.root, - path + path, + self.connection._sros ) if model_walker.current.config == False: raise make_exception(pysros_err_unsupported_compare_endpoint) rd = self.connection._request_data() current = rd.process_path(model_walker) - if self.connection.intensive_checks: + if self.connection._intensive_checks: rd.sanity_check() if not current.is_compare_supported_endpoint(): @@ -1031,7 +1189,7 @@ def _compare(self, output_format, user_path): print("COMPARE request") print(etree.dump(xml_action)) - reply = self.nc.rpc(xml_action) + reply = self._nc.rpc(xml_action) if self.debug: print("COMPARE reply") @@ -1054,11 +1212,11 @@ def _compare(self, output_format, user_path): def _commit(self): try: - self.nc.commit() + self._nc.commit() except nc_RPCError as e: if e.message and e.message.find("MGMT_CORE #2703:") > -1: - self.nc.discard_changes() - self.nc.commit() + self._nc.discard_changes() + self._nc.commit() raise make_exception( pysros_err_commit_conflicts_detected ) from None @@ -1249,7 +1407,7 @@ def delete(self, path, commit=True, *, annotations_only=False): :type path: str :param commit: Specify whether commit should be executed after delete. Default True. :type commit: bool - :param annotations_only: If specified, object where path is pointing is not deleted but + :param annotations_only: If specified, object where path is pointing is not deleted but only the annotation attached to the object. :type annotations_only: bool @@ -1353,9 +1511,9 @@ def lock(self): .. Reviewed by TechComms 20220624 """ with self.connection._process_connected(): - if self.target in ("running", "intended"): - raise make_exception(pysros_err_cannot_lock_and_unlock_running) - self.nc.lock(target=self.target) + if self.target not in ("candidate", "running"): + raise make_exception(pysros_err_cannot_lock_and_unlock_readonly_ds) + self._nc.lock(target=self.target) def unlock(self): """Unlock the configuration datastore. Transitions an exclusive candidate configuration to a candidate @@ -1370,9 +1528,9 @@ def unlock(self): .. Reviewed by TechComms 20220624 """ with self.connection._process_connected(): - if self.target in ("running", "intended"): - raise make_exception(pysros_err_cannot_lock_and_unlock_running) - self.nc.unlock(target=self.target) + if self.target not in ("candidate", "running"): + raise make_exception(pysros_err_cannot_lock_and_unlock_readonly_ds) + self._nc.unlock(target=self.target) def commit(self): """Commit the candidate configuration. @@ -1404,7 +1562,7 @@ def discard(self): with self.connection._process_connected(): if self.target in ('running', 'intended'): raise make_exception(pysros_err_cannot_modify_config) - self.nc.discard_changes() + self._nc.discard_changes() def compare(self, path="", *, output_format): """Perform a comparison of the uncommitted candidate configuration with the baseline @@ -1452,6 +1610,7 @@ def compare(self, path="", *, output_format): .. Reviewed by PLM 20221005 """ + self.connection._check_accessibility() with self.connection._process_connected(): return self._compare(output_format, path) diff --git a/pysros/model.py b/pysros/model.py index 871c430..aed47ff 100644 --- a/pysros/model.py +++ b/pysros/model.py @@ -3,8 +3,10 @@ import copy import locale import sys +import functools +from decimal import Decimal from enum import Enum, IntEnum, IntFlag -from typing import List, Optional +from typing import List, Optional, Union from .errors import * from .errors import make_exception @@ -12,6 +14,46 @@ from .yang_type import YangType +class YangVersion: + ver1_0: "YangVersion" + ver1_1: "YangVersion" + + def __init__(self, version: str = "1.0"): + self.value = version + + @property + def value(self): + return self._val + + @value.setter + def value(self, version: str): + self._val = Decimal(version) + + def __lt__(self, other): + return self._val < other._val + + def __le__(self, other): + return self._val <= other._val + + def __eq__(self, other): + return self._val == other._val + + def __ne__(self, other): + return self._val != other._val + + def __gt__(self, other): + return self._val > other._val + + def __ge__(self, other): + return self._val >= other._val + + def __repr__(self): + return f"YangVersion('{self._val}')" + +YangVersion.ver1_0 = YangVersion("1.0") +YangVersion.ver1_1 = YangVersion("1.1") + + class AModel: class StatementType(Enum): leaf_ = 0 @@ -38,14 +80,32 @@ class StatementType(Enum): deviation_ = 21 deviate_ = 22 annotate_ = 23 + belongs_to_ = 24 + refine_ = 25 + extended = 26 __slots__ = () + name: Identifier + children: "List[AModel]" + yang_type: Optional[YangType] + units: Optional[str] + namespace: Optional[str] + default: Optional[Union[str, List[str]]] + mandatory: Optional[str] + status: Optional[str] + presence_container: bool + user_ordered = False + local_keys: List[str] + data_def_stm: StatementType + parent: "AModel" + config: bool + def recursive_walk(self, cb): cont = cb(self) - if cont == False: + if cont is False: return - for i in self.children: + for i in self.children[:]: # during iteration you can modify children i.recursive_walk(cb) def __str__(self): @@ -57,7 +117,7 @@ def __repr__(self): def __deepcopy__(self, memo): raise make_exception(pysros_err_use_deepcopy) - def debug_print(self, prefix="", last=False): + def debug_print(self, prefix="", last=False, with_blueprints=False): class Colors: RED = '\033[1;31;48m' GREEN = '\033[1;32;48m' @@ -114,13 +174,22 @@ class AsciiArt: t_sufix.append(colorize(self.target_path, Colors.PURPLE)) print(f"""{prefix[:-1]}{field_prefix}{self.debug_flags()} {self.name} [{" ".join((t,*t_sufix))}]""") + if with_blueprints and self.blueprint: + blueprint_prefix = prefix + " " + for b in self.blueprint: + if b[0]: + print(f"{blueprint_prefix}> {' '.join(map(str, b[1]))} {{") + blueprint_prefix = blueprint_prefix + " " + else: + blueprint_prefix = blueprint_prefix[:-4] + print(f"{blueprint_prefix}> }}") remain = self.children_size - 1 for i in self.children: new_prefix = prefix + (" " if not remain else AsciiArt.VERTICAL_LINE) if not prefix: new_prefix = new_prefix[3:] - i.debug_print(new_prefix, not remain) + i.debug_print(new_prefix, not remain, with_blueprints=with_blueprints) remain -= 1 def debug_flags(self) -> str: @@ -198,15 +267,18 @@ class BuildingModel(AModel): "_parent", "config", "blueprint", + "arg", + "nsmap", + "yang_version", ) - def __init__(self, name: Identifier, data_def_stm: AModel.StatementType, parent): + def __init__(self, name: Union[Identifier, str], data_def_stm: AModel.StatementType, parent, yang_version): self.name = Identifier.builtin(name) if type(name) == str else name self.children: List[Model] = [] self.yang_type: Optional[YangType] = None self.units: Optional[str] = None self.namespace: Optional[str] = None - self.default: Optional[str] = None + self.default: Optional[Union[str, List[str]]] = None self.mandatory: Optional[str] = None self.status: Optional[str] = None self.presence_container = False @@ -218,6 +290,9 @@ def __init__(self, name: Identifier, data_def_stm: AModel.StatementType, parent) self.identity_bases = None if data_def_stm != AModel.StatementType.identity_ else [] self.config: bool = True self.blueprint = [] + self.arg = None + self.nsmap = None + self.yang_version = yang_version @property def parent(self): @@ -262,6 +337,12 @@ def delete_from_parent(self, *, quiet=True): ) del self.parent.children[idx] self.parent = None + return self + + def annihilate(self): + for child in self.children: + child.parent = self.parent + self.delete_from_parent(quiet=False) def deepcopy(self, parent): cls = self.__class__ diff --git a/pysros/model_builder.py b/pysros/model_builder.py index 615d69a..6d33bfe 100644 --- a/pysros/model_builder.py +++ b/pysros/model_builder.py @@ -1,15 +1,17 @@ # Copyright 2021-2024 Nokia import copy -from collections import defaultdict +from collections import defaultdict, OrderedDict from itertools import chain -from typing import DefaultDict, Dict, Optional, Set, Union +from typing import DefaultDict, Dict, Optional, Set, Union, List +from decimal import Decimal from .errors import * from .errors import InvalidPathError, make_exception -from .identifier import Identifier, LazyBindModule -from .model import AModel, BuildingModel, Model, StorageConstructionModel +from .identifier import Identifier, LazyBindModule, Wildcard +from .model import AModel, BuildingModel, Model, StorageConstructionModel, YangVersion from .model_path import ModelPath +from .path_utils import get_path from .model_walker import DataModelWalker, ModelWalker from .request_data import COMMON_NAMESPACES from .tokenizer import yang_parser @@ -24,12 +26,12 @@ class YangHandler: """Handler for yang processing.""" TAGS_WITH_MODEL = ( "container", "list", "leaf", "typedef", "module", "submodule", "uses", - "leaf-list", "import", "identity", "notification", "rpc", - "input", "output", "choice", "case", "deviate", "action" + "leaf-list", "import", "identity", "notification", "rpc", "belongs-to", + "input", "output", "choice", "case", "deviate", "action", ) TAGS_FORCE_MODULE = ( "grouping", "identity", "typedef", "uses", "augment", - "deviation", "type", "base" + "deviation", "type", "base", "refine", ) TAGS_ARG_IS_IDENTIFIER = ("base", "type", ) TAGS_ARG_IS_YANG_PATH_NOT_ABSOLUTE_SCHEMA_ID = ("path", ) @@ -75,11 +77,10 @@ class YangHandler: "list", "mandatory", "max-elements", - "md:annotation", "min-elements", "modifier", "module", - "must", + # "must", "namespace", "notification", "ordered-by", @@ -119,54 +120,61 @@ def __init__(self, builder: "ModelBuilder", root: BuildingModel): self.in_type = 0 self.derived_identities: Dict[Identifier, Set[Identifier]] = {} self.include_stack = [{}] + self.yang_version_stack = [YangVersion()] self.module = None - self.prefix = None + self._prefix = [None] self.ignore_depth = 0 def enter(self, name, arg): - if self.ignore_depth or name not in self.TAGS_SHOULD_BE_PROCESSED: + if self.ignore_depth or (name not in self.TAGS_SHOULD_BE_PROCESSED and ":" not in name): self.ignore_depth += 1 return self.full_path.append(name) - if name in ("input", "output"): + if name in ("input", "output", ): arg = name if name == "case" and not arg: arg = "unnamed" if name == 'typedef': new_id = self.get_identifier(arg, name in self.TAGS_FORCE_MODULE) assert not new_id.is_lazy_bound() - new = BuildingModel(new_id, BuildingModel.StatementType.typedef_, None) + new = BuildingModel(new_id, BuildingModel.StatementType.typedef_, None, self.yang_version) self.path.append(new) elif name in self.TAGS_WITH_MODEL: new_id = self.get_identifier(arg, name in self.TAGS_FORCE_MODULE) - new = BuildingModel(new_id, BuildingModel.StatementType[name.replace("-", "_") + "_"], self.model) + new = BuildingModel(new_id, BuildingModel.StatementType[name.replace("-", "_") + "_"], self.model, self.yang_version) self.path.append(new) - elif name in ("augment", "deviation"): + elif name in ("augment", "deviation", "refine", ): new_id = self.get_identifier(name, name in self.TAGS_FORCE_MODULE) - new = BuildingModel(new_id, BuildingModel.StatementType[name + "_"], None) - new.target_path = self.yang_path_to_model_path(arg) - (self.builder.augments if name == "augment" else self.builder.deviations).append(new) + if self.model.data_def_stm == Model.StatementType.uses_: + new = BuildingModel(new_id, BuildingModel.StatementType[name + "_"], self.model, self.yang_version) + new.target_path = get_path(arg, absolute_path_allowed=False, relative_path_allowed=True, key_allowed=False, prefix_resolver=self.resolve_prefix) + else: + # global augment or deviation + new = BuildingModel(new_id, BuildingModel.StatementType[name + "_"], None, self.yang_version) + (self.builder.augments if name == "augment" else self.builder.deviations).append(new) + new.target_path = get_path(arg, absolute_path_allowed=True, relative_path_allowed=False, key_allowed=False, prefix_resolver=self.resolve_prefix) self.path.append(new) elif name == "grouping": new_id = self.get_identifier(arg, name in self.TAGS_FORCE_MODULE) - new = BuildingModel(new_id, BuildingModel.StatementType[name.replace("-", "_") + "_"], None) + new = BuildingModel(new_id, BuildingModel.StatementType[name.replace("-", "_") + "_"], None, self.yang_version) self.path.append(new) assert new_id not in self.builder.groupings self.builder.groupings[new_id] = new - elif name == "md:annotation": - new_id = self.get_identifier(arg, name in self.TAGS_FORCE_MODULE) - new = BuildingModel(new_id, BuildingModel.StatementType.annotate_, None) + elif ":" in name: + new_id = Identifier.from_model_string(name) + new = BuildingModel(new_id, BuildingModel.StatementType.extended, self.model, self.yang_version) + new.arg = f"{self.module}:{arg}" + new.nsmap = self.prefixModuleMapping self.path.append(new) - self.builder.metadata.append(new) else: if name in self.TAGS_ARG_IS_IDENTIFIER: if name == "type" and should_be_buildin(arg): arg = Identifier.builtin(arg) else: - arg = self.get_identifier(arg, name in self.TAGS_FORCE_MODULE) + arg = self.get_identifier(arg, name in self.TAGS_FORCE_MODULE, force_allow_yang_string=True) elif name in self.TAGS_ARG_IS_YANG_PATH_NOT_ABSOLUTE_SCHEMA_ID: - arg = self.yang_path_to_model_path(arg, absolute_schema_id=False) + arg = get_path(arg, absolute_path_allowed=True, relative_path_allowed=True, key_allowed=True, key_expect_path=True, prefix_resolver=self.resolve_prefix, default_module=self.module) self.model.blueprint.append((True, (name, arg))) handle_name = f'handle_early_{name.replace("-", "_DASH_")}' @@ -199,7 +207,7 @@ def leave(self, name): popped = self.full_path.pop() assert popped == name - if name in (*self.TAGS_WITH_MODEL, "grouping", "augment", "deviation", "md:annotation"): + if name in (*self.TAGS_WITH_MODEL, "grouping", "augment", "deviation", "refine",) or ":" in name: self.path.pop() else: self.model.blueprint.append((False, name)) @@ -229,29 +237,16 @@ def construct_enter(self, instruction): handler = getattr(self, handle_name) handler(arg) - def get_identifier(self, s, force_module=False): + def get_identifier(self, s, force_module=False, force_allow_yang_string=False): + assert not force_module or self.module is not None if self.module is not None: module = LazyBindModule() if (self.grouping_depth and not force_module) else self.module return Identifier.from_yang_string(s, module, self.prefixModuleMapping) + parts = s.split(":") + if force_allow_yang_string and len(parts) > 1: + return Identifier(parts[0], parts[1]) return Identifier.builtin(s) - @staticmethod - def model_path_from_string(path: str, default_module: str, prefixModuleMapping: Dict[str, str], *, absolute_schema_id): - assert isinstance(path, str) - if not path.startswith(('/', '../')): - raise make_exception(pysros_err_cannot_pars_path, path=path) - - tokens = ModelWalker.tokenize_path(path, absolute_schema_id) - result = ModelPath(Identifier.from_yang_string( - p, default_module, prefixModuleMapping) for p in tokens) - if not result.is_valid(only_absolute_path=absolute_schema_id): - raise make_exception(pysros_err_invalid_yang_path, path=path) - return result - - def yang_path_to_model_path(self, path, absolute_schema_id=True): - default_module = self.module if absolute_schema_id else LazyBindModule() - return self.model_path_from_string(path, default_module, self.prefixModuleMapping, absolute_schema_id=absolute_schema_id) - @property def model(self): return self.path[-1] @@ -264,6 +259,14 @@ def model_type(self): def prefixModuleMapping(self): return self.include_stack[-1] + @property + def yang_version(self)->YangVersion: + return self.yang_version_stack[-1] + + @yang_version.setter + def yang_version(self, name): + self.yang_version_stack[-1].value = name + @property def last_yang_type(self): return self.model.yang_type[-1] if isinstance(self.model.yang_type, YangUnion) else self.model.yang_type @@ -280,7 +283,6 @@ def handle_base(self, ident): def handle_path(self, arg: str): if self.in_type: - assert isinstance(self.last_yang_type, LeafRef) self.last_yang_type.set_path(arg) def handle_type(self, identifier: str): @@ -299,8 +301,14 @@ def handle_namespace(self, arg: str): self.model.namespace = arg def handle_default(self, arg: str): - assert self.model.default is None - self.model.default = arg + if self.model.data_def_stm == Model.StatementType.leaf_list_: + if self.model.default is None: + self.model.default = [arg] + else: + self.model.default.append(arg) + else: + assert self.model.default is None + self.model.default = arg def handle_mandatory(self, arg: str): assert self.model.mandatory is None @@ -326,17 +334,17 @@ def handle_status(self, arg: str): def handle_enum(self, value: str): if self.in_type: - assert isinstance(self.last_yang_type, Enumeration), f"expected Enumeration, got {self.last_yang_type}" + assert isinstance(self.last_yang_type, (Enumeration, UnresolvedIdentifier, )), f"expected Enumeration, got {self.last_yang_type}" self.last_yang_type.add_enum(value) def handle_bit(self, value: str): if self.in_type: - assert isinstance(self.last_yang_type, Bits) - self.last_yang_type.add(value) + assert isinstance(self.last_yang_type, (Bits, UnresolvedIdentifier, )) + self.last_yang_type.add_bit(value) def handle_value(self, value: str): if self.in_type: - assert isinstance(self.last_yang_type, Enumeration) + assert isinstance(self.last_yang_type, (Enumeration, UnresolvedIdentifier, )) self.last_yang_type.set_last_enum_value(int(value)) def handle_presence(self, _arg): @@ -351,7 +359,9 @@ def handle_early_import(self, arg): def handle_early_include(self, arg): self.include_stack.append({}) + self.yang_version_stack.append(YangVersion()) self.builder.perform_parse(arg, self) + self.yang_version_stack.pop() self.include_stack.pop() def handle_key(self, arg): @@ -364,17 +374,24 @@ def handle_early_grouping(self, _arg): self.grouping_depth += 1 def handle_early_prefix(self, arg): - if self.full_path[-2] == "module": - self.prefix = arg - if self.full_path[-2] in ("import", "module"): + if self.full_path[-2] in ("belongs-to", ): + assert arg not in self.prefixModuleMapping + self.prefixModuleMapping[arg] = self.module + if self.full_path[-2] in ("import", "module", ): assert arg not in self.prefixModuleMapping self.prefixModuleMapping[arg] = self.model.name.name + def handle_early_yang_DASH_version(self, arg): + self.yang_version = arg + def handle_config(self, arg: str): if arg not in ('false', 'true'): raise make_exception(pysros_err_invalid_config) self.model.config = (arg == 'true') + def resolve_prefix(self, prefix: str): + return self.prefixModuleMapping[prefix] + def _dummy_getter(filename): assert False, "Missing getter" @@ -382,11 +399,12 @@ def _dummy_getter(filename): class ModelBuilder: """API for walking model tree.""" - def __init__(self, yang_getter=_dummy_getter, ns_map={}): + def __init__(self, yang_getter=_dummy_getter, ns_map={}, sros=True): self.root = BuildingModel( "root", BuildingModel.StatementType["container_"], - None + None, + YangVersion.ver1_1 ) self.metadata = [] self.types_to_resolve = dict() @@ -400,6 +418,7 @@ def __init__(self, yang_getter=_dummy_getter, ns_map={}): self.registered_modules = {} self._ns_map = ns_map self.yang_getter = yang_getter + self._sros = sros def get_module_content(self, yang_name): if yang_name in self.registered_modules: @@ -420,6 +439,8 @@ def DEBUG_parse_File(self, module_name, f): if module_name not in self.all_yangs: self.all_yangs.add(module_name) self.parsed_yangs.add(module_name) + if module_name not in self._ns_map: + self._ns_map[module_name] = f"NS:{module_name}" yang_parser(f, YangHandler(self, self.root)) def DEBUG_register_module(self, module_name, f): @@ -443,6 +464,7 @@ def resolve(self): self.resolve_config() self.resolve_namespaces() self.delete_blueprints() + self.resolve_metadata() self.resolve_metadata_exceptions() self.convert_model() @@ -461,28 +483,6 @@ def perform_parse(self, yang_name, yang_handler=None): yang_handler or YangHandler(self, self.root) ) - def add_child_to_uses(self, m: BuildingModel): - if m.data_def_stm == BuildingModel.StatementType.uses_: - assert not m.has_children - i = m - while True: - if i.data_def_stm == BuildingModel.StatementType.module_: - module = i.name.name - break - elif i.data_def_stm == BuildingModel.StatementType.augment_: - module = i.name.prefix - break - i = i.parent - - def resolve_unresolved(m: BuildingModel): - if m.name.prefix is LazyBindModule(): - m.prefix = module - - for i in self.groupings[m.name].children: - i.deepcopy(m) - m.recursive_walk(resolve_unresolved) - return False - def set_correct_types(self, m: BuildingModel): if isinstance(m.yang_type, (UnresolvedIdentifier, YangUnion)): tdm = resolve_typedefs_deep(TypeDefModel(m), self.resolved_types) @@ -490,7 +490,7 @@ def set_correct_types(self, m: BuildingModel): m.default = tdm.default m.units = tdm.units #RFC6020: If the type referenced by the leaf-list has a default value, it has no effect in the leaf-list. - if m.data_def_stm == AModel.StatementType.leaf_list_ and m.default: + if m.data_def_stm == AModel.StatementType.leaf_list_ and m.default and m.yang_version == YangVersion.ver1_0: m.default = None def resolve_typedefs(self): @@ -611,39 +611,72 @@ def replace_leafrefs(m: BuildingModel): if not isinstance(m.yang_type, LeafRef): return - w = DataModelWalker(m) + w = DataModelWalker(m, self._sros) assert m.data_def_stm in ( BuildingModel.StatementType.leaf_, BuildingModel.StatementType.leaf_list_ ) while isinstance(w.current.yang_type, LeafRef): - path = w.current.yang_type.path - module_name = w.current.name.prefix - if path._path and path._path[0].name != '..': - # absolute path - go to root - while not w.is_root: - w.go_to_parent() - w.go_to(path, module_name) + w.current.yang_type.path.move_walker(w, default_module=w.current.prefix) m.yang_type = w.current.yang_type self.walk_models(replace_leafrefs) def resolve_groupings(self): - def inner(m: BuildingModel): + module = None + def resolve_grouping(m: BuildingModel): if m.data_def_stm == BuildingModel.StatementType.uses_: - if m.has_children: - return False - for child in self.groupings[m.name].children: + grouping: BuildingModel = self.groupings[m.name] + grouping.recursive_walk(resolve_grouping) + uses_children, m.children = m.children, [] + for child in grouping.children: child.deepcopy(m) + self.process_uses_substmts(m, uses_children) + m.annihilate() + return False + def resolve_common(m: BuildingModel, module: str): + assert module is not LazyBindModule() + def resolve_unresolved_prefix(m: BuildingModel): + if m.prefix is LazyBindModule(): + m.prefix = module + grouping: BuildingModel = self.groupings[m.name] + uses_children, m.children = m.children, [] + for child in grouping.children: + child.deepcopy(m).recursive_walk(resolve_unresolved_prefix) + self.process_uses_substmts(m, uses_children) + m.annihilate() + return False + def resolve_data(m: BuildingModel): + nonlocal module + if m.data_def_stm == BuildingModel.StatementType.module_: + module = m.name.name + if m.data_def_stm == BuildingModel.StatementType.uses_: + return resolve_common(m, module) + def resolve_augment(m: BuildingModel): + if m.data_def_stm == BuildingModel.StatementType.uses_: + module = augment.prefix + return resolve_common(m, module) for grouping in self.groupings.values(): - grouping.recursive_walk(inner) - self.root.recursive_walk(self.add_child_to_uses) + grouping.recursive_walk(resolve_grouping) + self.root.recursive_walk(resolve_data) for augment in self.augments: - augment.recursive_walk(self.add_child_to_uses) + augment.recursive_walk(resolve_augment) + + def process_uses_substmts(self, uses: BuildingModel, substmts: List[BuildingModel]): + for m in substmts: + if m.data_def_stm == BuildingModel.StatementType.refine_: + w = ModelWalker(uses, self._sros) + m.target_path.move_walker(w, Wildcard()) + self.resolve_deviations_replace(w.current, m) + if m.data_def_stm == BuildingModel.StatementType.augment_: + w = ModelWalker(uses, self._sros) + m.target_path.move_walker(w, Wildcard()) + for i in m.children: + i.deepcopy(w.current) def process_augment(self, m: BuildingModel): if m.data_def_stm == BuildingModel.StatementType.augment_: - w = ModelWalker.path_parse(self.root, m.target_path) + w = ModelWalker.path_parse(self.root, self._sros, m.target_path, True) for i in m.children: i.deepcopy(w.current) @@ -652,7 +685,7 @@ def resolve_augments(self): while current: remaining = [] for augment in current: - w = ModelWalker(self.root) + w = ModelWalker(self.root, self._sros) try: w.go_to(augment.target_path) node = w.current @@ -681,7 +714,7 @@ def filter_blueprint(self, function, blueprint): def resolve_deviations(self): for deviation in self.deviations: - w = ModelWalker(self.root) + w = ModelWalker(self.root, self._sros) w.go_to(deviation.target_path) for deviate in deviation.children: if deviate.name.name == "add": @@ -691,15 +724,18 @@ def resolve_deviations(self): if instruction[0]: w.current.blueprint = list(self.filter_blueprint(lambda b: b[0:1] != instruction[1][0:1], w.current.blueprint)) if deviate.name.name == "replace": - depth = 0 - for instruction in deviate.blueprint: - if not depth: - w.current.blueprint = list(self.filter_blueprint(lambda b: b[0] != instruction[1][0], w.current.blueprint)) - depth += 1 if instruction[0] else -1 - w.current.blueprint = list(w.current.blueprint) + deviate.blueprint + self.resolve_deviations_replace(w.current, deviate) if deviate.name.name == "not-supported": w.current.delete_from_parent(quiet=False) + def resolve_deviations_replace(self, target: BuildingModel, deviate: BuildingModel): + depth = 0 + for instruction in deviate.blueprint: + if not depth: + target.blueprint = list(self.filter_blueprint(lambda b: b[0] != instruction[1][0], target.blueprint)) + depth += 1 if instruction[0] else -1 + target.blueprint = list(target.blueprint) + deviate.blueprint + def resolve_namespaces(self): def ns_set(m: BuildingModel): if m.namespace or m.name.prefix not in self._ns_map.keys(): @@ -730,8 +766,22 @@ def resolver(m: BuildingModel): self.walk_models(resolver) + def resolve_metadata(self): + def resolver(m: BuildingModel): + if m.data_def_stm == Model.StatementType.extended: + m.name = Identifier.from_yang_string(m.name.model_string, None, m.nsmap) + + if m.name == Identifier("ietf-yang-metadata", "annotation"): + m.delete_from_parent() + m.data_def_stm = Model.StatementType.annotate_ + m.name = Identifier.from_model_string(m.arg) + m.namespace = self._ns_map[m.name.prefix] + self.metadata.append(m) + return False + self.root.recursive_walk(resolver) + def resolve_metadata_exceptions(self): - new = BuildingModel(Identifier("ietf-netconf", "operation"), BuildingModel.StatementType.annotate_, None) + new = BuildingModel(Identifier("ietf-netconf", "operation"), BuildingModel.StatementType.annotate_, None, YangVersion.ver1_0) new.namespace = COMMON_NAMESPACES["ncbase"] type = Enumeration() type.add_enums("merge", "replace", "create", "delete", "remove") @@ -740,7 +790,7 @@ def resolve_metadata_exceptions(self): def convert_model(self): new_root = StorageConstructionModel("root", BuildingModel.StatementType["container_"], None) - w = DataModelWalker(self.root) + w = DataModelWalker(self.root, self._sros) stack = [new_root] @@ -823,7 +873,8 @@ def resolve_typedefs_shallow(t: TypeDefModel, typedefs: Dict[Identifier, TypeDef u_range = yang_type.yang_range frac_d = yang_type.fraction_digits length = yang_type.length - if any((u_range, frac_d, length)): + enums = yang_type.enums + if any((u_range, frac_d, length, enums, )): yang_type = copy.copy(r_model.yang_type) if u_range: yang_type.yang_range = merge_typedef_ranges(yang_type.yang_range, u_range) @@ -831,6 +882,11 @@ def resolve_typedefs_shallow(t: TypeDefModel, typedefs: Dict[Identifier, TypeDef yang_type.fraction_digits = frac_d if length: yang_type.length = length + if enums: + if isinstance(yang_type, (Enumeration, Bits, )): + yang_type.enums = OrderedDict(filter(lambda e: e[0] in enums, yang_type.enums.items())) + else: + yang_type.enums = enums else: yang_type = r_model.yang_type default = default or r_model.default diff --git a/pysros/model_path.py b/pysros/model_path.py index 4c46e48..3dd4a5d 100644 --- a/pysros/model_path.py +++ b/pysros/model_path.py @@ -20,8 +20,8 @@ def is_valid(self, *, only_absolute_path: bool): for p in self._path: if p.is_valid(): continue - if only_absolute_path or p.name != "..": - return False + if only_absolute_path or p.name != ".." or p.name != ".": + return True return True def repr_path(self): @@ -32,3 +32,6 @@ def __hash__(self): def __eq__(self, other): return isinstance(other, ModelPath) and self._path == other._path + + def __str__(self): + return "/" + "/".join(i.debug_string for i in self._path) diff --git a/pysros/model_walker.py b/pysros/model_walker.py index 7c30c74..63d7951 100644 --- a/pysros/model_walker.py +++ b/pysros/model_walker.py @@ -9,7 +9,7 @@ from .errors import * from .errors import InvalidPathError, SrosMgmtError, make_exception from .identifier import Identifier -from .model import Model +from .model import Model, AModel from .wrappers import Container, Leaf, LeafList from .yang_type import * @@ -39,15 +39,16 @@ class ModelWalker: Model.StatementType.augment_ ) - def __init__(self, model: Model): + def __init__(self, model: Model, sros: bool): self.path: List[Model] = [] self.keys = [] while model.parent is not None: - if model.data_def_stm in self._expected_dds: + if model.data_def_stm in self._expected_dds or not self.path: self.path.insert(0, model) self.keys.insert(0, dict()) model = model.parent self.model = model + self._sros = sros @property def current(self): @@ -91,12 +92,14 @@ def go_to(self, path: ModelPath, module: Optional[str] = None): self.go_to_parent() continue - prefix = self.path[-1].name.prefix if self.path else module - assert prefix + #prefix = self.path[-1].name.prefix if self.path else None + prefix = None try: - self.go_to_child( - Identifier(prefix, p.name)) + if prefix: + self.go_to_child(Identifier(prefix, p.name)) + else: + self.go_to_child(p.name) except SrosMgmtError as e: raise InvalidPathError(*e.args) from None @@ -115,7 +118,7 @@ def check_field_value(self, value, *, json=False, strict=False, is_convert=False assert False, "Checking field value for non-field walker" def get_parent(self): - res = self.__class__(self.model) + res = self.__class__(self.model, self._sros) if self.path: res.path = self.path[:-1] res.keys = self.keys[:-1] @@ -128,7 +131,7 @@ def get_child(self, child_name: Union[str, Identifier]): return res def copy(self): - res = self.__class__(self.model) + res = self.__class__(self.model, self._sros) res.path = self.path[:] res.keys = copy.deepcopy(self.keys) return res @@ -219,6 +222,18 @@ def visit_child(self, child: Union[str, Identifier]): finally: self.go_to_parent() + @contextlib.contextmanager + def visit_parent(self): + if not self.path: + raise make_exception(pysros_err_cannot_call_go_to_parent) + keys, model = self.keys.pop(), self.path.pop() + try: + yield + finally: + assert model.parent == self.current + self.keys.append(keys) + self.path.append(model) + def get_child_type(self, child_name: Union[str, Identifier]): return self._get_child(child_name).yang_type @@ -360,25 +375,21 @@ def _tokenize_(cls, string): res.append(i) @classmethod - def user_path_parse(cls, *args, accept_root=False, **kwargs): - res = cls.path_parse(*args, **kwargs) + def user_path_parse(cls, model_root, path, sros, *, accept_root=False, verify_keys=True): + res = cls.path_parse(model_root, path, sros, verify_keys) assert isinstance(res, ModelWalker) if not accept_root and res.is_root: raise make_exception(pysros_err_root_path) - for elem in res.path: - if elem.is_region_blocked: - raise make_exception( - pysros_err_unknown_element, element=elem.name.name - ) return res @classmethod - def path_parse(cls, model_root, path_string): + def path_parse(cls, model_root, path_string, sros, verify_keys): + assert isinstance(verify_keys, bool) if not isinstance(path_string, str): raise make_exception(pysros_err_path_should_be_string) if path_string == "/": - return cls(model_root) + return cls(model_root, sros) if not path_string.startswith('/'): if not path_string: @@ -387,6 +398,10 @@ def path_parse(cls, model_root, path_string): if path_string.endswith('/'): raise make_exception(pysros_err_invalid_identifier) + correct_char = lambda c: 32 <= ord(c) <= 127 or c in '\t\r\n' + if any(not correct_char(c) for c in path_string): + raise make_exception(pysros_err_invalid_parse_error) + iterator = iter(cls._tokenize(path_string)) def next_token(*accepts, err, **kwarg): @@ -412,7 +427,7 @@ def next_token(*accepts, err, **kwarg): return n raise make_exception(err, **kwarg) - res = cls(model_root) + res = cls(model_root, sros) missing_keys = set() elem = 'root' @@ -433,18 +448,23 @@ def next_token(*accepts, err, **kwarg): res.go_to_child(elem) missing_keys.update(res.current.local_keys) except Exception: - raise make_exception( - pysros_err_unknown_element, - element=elem) from None + try: + res.go_to_child(Identifier(res.current.name.prefix, elem)) + missing_keys.update(res.current.local_keys) + except Exception: + raise make_exception( + pysros_err_unknown_element, + element=elem) from None continue if i == "[": _, key_name = next_token( 'string', err=pysros_err_invalid_identifier ) - next_token('=', err=pysros_err_expected_equal_operator) - _, value = next_token( - 'string', err=pysros_err_invalid_identifier - ) + if verify_keys: + next_token('=', err=pysros_err_expected_equal_operator) + _, value = next_token( + 'string', err=pysros_err_invalid_identifier + ) next_token(']', err=pysros_err_expected_end_bracket) if key_name not in missing_keys: try: @@ -457,7 +477,8 @@ def next_token(*accepts, err, **kwarg): pysros_err_cannot_specify_non_key_leaf ) missing_keys.remove(key_name) - res.local_keys[key_name] = value + if verify_keys: + res.local_keys[key_name] = value return res @classmethod @@ -504,14 +525,27 @@ def next_token(t): def __str__(self): return "/" + "/".join(str(name.name) + "".join(f"[{key}={value}]" for key, value in keys.items()) for name, keys in zip(self.path, self.keys)) + def has_unique_child(self, child_name: str): + try: + self._get_child(child_name) + except SrosMgmtError: + return False + return True + def _get_child(self, child_name: Union[str, Identifier]): children = list(self.current.children) + res = None while children: child = children.pop() if child.name == child_name and child.data_def_stm in self._expected_dds and self._is_allowed(child): - return child + if res is None: + res = child + else: + raise make_exception(pysros_err_ambiguous_model_node) if child.data_def_stm in self._recursive_visited_dds: children.extend(child.children) + if res is not None: + return res raise make_exception( pysros_err_unknown_child, child_name=child_name, @@ -573,7 +607,57 @@ def _is_allowed(self, model): return False if not self.return_blocked_regions and model.is_region_blocked: return False - return any(i in model.name.prefix for i in ("nokia", "openconfig")) + if self._sros: + return any(i in model.name.prefix for i in ("nokia", "openconfig")) + return True + + def _construct_path_without_key_values(self): + path = "" + parent_module_name = None + for node in self.path: + current_module_name = node.name.prefix + current_node_name = f"{node.name.name}" + if node.data_def_stm == AModel.StatementType.list_: + current_node_name += "".join((f"[{key}]" for key in node.local_keys)) + if parent_module_name != current_module_name: + path += f"/{current_module_name}:{current_node_name}" + parent_module_name = current_module_name + else: + path += f"/{current_node_name}" + yield path + + def iterate_children(self, *, enter_fnc=None, action_io): + children = list(self.current.children) + while children: + child = children.pop() + if child.data_def_stm in self._expected_dds and self._is_allowed(child): + self.path.append(child) + self.keys.append(dict()) + if enter_fnc: + yield from enter_fnc(self, action_io) + yield from self.iterate_children(enter_fnc=enter_fnc, action_io=action_io) + self.path.pop() + self.keys.pop() + if child.data_def_stm in self._recursive_visited_dds: + children.extend(child.children) + + def _export_paths(self, args, action_io): + if self.current is None: + return + if not self.is_root: + if self.current.name.name in self.current.parent.local_keys: + return + if self.current.status: + if action_io in ("input", "output"): + if len(self.path) > 1: + if any((node.data_def_stm == Model.StatementType.action_ for node in self.path)): + yield from self._construct_path_without_key_values() + elif action_io == "action_only": + if len(self.path) > 1: + if self.path[-1].data_def_stm == Model.StatementType.action_: + yield from self._construct_path_without_key_values() + else: + yield from self._construct_path_without_key_values() class ActionInputFilteredDataModelWalker(FilteredDataModelWalker): @@ -603,3 +687,29 @@ class ActionOutputFilteredDataModelWalker(FilteredDataModelWalker): Model.StatementType.output_ ) +class JsonInstanceModelWalkerWithActionInput(ActionInputFilteredDataModelWalker): + def export_paths(self): + if self.current.data_def_stm in (AModel.StatementType.leaf_, AModel.StatementType.leaf_list_): + if any((node.data_def_stm == Model.StatementType.action_ for node in self.path)): + yield from self._construct_path_without_key_values() + yield from self.iterate_children(enter_fnc=self._export_paths, action_io="input") + + +class JsonInstanceModelWalkerWithActionOutput(ActionOutputFilteredDataModelWalker): + def export_paths(self): + if self.current.data_def_stm in (AModel.StatementType.leaf_, AModel.StatementType.leaf_list_): + if any((node.data_def_stm == Model.StatementType.action_ for node in self.path)): + yield from self._construct_path_without_key_values() + yield from self.iterate_children(enter_fnc=self._export_paths, action_io="output") + + +class JsonInstanceModelWalkerActionOnly(ActionInputFilteredDataModelWalker): + def export_paths(self): + yield from self.iterate_children(enter_fnc=self._export_paths, action_io="action_only") + +class JsonInstanceDataModelWalker(FilteredDataModelWalker): + def export_paths(self): + if self.current.data_def_stm in (AModel.StatementType.leaf_, AModel.StatementType.leaf_list_): + if all((node.data_def_stm != Model.StatementType.action_ for node in self.path)): + yield from self._construct_path_without_key_values() + yield from self.iterate_children(enter_fnc=self._export_paths, action_io=None) diff --git a/pysros/path_utils.py b/pysros/path_utils.py new file mode 100644 index 0000000..0d68b8c --- /dev/null +++ b/pysros/path_utils.py @@ -0,0 +1,448 @@ +from .model_path import ModelPath +from .identifier import Identifier +from .singleton import _Singleton +from .model_walker import ModelWalker + +import re +import functools +import contextlib +from typing import Optional, List, Iterable, Tuple, Dict, Union + +class LevelUp(metaclass=_Singleton): + def __str__(self): + return f"{self.__class__.__name__}()" + + def __hash__(self): + return id(self) + + def __eq__(self, other): + return self is other + + +class _Path: + def __init__(self): + self._path: List[Identifier] = [] + self._keys: List[Dict[Identifier, Union[str, _Path]]] = [] + self.default_module = None + + def append(self, i: Identifier): + self._path.append(i) + self._keys.append({}) + + def append_level_up(self): + self.append(LevelUp()) + + def move_walker(self, w: ModelWalker, default_module=None): + for name, keys in zip(self._path, self._keys): + if name is LevelUp(): + w.go_to_parent() + elif name.is_lazy_bound(): + w.go_to_child(Identifier(default_module or w.current.prefix, name.name)) + else: + w.go_to_child(name) + if not w.is_root: + w.local_keys.update(keys) + + @property + def top_keys(self): + return self._keys[-1] + + +class AbsolutePath(_Path): + def move_walker(self, w: ModelWalker, *args, **kwargs): + while not w.is_root: + w.go_to_parent() + super().move_walker(w, *args, **kwargs) + + +class RelativePath(_Path): + pass + +""" +leafref: +=================================================================== + leafref-specification = + ;; these stmts can appear in any order + path-stmt + [require-instance-stmt] + + path-stmt = path-keyword sep path-arg-str stmtend + + path-arg-str = < a string that matches the rule > + < path-arg > +=================================================================== + +augment: +=================================================================== + augment-arg-str = < a string that matches the rule > + < augment-arg > + + augment-arg = absolute-schema-nodeid +=================================================================== + +deviation: +=================================================================== + deviation-arg-str = < a string that matches the rule > + < deviation-arg > + + deviation-arg = absolute-schema-nodeid +=================================================================== + +uses: +=================================================================== + refine-arg-str = < a string that matches the rule > + < refine-arg > + + refine-arg = descendant-schema-nodeid + + uses-augment-arg-str = < a string that matches the rule > + < uses-augment-arg > + + uses-augment-arg = descendant-schema-nodeid +=================================================================== + +json-instance-path: +=================================================================== + An "instance-identifier" value is encoded as a string that is + analogical to the lexical representation in XML encoding; see + Section 9.13.2 in [RFC7950]. However, the encoding of namespaces in + instance-identifier values follows the rules stated in Section 4, + namely: + + o The leftmost (top-level) data node name is always in the + namespace-qualified form. + + o Any subsequent data node name is in the namespace-qualified form + if the node is defined in a module other than its parent node, and + the simple form is used otherwise. This rule also holds for node + names appearing in predicates. +=================================================================== +""" + +""" +path-arg: + leafref + +absolute-schema-nodeid: + augment + deviation + +descendant-schema-nodeid: + uses-refine + uses-augment + +json-instance-path: + pysros +""" + +""" + path-arg = absolute-path / relative-path + + absolute-path = 1*("/" (node-identifier *path-predicate)) + + relative-path = 1*("../") descendant-path + + descendant-path = node-identifier + [*path-predicate absolute-path] + + path-predicate = "[" *WSP path-equality-expr *WSP "]" + + path-equality-expr = node-identifier *WSP "=" *WSP path-key-expr + + path-key-expr = current-function-invocation *WSP "/" *WSP + rel-path-keyexpr + + rel-path-keyexpr = 1*(".." *WSP "/" *WSP) + *(node-identifier *WSP "/" *WSP) + node-identifier + + node-identifier = [prefix ":"] identifier + + current-function-invocation = current-keyword *WSP "(" *WSP ")" + + current-keyword = %s"current" + +absolute-path: + /a + /a/b + /prefix:a + /a/b[key=current()/../c] + /a/b[key=current()/../c][key2=current()/../c][key3=current()/../c] + /a/b[prefix:key=current()/../c] + /a/b[ prefix:key = current ( ) / .. / c] + #not in grammar, but probably still correct + / a / b + +relative-path: + ../a + ../a/b + ../../../a/b + ../prefix:a + ../a/b[key=current()/../c] + ../a/b[key=current()/../c][key2=current()/../c][key3=current()/../c] + ../a/b[prefix:key=current()/../c] + .. / a / b [ prefix:key = current ( ) / .. / c] + +""" + +""" + ;; Schema Node Identifiers + + schema-nodeid = absolute-schema-nodeid / + descendant-schema-nodeid + + absolute-schema-nodeid = 1*("/" node-identifier) + + descendant-schema-nodeid = + node-identifier + [absolute-schema-nodeid] + + node-identifier = [prefix ":"] identifier + +absolute-schema-nodeid: + /a + /a/b/c + /prefix:a + +descendant-schema-nodeid: + a + a/b/c + prefix:a +""" + +""" + ;; Instance Identifiers + + instance-identifier = 1*("/" (node-identifier + [1*key-predicate / + leaf-list-predicate / + pos])) + + key-predicate = "[" *WSP key-predicate-expr *WSP "]" + + key-predicate-expr = node-identifier *WSP "=" *WSP quoted-string + + leaf-list-predicate = "[" *WSP leaf-list-predicate-expr *WSP "]" + + leaf-list-predicate-expr = "." *WSP "=" *WSP quoted-string + + pos = "[" *WSP positive-integer-value *WSP "]" + + quoted-string = (DQUOTE string DQUOTE) / (SQUOTE string SQUOTE) + + node-identifier = [prefix ":"] identifier + +Instance Identifiers: + /ex:system/ex:services/ex:ssh + /ex:system/ex:services/ex:ssh/ex:port + /ex:system/ex:user[ex:name='fred'] + /ex:system/ex:user[ex:name='fred']/ex:type + /ex:system/ex:server[ex:ip='192.0.2.1'][ex:port='80'] + /ex:system/ex:service[ex:name='foo'][ex:enabled=''] + /ex:system/ex:services/ex:ssh/ex:cipher[.='blowfish-cbc'] + /ex:stats/ex:port[3] + / ex:stats / ex:port [ 3 ] + +Json instance identifier: + /mod:system/services/ssh + /mod:system/user[name='fred'] + /mod:system/services/mod2:ssh + +Pysros instance identifier: + /system/services/ssh + /system/user[name='fred'] + /system/services/ssh +""" + +SLASH_RE = r'\s*(?P[/])\s*' +BRACKET_OPEN_RE = r'\s*(?P\[)\s*' +BRACKET_CLOSE_RE = r'\s*(?P\])\s*' +EQ_RE = r'\s*(?P=)\s*' +_NAME_RE = r"""[a-zA-Z_][a-zA-Z0-9_.-]*""" +NAME_RE = fr"""(?P{_NAME_RE})""" +IDENTIFIER_RE = fr"""(?P{_NAME_RE}:{_NAME_RE})""" +CURRENT_NODE_RE = r"(?P[.])" +PARENT_NODE_RE = r"(?P[.][.])" +QUOTED_STR_RE = r"""(?P(?:["](?:[^\\"]|[\\][\\nt"])*["])|(?:['](?:[^'])*[']))""" +CURRENT_FNC_RE = r"\s*(?Pcurrent\s*[(]\s*[)])\s*" + + +class Tokenizer: + def __init__(self, s: str): + self.s = s + self.pos = 0 + self.current_match = None + + def consume(self, regex: Iterable[str]): + if self.finished: + return False + r = re.compile(fr"(?:{'|'.join(regex)})") + self.current_match = r.match(self.s, self.pos) + if self.current_match is None: + return False + self.pos = self.current_match.end() + return True + + def match(self, regex: Iterable[str]): + if self.finished: + return False + r = re.compile(fr"(?:{'|'.join(regex)})") + match = r.match(self.s, self.pos) + if match is None: + return False + return True + + @property + def kind(self): + return self.current_match.lastgroup + + @property + def token(self): + return self.current_match.group(self.current_match.lastindex) + + @property + def finished(self): + return self.pos >= len(self.s) + + +_ = (SLASH_RE, BRACKET_OPEN_RE, BRACKET_CLOSE_RE, EQ_RE, CURRENT_FNC_RE, PARENT_NODE_RE, CURRENT_NODE_RE, IDENTIFIER_RE, NAME_RE, QUOTED_STR_RE, ) + +""" + absolute_path + path_segment+ + + relative_path + (PARENT_NODE / SLASH) * / ( node_segment / absolute_path ? ) ? + + node + (NAME|IDENTIFIER) + path_segment + SLASH / node_segment + node_segment + node / [ key_segment* ] + node /* for schema node id */ + key_segment + BRACKET_OPEN / node / EQ / path_key_val / BRACKET_CLOSE + path_key_val + CURRENT_FNC / SLASH /relative_path + QUOTED_STR /* for json instance path */ + +""" + + +class Parser: + def __init__(self, s: str, *, key_allowed, absolute_path_allowed, relative_path_allowed, prefix_resolver=None, key_expect_path=None): + assert key_expect_path is not None or not key_allowed + self.tokenizer = Tokenizer(s) + self.key_allowed = key_allowed + self.absolute_path_allowed = absolute_path_allowed + self.relative_path_allowed = relative_path_allowed + self.key_expect_path = key_expect_path + if prefix_resolver is not None: + self.prefix_identifier_resolver = prefix_resolver + else: + self.prefix_identifier_resolver = lambda x: x + + def parse(self): + if self.peek_slash(): + self.path = AbsolutePath() + assert self.absolute_path_allowed + else: + self.path = RelativePath() + assert self.relative_path_allowed + while self.try_levelup(): + if not self.tokenizer.finished: + self.slash() + assert not self.tokenizer.finished + self.path.append_level_up() + + if not self.tokenizer.finished: + component = self.identifier_with_keys() + self.path.append(component[0]) + for name, val in component[1]: + self.path.top_keys[name] = val + + while not self.tokenizer.finished: + component = self.path_component() + self.path.append(component[0]) + for name, val in component[1]: + self.path.top_keys[name] = val + + def try_levelup(self): + return self.tokenizer.consume((PARENT_NODE_RE, )) + + def peek_slash(self): + return self.tokenizer.match((SLASH_RE, )) + + def try_slash(self): + return self.tokenizer.consume((SLASH_RE, )) + + def peek_bracket_open(self): + return self.tokenizer.match((BRACKET_OPEN_RE, )) + + def slash(self): + assert self.tokenizer.consume((SLASH_RE, )) + assert self.tokenizer.kind == "SLASH" + + def path_component(self) -> Tuple[Identifier, List[Tuple[Identifier, str]]]: + self.slash() + return self.identifier_with_keys() + + def key_segment(self) -> Tuple[Identifier, str]: + assert self.tokenizer.consume((BRACKET_OPEN_RE, )) + name = self.identifier() + assert self.tokenizer.consume((EQ_RE, )) + if self.key_expect_path: + val = self.leaflist_pathj_key() + else: + val = self.quoted_str() + assert self.tokenizer.consume((BRACKET_CLOSE_RE, )) + return (name, val, ) + + def leaflist_pathj_key(self): + res = RelativePath() + self.current_fnc() + + while self.try_slash(): + if self.try_levelup(): + res.append_level_up() + else: + res.append(self.identifier()) + + def identifier_with_keys(self) -> Tuple[Identifier, List[Tuple[Identifier, str]]]: + identifier = self.identifier() + keys = [] + if self.key_allowed: + while self.peek_bracket_open(): + keys.append(self.key_segment()) + return (identifier, keys, ) + + def identifier(self) -> Identifier: + assert self.tokenizer.consume((IDENTIFIER_RE, NAME_RE, )) + name = self.tokenizer.token + if self.tokenizer.kind == "IDENTIFIER": + prefix, name = name.split(":") + return Identifier(self.prefix_identifier_resolver(prefix), name) + return Identifier.lazy_bound(name) + + def quoted_str(self): + assert self.tokenizer.consume((QUOTED_STR_RE, )) + return self.tokenizer.token[1:-1] + + def current_fnc(self): + assert self.tokenizer.consume((CURRENT_FNC_RE, )) + assert self.tokenizer.kind == "CURRENT_FNC" + + +def parse(w: ModelWalker, *args, **kwargs): + get_path(*args, **kwargs).move_walker(w) + + +@functools.wraps(Parser) +def get_path(*args, default_module=None, **kwargs): + parser = Parser(*args, **kwargs) + parser.parse() + res = parser.path + if default_module: + res.default_module = default_module + return res diff --git a/pysros/request_data.py b/pysros/request_data.py index 90fcd1d..134002c 100644 --- a/pysros/request_data.py +++ b/pysros/request_data.py @@ -43,6 +43,8 @@ _get_tag_name = lambda x: etree.QName(x).localname _get_tag_ns = lambda x: etree.QName(x).namespace +_get_tag_prefix = lambda x, rev_ns_map: rev_ns_map.get(_get_tag_ns(x), "") +_get_qual_tag_name = lambda x, rev_ns_map: f"""{rev_ns_map.get(_get_tag_ns(x), "")}:{_get_tag_name(x)}""" _text_in_tag_tail = lambda x: x.tail and x.tail.strip() _text_in_tag_text = lambda x: x.text and x.text.strip() _create_root_ele = lambda: etree.Element("dummy-root", nsmap={"nokia-attr": COMMON_NAMESPACES["attrs"]}) @@ -107,7 +109,8 @@ class _Action(Enum): basic = auto() convert = auto() - def __init__(self, root: Model, annotations: Dict[str, Model], annotations_no_module: Dict[str, Model], ns_map: dict, ns_map_rev: dict, *, action: _Action = _Action.basic, walker=FilteredDataModelWalker): + def __init__(self, root: Model, annotations: Dict[str, Model], annotations_no_module: Dict[str, Model], ns_map: dict, ns_map_rev: dict, sros: bool, *, action: _Action = _Action.basic, walker=FilteredDataModelWalker): + self._sros = sros self._root = root self._Walker = walker self._data = _ListStorage(root, self) @@ -123,28 +126,28 @@ def process_path(self, path: Union[str, FilteredDataModelWalker], *, strict=Fals .. Reviewed by TechComms 20210712 """ - walker = path if isinstance(path, ModelWalker) else self._Walker.user_path_parse(self._root, path, accept_root=(self._action == self._Action.convert)) + walker = path if isinstance(path, ModelWalker) else self._Walker.user_path_parse(self._root, path, self._sros, accept_root=(self._action == self._Action.convert)) current = _ASetter.create_setter(self._data, self) for elem, keys in zip(walker.path, walker.keys): if not isinstance(current, _MoSetter): raise make_exception(pysros_err_missing_keys, element=current._walker.get_name()) if elem.data_def_stm not in FIELD_STATEMENT_TYPES: - if strict and not current.child_mos.is_created(elem.name.name): + if strict and not current.child_mos.is_created(elem.name): raise make_exception(pysros_err_no_data_found) - current = current.child_mos.get_or_create(elem.name.name) + current = current.child_mos.get_or_create(elem.name) if keys: if strict and not current.entry_exists_nocheck(keys): raise make_exception(pysros_err_no_data_found) current = current.entry_nocheck(keys) - elif current.keys.can_contains(elem.name.name): - current = current.keys.get(elem.name.name) + elif current.keys.can_contains(elem.name): + current = current.keys.get(elem.name) else: - if not current.fields.contains(elem.name.name): + if not current.fields.contains(elem.name): if strict: raise make_exception(pysros_err_no_data_found) else: - current.fields.set_getValue(elem.name.name) - current = current.fields.get(elem.name.name) + current.fields.set_getValue(elem.name) + current = current.fields.get(elem.name) return current @@ -254,7 +257,7 @@ def to_model(self, *, key_filter={}): @property def _walker(self): - return self.rd._Walker(self._model) + return self.rd._Walker(self._model, self.rd._sros) def _resolve_xml_name(self, model, ns_map): if model.name.prefix in ns_map: @@ -348,6 +351,7 @@ def debug_dump(self, indent=0): print(f"{' '*(indent+1)*(not self._walker.is_local_key)}{self._model.name} = {self._value}{self._debug_dump_metadata()}", end="\n"*(not self._walker.is_local_key)) def sanity_check(self): + name = self._walker.get_type().json_name() expected_type = { "string": str, "empty": Empty.__class__, @@ -366,7 +370,7 @@ def sanity_check(self): "uint16": int, "uint32": int, "uint64": int, - }[self._walker.get_type().json_name()] + }[name] if not isinstance(expected_type, tuple): expected_type = (expected_type, ) val = self._value @@ -376,7 +380,8 @@ def sanity_check(self): if self._walker.is_leaflist and not isinstance(self._value, (FieldValuePlaceholder, GetValuePlaceholder, )) and self.metadata != None: assert not len(self.metadata) or len(self.metadata) == len(self._value) - assert all(isinstance(v, (*expected_type, FieldValuePlaceholder, GetValuePlaceholder, )) for v in val) + if self.rd._action == RequestData._Action.convert and name in INTEGRAL_LEAF_TYPE and val and val[0] is not None: + assert all(isinstance(v, (*expected_type, FieldValuePlaceholder, GetValuePlaceholder, )) for v in val) def has_metadata(self): if self._walker.is_leaflist: @@ -394,12 +399,12 @@ def __init__(self, model: Model, local_keys, rd: RequestData): super().__init__(rd) self._model = model self._local_keys = self._prepare_keys(local_keys) - self._child = {} + self._child: Dict[Identifier, _AStorage] = {} self._operation = "" def _prepare_keys(self, keys): w = self._walker - return {k: _FieldStorage(w.get_child(k).current, self.rd, value=copy.deepcopy(v)) for k, v in keys.items()} + return {w.get_child(k).get_name(): _FieldStorage(w.get_child(k).current, self.rd, value=copy.deepcopy(v)) for k, v in keys.items()} def _to_xml(self, ns_map, root): root_attr = {} @@ -412,7 +417,7 @@ def _to_xml(self, ns_map, root): v._to_xml(ns_map, root) def to_xml(self, ns_map, root): - for k, v in self._child.items(): + for v in self._child.values(): v._to_xml(ns_map, root) def _leaf_placeholder_to_xml(self, value, root, walker, ns_map): @@ -422,31 +427,25 @@ def to_model(self, *, key_filter={}): data = {} is_selection_filter = {} in key_filter.values() for k, v in self._local_keys.items(): - if is_selection_filter and k not in key_filter: + if is_selection_filter and k.model_string not in key_filter and k.name not in key_filter: continue - data[k] = v.to_model(key_filter=(key_filter.get(k, {}))) + data[k.name] = v.to_model(key_filter=(key_filter.get(k, {}))) for k, v in self._child.items(): key_proxy = DictionaryKeysProxy(self.rd._unwrap(key_filter)) - if is_selection_filter and self._walker.get_child(k).current.name not in key_proxy: + if is_selection_filter and k.model_string not in key_proxy and k.name not in key_proxy: continue - data[k] = v.to_model(key_filter=(key_filter.get(k, {}))) - if not data[k] and (not isinstance(data[k], Wrapper) or not len(data[k].annotations)) and not self._walker.get_child(k).has_explicit_presence(): - del data[k] + name = k.name if self._walker.has_unique_child(k.name) else k.model_string + data[name] = v.to_model(key_filter=(key_filter.get(k, {}))) + if not data[name] and (not isinstance(data[name], Wrapper) or not len(data[name].annotations)) and not self._walker.get_child(name).has_explicit_presence(): + del data[name] if self._walker.is_root: return data if self._model.data_def_stm == Model.StatementType.action_: return Action._with_model(data, self._model, annotations=self._metadata_to_model(self.metadata)) return Container._with_model(data, self._model, annotations=self._metadata_to_model(self.metadata)) - def keys_equal(self, keys): - """Compare if keys are equal. Keys are expected as dict(name->value). - - .. Reviewed by TechComms 20210712 - """ - return self._local_keys == keys - def get_keys_flat(self): - keys = tuple(self._local_keys[key]._value for key in self._model.local_keys) + keys = tuple(self._local_keys[Identifier(self._walker.current.prefix, key)]._value for key in self._model.local_keys) if len(keys) == 1: keys = keys[0] return keys @@ -471,11 +470,12 @@ def debug_dump(self, indent=0): if isinstance(v, _AStorage): v.debug_dump(indent + 1) else: - print(f"{' '*(indent + 1)}{k} = {v}") + print(f"{' '*(indent + 1)}{k.debug_string} = {v}") print((" " * indent) + "}") def sanity_check(self): - for v in self._child.values(): + for k, v in self._child.items(): + assert isinstance(k, Identifier) v.sanity_check() @@ -594,8 +594,8 @@ def _unwrap(self, val): class RdJsonEncoder(json.JSONEncoder): def add_ns(self, d, is_root, walker: ModelWalker): if is_root: - return {walker.get_child(k).get_name().model_string: (v if not isinstance(v, _FieldStorage) else v._value) for k, v in d.items()} - return {walker.get_child(k).get_name().model_string if walker.get_name().prefix != walker.get_child(k).get_name().prefix else k: (v if not isinstance(v, _FieldStorage) else v._value) for k, v in d.items()} + return {k.model_string: (v if not isinstance(v, _FieldStorage) else v._value) for k, v in d.items()} + return {k.model_string if walker.get_name().prefix != k.prefix else k.name: (v if not isinstance(v, _FieldStorage) else v._value) for k, v in d.items()} def stringify(self, x, dds): return [str(v) for v in x] if dds == Model.StatementType.leaf_list_ else str(x) @@ -608,6 +608,8 @@ def convert(self, o: _FieldStorage): def convert_child(self, o, k, v): if v is None: return "" + if ":" not in k: + k = Identifier(o._walker.get_name().prefix, k) return self.stringify(v, o._walker.get_child_dds(k)) if o._walker.get_child_dds(k) in FIELD_STATEMENT_TYPES and o._walker.get_child_type(k).json_name() in ("int64", "uint64") else v def convert_metadata(self, metadata): @@ -639,7 +641,7 @@ def default(self, o): metadata_sources = itertools.chain(metadata_sources, ((self.force_key, o._local_keys[self.force_key]), )) res.update(self.add_ns(o._child, is_root, o._walker)) - res = {k: self.convert_child(o, k, v) for k, v in res.items()} + res = {k.name if isinstance(k, Identifier) else k: self.convert_child(o, k, v) for k, v in res.items()} metadata = {k: self.convert_metadata(v.metadata) for k, v in metadata_sources if isinstance(v, _FieldStorage) and v.has_metadata()} metadata = self.add_ns(metadata, is_root, o._walker) res.update((f"@{k}", v) for k, v in metadata.items()) @@ -670,7 +672,7 @@ def __init__(self, storage: "_AStorage", rd: RequestData): @property def _walker(self): - return self.rd._Walker(self._storage._model) + return self.rd._Walker(self._storage._model, self.rd._sros) @staticmethod def create_setter(storage: "_AStorage", rd: RequestData): @@ -791,7 +793,8 @@ def _find_metadata_model_by_annotation(self, ann: Annotation) -> Model: return m _raise_unknown_attribute(ann) - def _find_metadata_model_by_name(self, name) -> Model: + def _find_metadata_model_by_name(self, name: str) -> Model: + #part of public API - metadata will be most likely set as string id = Identifier.from_model_string(name) if id.is_lazy_bound(): candidates = self.rd._modeled_metadata_no_module.get(name) @@ -852,7 +855,7 @@ def to_json(self, pprint=True): members = { "root": self._storage, "root_setter": self, - "force_key": getattr(self, "_key_name", None), + "force_key": self._walker.get_name() if isinstance(self, _KeySetter) else None, "metadata_models": self.rd._modeled_metadata, } encoder = type("RdJsonEncoderSpecialization", (RdJsonEncoder, ), members) @@ -953,9 +956,8 @@ class has its own implementation of to_model method. .. Reviewed by TechComms 20210713 """ - def __init__(self, storage: "_FieldStorage", leaf_name: str, rd: RequestData): + def __init__(self, storage: "_FieldStorage", rd: RequestData): super().__init__(storage, rd) - self._leaf_name = leaf_name def set(self, value): if not self._storage.metadata: @@ -965,7 +967,7 @@ def set(self, value): metadata=self._storage.metadata): raise make_exception( pysros_err_incorrect_leaf_value, - leaf_name=self._leaf_name + leaf_name=self._walker.get_name().name ) else: val = self._as_storage_type(value) @@ -1018,6 +1020,12 @@ def _set_as_json(self, value, is_root=False): self._set_metadata(json_metadata=metadata[1]) def set_or_append_as_xml(self, value_element): + if _get_tag_name(value_element) != self._walker.get_name().name: + raise make_exception( + pysros_err_unknown_child, + child_name=_get_tag_name(value_element), + path=self._walker._get_path() + ) is_leaflist = self._walker.get_dds() == Model.StatementType.leaf_list_ self._set_metadata(xml_metadata=_metadata_from_xml(value_element, self.rd._ns_map_rev), nsmap=value_element.nsmap) if _text_in_tag_tail(value_element) or (is_leaflist and _text_in_tag_text(value_element.getparent())): @@ -1050,9 +1058,8 @@ class _KeySetter(_ASetter): """Interface for keys. Most of this class are stub methods to provide setter interface to keys.""" - def __init__(self, storage: "_MoStorage", key_name: str, rd: RequestData): + def __init__(self, storage: "_MoStorage", key_name: Identifier, rd: RequestData): super().__init__(storage._local_keys[key_name], rd) - self._key_name = key_name def set(self, value): if not self._storage.metadata: @@ -1060,9 +1067,9 @@ def set(self, value): value = self.rd._unwrap(value) if not self._walker.check_field_value(value, is_convert=self.rd._action == RequestData._Action.convert, metadata=self._storage.metadata): - raise make_exception(pysros_err_incorrect_leaf_value, leaf_name=self._key_name) + raise make_exception(pysros_err_incorrect_leaf_value, leaf_name=self._walker.get_name().name) elif self._as_storage_type(value) != self._storage._value: - raise make_exception(pysros_err_key_val_mismatch, key_name=self._key_name) + raise make_exception(pysros_err_key_val_mismatch, key_name=self._walker.get_name().name) def set_getValue(self): pass @@ -1089,12 +1096,18 @@ def merge(self): def set_as_xml(self, value): if _text_in_tag_tail(value): - _raise_invalid_text_exception(value_element) + _raise_invalid_text_exception(value) if not len(value): raise make_exception(pysros_err_malformed_xml) for v in value: if _text_in_tag_tail(v): _raise_invalid_text_exception(v) + if _get_tag_name(v) != self._walker.get_name().name: + raise make_exception( + pysros_err_unknown_child, + child_name=_get_tag_name(v), + path=self._walker._get_path() + ) self._set_metadata(xml_metadata=_metadata_from_xml(v, self.rd._ns_map_rev), nsmap=value.nsmap) self.set(self._walker.as_model_type(v.text or "")) @@ -1167,13 +1180,13 @@ def _tuple_to_dict(self, t): return {k: v for k, v in zip(self._walker.get_local_key_names(), (t if isinstance(t, tuple) else (t, )))} def _extract_keys(self, entry): - return {k: self._as_storage_type(v, child_name=k) for k, v in entry.items() if k in self._walker.get_local_key_names()} + return {self._walker.get_child(k).get_name(): self._as_storage_type(v, child_name=k) for k, v in entry.items() if k in self._walker.get_local_key_names()} def _convert_keys_to_model(self, entry): try: def unwrap(v): return v.data if isinstance(v, Wrapper) else v - val = {k: (GetValuePlaceholder() if unwrap(v) in (GetValuePlaceholder(), {}) else self._walker.as_child_model_type(k, unwrap(v))) for k, v in entry.items() if v is not FieldValuePlaceholder()} + val = {self._walker.get_child(k).get_name(): (GetValuePlaceholder() if unwrap(v) in (GetValuePlaceholder(), {}) else self._walker.as_child_model_type(k, unwrap(v))) for k, v in entry.items() if v is not FieldValuePlaceholder()} return val except: raise make_exception(pysros_err_invalid_key_in_path) from None @@ -1233,10 +1246,12 @@ def entry_exists_nocheck(self, value): def entry_xml(self, value): keys = {} for e in value: - if _get_tag_name(e) in self._walker.get_local_key_names(): + name = _get_tag_name(e) + if name in self._walker.get_local_key_names(): if _text_in_tag_tail(e): _raise_invalid_text_exception(e) - keys[_get_tag_name(e)] = e.text or "" + keys[name] = e.text or "" + keys[name] = self._fix_xml_identityref_prefix(keys[name], self._walker.get_child_type(Identifier(self.rd._ns_map_rev[self._walker.current.namespace], name)), nsmap=e.nsmap) if set(keys.keys()) != set(self._walker.get_local_key_names()): raise make_exception( pysros_err_malformed_keys, @@ -1299,7 +1314,7 @@ def __init__(self, setter: "_MoSetter"): @property def _walker(self): - return self._setter.rd._Walker(self._setter._storage._model) + return self._setter.rd._Walker(self._setter._storage._model, self._setter.rd._sros) class _Keys(_AChild): """Interface for retrieving and setting keys. @@ -1307,33 +1322,35 @@ class _Keys(_AChild): .. Reviewed by TechComms 20210712 """ - def set(self, name, value): - name = Identifier.from_model_string(name).name + def set(self, name :str, value): + name = self._walker.get_child(name).get_name() self.get(name).set(value) - def set_as_json(self, name, value): - name = Identifier.from_model_string(name).name - self.get(name)._set_as_json({name: value}) + def set_as_json(self, name :Identifier, value): + self.get(name)._set_as_json({name.name: value}) - def set_getValue(self, name): + def set_getValue(self, name: Union[str, Identifier]): + name = self._walker.get_child(name).get_name() if not self.contains(name): self._setter._storage._local_keys[name] = _FieldStorage(self._setter._walker.get_child(name).current, self._setter.rd, value=FieldValuePlaceholder()) self.get(name).set_getValue() - def set_placeholder(self, name): + def set_placeholder(self, name: str): + name = self._walker.get_child(name).get_name() if not self.contains(name): self._setter._storage._local_keys[name] = _FieldStorage(self._setter._walker.get_child(name).current, self._setter.rd, value=FieldValuePlaceholder()) self.get(name).set_placeholder() - def get(self, name): + def get(self, name: Union[str, Identifier]): if not self.can_contains(name): raise make_exception(pysros_err_unknown_child, child_name=name, path=self._walker._get_path()) + name = self._walker.get_child(name).get_name() return _KeySetter(self._setter._storage, name, self._setter.rd) - def contains(self, name): + def contains(self, name: Identifier): return name in self._setter._storage._local_keys - def can_contains(self, name): + def can_contains(self, name: Union[str, Identifier]): return self._walker.has_local_key_named(name) class _Fields(_AChild): @@ -1342,41 +1359,43 @@ class _Fields(_AChild): .. Reviewed by TechComms 20210712 """ - def set(self, name, value): - name = Identifier.from_model_string(name).name + def set(self, name: str, value): + name = self._walker.get_child(name).get_name() self.get(name).set(value) - def set_as_json(self, name, value): - name = Identifier.from_model_string(name).name - self.get(name)._set_as_json({name: value}) + def set_as_json(self, name :Identifier, value): + self.get(name)._set_as_json({name.name: value}) def get_as_json(self, name): name = Identifier.from_model_string(name).name return self.get(name) - def set_getValue(self, name): + def set_getValue(self, name: Union[str, Identifier]): + name = self._walker.get_child(name).get_name() self.get(name).set_getValue() - def get(self, name): + def get(self, name: Union[str, Identifier]): + name = self._walker.get_child(name).get_name() if not self.can_contains(name): raise make_exception(pysros_err_unknown_child, child_name=name, path=self._walker._get_path()) + name = self._walker.get_child(name).get_name() if not self.contains(name): self._setter._storage._child[name] = _FieldStorage(self._setter._walker.get_child(name).current, self._setter.rd) - return _LeafSetter(self._setter._storage._child[name], name, self._setter.rd) + return _LeafSetter(self._setter._storage._child[name], self._setter.rd) - def get_nonexisting(self, name): + def get_nonexisting(self, name: Identifier): if not self.can_contains(name): raise make_exception(pysros_err_unknown_child, child_name=name, path=self._walker._get_path()) if self.contains(name) and self._walker.get_child_dds(name) != Model.StatementType.leaf_list_: raise make_exception(pysros_err_multiple_occurences_of_node) if not self.contains(name): self._setter._storage._child[name] = _FieldStorage(self._setter._walker.get_child(name).current, self._setter.rd) - return _LeafSetter(self._setter._storage._child[name], name, self._setter.rd) + return _LeafSetter(self._setter._storage._child[name], self._setter.rd) - def contains(self, name): + def contains(self, name: Identifier): return name in self._setter._storage._child - def can_contains(self, name): + def can_contains(self, name: Union[str, Identifier]): return self._walker.has_field_named(name) class _ChildMos(_AChild): @@ -1384,13 +1403,15 @@ class _ChildMos(_AChild): .. Reviewed by TechComms 20210712 """ - def set(self, name, value): - self.get_or_create(Identifier.from_model_string(name).name).set(value) + def set(self, name: str, value): + name = self._walker.get_child(name).get_name() + self.get_or_create(name).set(value) - def set_as_json(self, name, value): - self.get_or_create(Identifier.from_model_string(name).name)._set_as_json(value) + def set_as_json(self, name: Identifier, value): + self.get_or_create(name)._set_as_json(value) - def get_or_create(self, name): + def get_or_create(self, name: Union[str, Identifier]): + name = self._walker.get_child(name).get_name() if self._walker.get_child_dds(name) in MO_STATEMENT_TYPES: if not self.is_created(name): self._setter._storage._child[name] = _ListStorage(self._setter._walker.get_child(name).current, self._setter.rd) @@ -1398,7 +1419,8 @@ def get_or_create(self, name): else: raise KeyError(name) - def get(self, name): + def get(self, name: Union[str, Identifier]): + name = self._walker.get_child(name).get_name() if self._walker.get_child_dds(name) in MO_STATEMENT_TYPES: if self.is_created(name): if not self._walker.get_child_dds(name) == Model.StatementType.list_: @@ -1411,7 +1433,7 @@ def get(self, name): else: raise KeyError(name) - def is_created(self, name): + def is_created(self, name: Identifier): return ( self._walker.get_child_dds(name) in MO_STATEMENT_TYPES and name in self._setter._storage._child @@ -1448,11 +1470,11 @@ def set(self, value, *, wrapped=None): pysros_err_unknown_dds, dds=walker.get_child_dds(k) ) - name = Identifier.from_model_string(k).name + name = self._walker.get_child(k).get_name() if name in children_to_set: raise make_exception( pysros_err_duplicate_found, - duplicate=name + duplicate=name.name ) children_to_set.add(name) @@ -1501,19 +1523,28 @@ def set_as_xml(self, value): if _text_in_tag_tail(value): _raise_invalid_text_exception(value) for e in value: - if not walker.has_child(_get_tag_name(e)) or not self.rd.xml_tag_has_correct_ns(e, walker): + if _get_tag_ns(e) is None: + if self._walker.has_unique_child(_get_tag_name(e)): + name = self._walker.get_child(_get_tag_name(e)).get_name() + elif self._walker.current.name.is_builtin(): + raise make_exception(pysros_err_unknown_child, child_name=_get_tag_name(e), path=walker._get_path()) + else: + name = Identifier(self.rd._ns_map_rev[self._walker.current.namespace], _get_tag_name(e)) + else: + name = Identifier(_get_tag_prefix(e, self.rd._ns_map_rev), _get_tag_name(e)) + if not walker.has_child(name): raise make_exception(pysros_err_unknown_child, child_name=_get_tag_name(e), path=walker._get_path()) - if walker.is_region_blocked_in_child(_get_tag_name(e)): + if walker.is_region_blocked_in_child(name): continue else: - if walker.get_child_dds(_get_tag_name(e)) in FIELD_STATEMENT_TYPES: + if walker.get_child_dds(name) in FIELD_STATEMENT_TYPES: if _get_tag_name(e) not in walker.get_local_key_names(): - self.fields.get_nonexisting(_get_tag_name(e)).set_or_append_as_xml(e) + self.fields.get_nonexisting(name).set_or_append_as_xml(e) elif self.rd._action == RequestData._Action.convert: xml = to_ele(f"<{_get_tag_name(value)}>{etree.tostring(e, encoding='unicode')}") - _KeySetter(self._storage, _get_tag_name(e), self.rd).set_as_xml(xml) - elif walker.get_child_dds(_get_tag_name(e)) in MO_STATEMENT_TYPES: - self.child_mos.get(_get_tag_name(e)).entry_xml(e).set_as_xml(e) + _KeySetter(self._storage, name, self.rd).set_as_xml(xml) + elif walker.get_child_dds(name) in MO_STATEMENT_TYPES: + self.child_mos.get(name).entry_xml(e).set_as_xml(e) def _set_as_json(self, value, is_root=False): if not isinstance(value, dict): @@ -1536,8 +1567,16 @@ def check_duplicates(name): continue elif k.startswith("@"): k = k[1:] - self._walker.get_child(k) - name = Identifier.from_model_string(k).name + try: + name = self._walker.get_child(k).get_name() + except SrosMgmtError as e: + try: + if ":" not in k: + name = self._walker.get_child(Identifier(self._walker.get_name().prefix, k)).get_name() + else: + raise e from None + except: + raise e from None if self.keys.can_contains(name): self.keys.get(name)._set_metadata(json_metadata=v) @@ -1546,22 +1585,30 @@ def check_duplicates(name): if not self.fields.can_contains(name): raise make_exception( pysros_err_unknown_field, - field_name=name, path=self._walker._get_path() + field_name=name.name, path=self._walker._get_path() ) if k not in value: raise make_exception( pysros_err_annotation_without_value, - child_name=name, path=self._walker._get_path() + child_name=name.name, path=self._walker._get_path() ) self.fields.get(name)._set_as_json({k: value[k], f"@{k}": v}) check_duplicates(name) already_set.add(name) continue - self._walker.get_child(k) - name = Identifier.from_model_string(k).name + try: + name = self._walker.get_child(k).get_name() + except SrosMgmtError as e: + try: + if ":" not in k: + name = self._walker.get_child(Identifier(self._walker.get_name().prefix, k)).get_name() + else: + raise e from None + except: + raise e from None if name in already_set: continue - if name in self._walker.get_local_key_names(): + if name.name in self._walker.get_local_key_names(): self.keys.set_as_json(name, v) elif self._walker.get_child_dds(name) in FIELD_STATEMENT_TYPES: if f"@{k}" in value: diff --git a/pysros/wrappers.py b/pysros/wrappers.py index a239c2f..56b5d81 100644 --- a/pysros/wrappers.py +++ b/pysros/wrappers.py @@ -101,18 +101,28 @@ def __getattr__(self, attr): if attr == "units" and self._model.units: return self._model.units if attr == "default" and self._model.default: - assert isinstance(self._model.default, str) - if self._model.yang_type.json_name() == "boolean": - if self._model.default == "true": - return True - if self._model.default == "false": - return False - if self._model.yang_type.json_name() in INTEGRAL_LEAF_TYPE: - try: - return int(self._model.default) - except ValueError: - pass - return self._model.default + assert isinstance(self._model.default, (list, str)) + default = self._model.default + res = [] + if isinstance(default, str): + default = [default] + for d in default: + if self._model.yang_type.json_name() == "boolean": + if d == "true": + res.append(True) + if d == "false": + res.append(False) + continue + if self._model.yang_type.json_name() in INTEGRAL_LEAF_TYPE: + try: + res.append(int(d)) + except ValueError: + pass + continue + res.append(d) + if isinstance(self._model.default, str): + res = res[0] + return res if attr == "mandatory": if self._model.data_def_stm in ( self._model.StatementType.leaf_, @@ -745,8 +755,8 @@ class Annotations(collections.UserList): An :py:class:`.Annotation` in pySROS is treated in a similar way as any other YANG structure such as a :py:class:`.Leaf`. A :py:class:`.Annotation` - class wrapper encodes the structures required to define and use a YANG modeled annotation. Unlike other - wrappers, because a YANG modeled annotation can be in a different YANG namespace from the node it is + class wrapper encodes the structures required to define and use a YANG-modeled annotation. Unlike other + wrappers, because a YANG-modeled annotation can be in a different YANG namespace from the node it is attached to, additional information is needed. The :py:meth:`pysros.management.Connection.convert` method and the :py:meth:`pysros.management.Datastore.set` diff --git a/pysros/yang_type.py b/pysros/yang_type.py index ddcaeb6..c067b8d 100644 --- a/pysros/yang_type.py +++ b/pysros/yang_type.py @@ -233,6 +233,7 @@ def __init__(self, identifier: Identifier, yang_range: Optional[str] = None, fra self.yang_range = yang_range self.fraction_digits = fraction_digits self.length = length + self.enums: Set[str] = set() # Also for bits assert not self.identifier.is_builtin() def __str__(self): @@ -270,6 +271,15 @@ def to_value(self, _val: str) -> Any: def check_field_value(self, value: Any, json=False, strict=False, is_convert=False, metadata=None) -> bool: return False + def add_enum(self, name: str): + self.enums.add(name) + + def add_bit(self, name: str): + self.add_enum(name) + + def set_last_enum_value(self, val: int): + pass + class YangUnion(YangTypeBase): def __init__(self, types: List["YangType"] = []): @@ -357,18 +367,54 @@ def as_storage_type(self, obj, is_convert, idref_cb): return super().as_storage_type(obj, is_convert, idref_cb) -class Enumeration(OrderedDict, YangTypeBase): +class Enumeration(YangTypeBase): + def __init__(self, *args, **kwargs): + self.enums = OrderedDict(*args, **kwargs) + def __str__(self): return f"enumeration[{'|'.join(map(str, self.keys()))}]" def __hash__(self): return hash(tuple(self)) + def __iter__(self): + return iter(self.enums) + + def __reversed__(self): + return reversed(self.enums) + + def __len__(self): + return len(self.enums) + + def __getitem__(self, name): + return self.enums[name] + + def __setitem__(self, name, val): + self.enums[name] = val + + def __delitem__(self, name): + del self.enums[name] + + def __eq__(self, rhs): + return self.enums == rhs.enums + + def __repr__(self): + return f"Enumeration({repr(self.enums)})" + + def keys(self): + return self.enums.keys() + + def values(self): + return self.enums.values() + + def items(self): + return self.enums.items() + def add_enum(self, name: str): val = 1 + max(self.values()) if self else 0 self[name] = val - def add_enums(self, *names: Iterable[str]): + def add_enums(self, *names: str): for name in names: self.add_enum(name) @@ -388,22 +434,42 @@ def check_field_value(self, value: Any, json=False, strict=False, is_convert=Fal return isinstance(value, str) -class Bits(set, YangTypeBase): - def __str__(self): - return f"bits[{' '.join(sorted(self))}]" +class Bits(YangTypeBase): + def __init__(self, vals=()): + self.enums = {} + for i in vals: + self.add_bit(i) + + def __iter__(self): + return iter(self.enums) + + def __reversed__(self): + return reversed(self.enums) + + def __len__(self): + return len(self.enums) + + def __eq__(self, rhs): + return self.enums == rhs.enums def __repr__(self): - return f"Bits(({', '.join(map(repr, sorted(self)))}))" + return f"Bits({repr(list(self.keys()))})" + + def keys(self): + return self.enums.keys() + + def values(self): + return self.enums.values() + + def items(self): + return self.enums.items() + + def __str__(self): + return f"bits[{' '.join(sorted(self))}]" def __hash__(self): return hash(frozenset(self)) - def __eq__(self, other): - return ( - YangTypeBase.__eq__(self, other) and - frozenset(self) == frozenset(other) - ) - def is_valid_value(self, val: Any) -> bool: return isinstance(val, str) @@ -419,6 +485,9 @@ def check_field_value(self, value: Any, json=False, strict=False, is_convert=Fal return True return self.is_valid_value(value) + def add_bit(self, name): + self.enums[name] = None + class LeafRef(YangTypeBase): def __init__(self, path=None): @@ -429,7 +498,6 @@ def __init__(self, path=None): self._path = ModelPath(path) def set_path(self, path: ModelPath): - assert isinstance(path, ModelPath) self._path = path def __eq__(self, other): diff --git a/setup.py b/setup.py index fb6d5bd..98beca6 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,14 @@ # Copyright 2021-2024 Nokia from setuptools import setup +from pysros import __version__ with open("README.md", "r", encoding="utf-8") as f: long_description = f.read() setup( name='pysros', - version='24.7.1', + version=__version__, packages=['pysros'], url='https://www.nokia.com', license='Copyright 2021-2024 Nokia. License available in the LICENSE.md file.',