From 2696abcbf6eed032c011b740a9c78d676f60f43b Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 1 May 2022 21:20:39 +0530 Subject: [PATCH 01/14] views: Add CachedView --- confuse/core.py | 118 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/confuse/core.py b/confuse/core.py index 6c6c4b0..8401ee8 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -397,9 +397,12 @@ def add(self, obj): self.sources.append(ConfigSource.of(obj)) def set(self, value): + print(f"Rootview {self!r} set", value) self.sources.insert(0, ConfigSource.of(value)) def resolve(self): + # print(f"Ress {self.sources}") + # print("Ress", *((dict(s), s) for s in self.sources)) return ((dict(s), s) for s in self.sources) def clear(self): @@ -424,7 +427,7 @@ def get_redactions(self): class Subview(ConfigView): """A subview accessed via a subscript of a parent view.""" - def __init__(self, parent, key): + def __init__(self, parent: ConfigView, key): """Make a subview of a parent view for a given subscript key. """ self.parent = parent @@ -466,6 +469,7 @@ def resolve(self): yield value, source def set(self, value): + print("Setting", self.key, value) self.parent.set({self.key: value}) def add(self, value): @@ -722,3 +726,115 @@ def clear(self): # "Validated" configuration views: experimental! + + +_undefined = object() +_invalid = object() + + +class CachedHandle(object): + def __init__(self, view: ConfigView, template: templates.Template) -> None: + self._value = _undefined + self.view = view + self._template = template + print(f"Creating handle 0x{id(self):x} from {view=!r} 0x{id(view):x} {template=}") + + def get(self): + if self._value is _invalid: + raise ConfigError("Cache has been invalidated") + if self._value is _undefined: + print(f"Getting from 0x{id(self.view):x}") + self._value = self.view.get(self._template) + return self._value + + def unset(self): + self._value = _undefined + + def invalidate(self): + self._value = _invalid + + +class CachedConfigView(Subview): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.handles = {} + self.subviews = {} + + def get_handle(self, template: templates.Template): + template_id = id(template) + return self.handles.setdefault(template_id, CachedHandle(self, template)) + + def __getitem__(self, key): + if key not in self.subviews: + val = CachedConfigView(self, key) + self.subviews[key] = val + print(f"New {key=} {val=!r} 0x{id(val):x}") + else: + val = self.subviews[key] + print(f"cfg from cache {key=} {val=}") + return val + # return self.subviews.setdefault(key, ) + + def __setitem__(self, key, value): + print(f"clearing in {self!r} 0x{id(self):x}", self.handles, self.parent) + subview: CachedConfigView = self[key] + print(f"{subview=} 0x{id(subview):x} {subview.handles}") + for handle in subview.handles.values(): + handle.unset() + recursive_invalidate(subview, value) + return super().__setitem__(key, value) + + +def recursive_invalidate(view: CachedConfigView, val): + print(f"Rec {view=!r} {view.handles}") + for subview in view.subviews.values(): + if isinstance(val, dict): + if subview.key not in val: + for handle in subview.handles.values(): + handle.invalidate() + subval = None + else: + for handle in subview.handles.values(): + handle.unset() + subval = val[subview.key] + elif isinstance(val, list): + if not (isinstance(subview.key, int) and subview.key < len(val)): + for handle in subview.handles.values(): + handle.invalidate() + subval = None + else: + for handle in subview.handles.values(): + handle.unset() + subval = val[subview.key] + else: + for handle in subview.handles.values(): + handle.invalidate() + subval = None + recursive_invalidate(subview, subval) + + +class CachedConfiguration(Configuration): + def __init__(self, *args, **kwargs): + Configuration.__init__(self, *args, **kwargs) + self.subviews = {} + + def __getitem__(self, key): + """Get a subview of this view.""" + if key not in self.subviews: + val = CachedConfigView(self, key) + self.subviews[key] = val + print(f"New config {key=} {val=!r} 0x{id(val):x}") + else: + val = self.subviews[key] + print(f"cfg from cache {key=} {val=}") + return val + + def __setitem__(self, key, value): + print(f"clearing in {self!r} 0x{id(self):x}") + subview: CachedConfigView = self[key] + print(f"{subview=} 0x{id(subview):x} {subview.handles}") + for handle in subview.handles.values(): + handle.unset() + recursive_invalidate(subview, value) + return super().__setitem__(key, value) + From 6dda3619c57b51126997e809bf706ad03fca474d Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 1 May 2022 22:00:36 +0530 Subject: [PATCH 02/14] views: Add ConfigHandleInvalidatedError and remove prints --- confuse/core.py | 53 +++++++++++++++++-------------------------- confuse/exceptions.py | 8 ++++++- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/confuse/core.py b/confuse/core.py index 8401ee8..ed69880 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -26,7 +26,8 @@ from . import templates from . import yaml_util from .sources import ConfigSource, EnvSource, YamlSource -from .exceptions import ConfigTypeError, NotFoundError, ConfigError +from .exceptions import (ConfigTypeError, NotFoundError, ConfigError, + ConfigHandleInvalidatedError) CONFIG_FILENAME = 'config.yaml' DEFAULT_FILENAME = 'config_default.yaml' @@ -737,11 +738,10 @@ def __init__(self, view: ConfigView, template: templates.Template) -> None: self._value = _undefined self.view = view self._template = template - print(f"Creating handle 0x{id(self):x} from {view=!r} 0x{id(view):x} {template=}") def get(self): if self._value is _invalid: - raise ConfigError("Cache has been invalidated") + raise ConfigHandleInvalidatedError() if self._value is _undefined: print(f"Getting from 0x{id(self.view):x}") self._value = self.view.get(self._template) @@ -765,29 +765,19 @@ def get_handle(self, template: templates.Template): return self.handles.setdefault(template_id, CachedHandle(self, template)) def __getitem__(self, key): - if key not in self.subviews: - val = CachedConfigView(self, key) - self.subviews[key] = val - print(f"New {key=} {val=!r} 0x{id(val):x}") - else: - val = self.subviews[key] - print(f"cfg from cache {key=} {val=}") - return val - # return self.subviews.setdefault(key, ) + return self.subviews.setdefault(key, CachedConfigView(self, key)) def __setitem__(self, key, value): - print(f"clearing in {self!r} 0x{id(self):x}", self.handles, self.parent) subview: CachedConfigView = self[key] - print(f"{subview=} 0x{id(subview):x} {subview.handles}") for handle in subview.handles.values(): handle.unset() - recursive_invalidate(subview, value) + _recursive_invalidate(subview, value) return super().__setitem__(key, value) -def recursive_invalidate(view: CachedConfigView, val): - print(f"Rec {view=!r} {view.handles}") +def _recursive_invalidate(view: CachedConfigView, val): for subview in view.subviews.values(): + # TODO: Simplify this if else if isinstance(val, dict): if subview.key not in val: for handle in subview.handles.values(): @@ -810,31 +800,30 @@ def recursive_invalidate(view: CachedConfigView, val): for handle in subview.handles.values(): handle.invalidate() subval = None - recursive_invalidate(subview, subval) + _recursive_invalidate(subview, subval) -class CachedConfiguration(Configuration): +class CachedRootView(RootView): def __init__(self, *args, **kwargs): - Configuration.__init__(self, *args, **kwargs) + RootView.__init__(self, *args, **kwargs) self.subviews = {} def __getitem__(self, key): """Get a subview of this view.""" - if key not in self.subviews: - val = CachedConfigView(self, key) - self.subviews[key] = val - print(f"New config {key=} {val=!r} 0x{id(val):x}") - else: - val = self.subviews[key] - print(f"cfg from cache {key=} {val=}") - return val + return self.subviews.setdefault(key, CachedConfigView(self, key)) def __setitem__(self, key, value): - print(f"clearing in {self!r} 0x{id(self):x}") subview: CachedConfigView = self[key] - print(f"{subview=} 0x{id(subview):x} {subview.handles}") for handle in subview.handles.values(): handle.unset() - recursive_invalidate(subview, value) - return super().__setitem__(key, value) + _recursive_invalidate(subview, value) + return RootView.__setitem__(self, key, value) + + +class CachedConfiguration(Configuration): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.subviews = {} + __getitem__ = CachedRootView.__getitem__ + __setitem__ = CachedRootView.__setitem__ diff --git a/confuse/exceptions.py b/confuse/exceptions.py index 782260e..e09d6ee 100644 --- a/confuse/exceptions.py +++ b/confuse/exceptions.py @@ -4,7 +4,7 @@ __all__ = [ 'ConfigError', 'NotFoundError', 'ConfigValueError', 'ConfigTypeError', - 'ConfigTemplateError', 'ConfigReadError'] + 'ConfigTemplateError', 'ConfigReadError', 'ConfigHandleInvalidatedError'] YAML_TAB_PROBLEM = "found character '\\t' that cannot start any token" @@ -54,3 +54,9 @@ def __init__(self, name, reason=None): message += u': {0}'.format(reason) super(ConfigReadError, self).__init__(message) + + +class ConfigHandleInvalidatedError(ConfigError): + """Cached handle has been invalidated""" + def __init__(self) -> None: + super().__init__("Cache has been invalidated") From 9522958e8de6bc34aa451d7561020273c3e57f4f Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 1 May 2022 22:00:53 +0530 Subject: [PATCH 03/14] tests: Add unit tests for cached views --- test/test_cached_view.py | 65 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 test/test_cached_view.py diff --git a/test/test_cached_view.py b/test/test_cached_view.py new file mode 100644 index 0000000..2c3fedc --- /dev/null +++ b/test/test_cached_view.py @@ -0,0 +1,65 @@ +from __future__ import division, absolute_import, print_function + +import confuse +from confuse import CachedConfigView, CachedHandle, CachedRootView +import sys +import unittest + +from confuse.exceptions import ConfigHandleInvalidatedError +from confuse.templates import Sequence + +PY3 = sys.version_info[0] == 3 + + +class CachedViewTest(unittest.TestCase): + + def setUp(self) -> None: + self.config = CachedRootView([confuse.ConfigSource.of( + {"a": ["b", "c"], + "x": {"y": [1, 2], "w": "z"}})]) + return super().setUp() + + def test_basic(self): + view: CachedConfigView = self.config['x']['y'] + handle: CachedHandle = view.get_handle(Sequence(int)) + self.assertEqual(handle.get(), [1, 2]) + + def test_update(self): + view: CachedConfigView = self.config['x']['y'] + handle: CachedHandle = view.get_handle(Sequence(int)) + self.config['x']['y'] = [4, 5] + self.assertEqual(handle.get(), [4, 5]) + + def test_subview_update(self): + view: CachedConfigView = self.config['x']['y'] + handle: CachedHandle = view.get_handle(Sequence(int)) + self.config['x'] = {'y': [4, 5]} + self.assertEqual(handle.get(), [4, 5]) + + def test_invalidation(self): + view: CachedConfigView = self.config['x']['y'] + handle: CachedHandle = view.get_handle(Sequence(int)) + + self.config['x'] = {'p': [4, 5]} + # new dict doesn't have a 'y' key + with self.assertRaises(ConfigHandleInvalidatedError): + handle.get() + + def test_multi_handle_invalidation(self): + view: CachedConfigView = self.config['x']['w'] + handle = view.get_handle(str) + self.assertEqual(handle.get(), 'z') + + self.config['x'] = {'y': [4, 5]} + # new dict doesn't have a 'w' key + with self.assertRaises(ConfigHandleInvalidatedError): + handle.get() + + def test_list_update(self): + pass + + def test_root_update(self): + pass + + def test_root_invalidated(self): + pass From b891b3f253d325028f7fb2335bc01cbddfa12251 Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 1 May 2022 22:03:05 +0530 Subject: [PATCH 04/14] core: Remove some debug prints --- confuse/core.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/confuse/core.py b/confuse/core.py index ed69880..291348a 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -398,12 +398,9 @@ def add(self, obj): self.sources.append(ConfigSource.of(obj)) def set(self, value): - print(f"Rootview {self!r} set", value) self.sources.insert(0, ConfigSource.of(value)) def resolve(self): - # print(f"Ress {self.sources}") - # print("Ress", *((dict(s), s) for s in self.sources)) return ((dict(s), s) for s in self.sources) def clear(self): @@ -470,7 +467,6 @@ def resolve(self): yield value, source def set(self, value): - print("Setting", self.key, value) self.parent.set({self.key: value}) def add(self, value): @@ -743,7 +739,6 @@ def get(self): if self._value is _invalid: raise ConfigHandleInvalidatedError() if self._value is _undefined: - print(f"Getting from 0x{id(self.view):x}") self._value = self.view.get(self._template) return self._value From c4a2050a4f0628656e2557eba2346235f71ef850 Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 1 May 2022 22:13:48 +0530 Subject: [PATCH 05/14] core: Simplify _recursive_invalidate for cached views --- confuse/core.py | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/confuse/core.py b/confuse/core.py index 291348a..977bc80 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -770,31 +770,20 @@ def __setitem__(self, key, value): return super().__setitem__(key, value) -def _recursive_invalidate(view: CachedConfigView, val): +def _recursive_invalidate(view: CachedConfigView, new_val): + """Invalidate the cached handles for (sub)keys that are not present in new_val""" for subview in view.subviews.values(): - # TODO: Simplify this if else - if isinstance(val, dict): - if subview.key not in val: - for handle in subview.handles.values(): - handle.invalidate() - subval = None - else: - for handle in subview.handles.values(): - handle.unset() - subval = val[subview.key] - elif isinstance(val, list): - if not (isinstance(subview.key, int) and subview.key < len(val)): - for handle in subview.handles.values(): - handle.invalidate() - subval = None - else: - for handle in subview.handles.values(): - handle.unset() - subval = val[subview.key] - else: + try: + subval = new_val[subview.key] + except (KeyError, IndexError, TypeError): + # the old key doesn't exist in the new value anymore; invalidate. for handle in subview.handles.values(): handle.invalidate() subval = None + else: + # old key is present, possibly with a new value; unset. + for handle in subview.handles.values(): + handle.unset() _recursive_invalidate(subview, subval) From 37491e8363d340023a314ff85c8dd67902396ff8 Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 1 May 2022 22:39:42 +0530 Subject: [PATCH 06/14] core: Code cleanup for cached views --- confuse/core.py | 50 ++++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/confuse/core.py b/confuse/core.py index 977bc80..c3ab0fd 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -725,46 +725,54 @@ def clear(self): # "Validated" configuration views: experimental! -_undefined = object() -_invalid = object() - - class CachedHandle(object): + """Handle for a cached value computed by applying a template on the view""" + # some sentinel objects + _UNDEFINED = object() + _INVALID = object() + def __init__(self, view: ConfigView, template: templates.Template) -> None: - self._value = _undefined + self.value = self._UNDEFINED self.view = view - self._template = template + self.template = template def get(self): - if self._value is _invalid: + if self.value is self._INVALID: raise ConfigHandleInvalidatedError() - if self._value is _undefined: - self._value = self.view.get(self._template) - return self._value + if self.value is self._UNDEFINED: + self.value = self.view.get(self.template) + return self.value def unset(self): - self._value = _undefined + """Unset the cached value, will be repopulated on next get()""" + self.value = self._UNDEFINED def invalidate(self): - self._value = _invalid + """Invalidate the handle, will raise ConfigHandleInvalidatedError on get()""" + self.value = self._INVALID class CachedConfigView(Subview): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.handles = {} + self.handles = [] # keep track of all the handles from this view self.subviews = {} def get_handle(self, template: templates.Template): - template_id = id(template) - return self.handles.setdefault(template_id, CachedHandle(self, template)) + handle = CachedHandle(self, template) + self.handles.append(handle) + return handle + + def get(self, template=templates.REQUIRED): + """You probably want to use get_handle instead.""" + return super().get(template) def __getitem__(self, key): return self.subviews.setdefault(key, CachedConfigView(self, key)) def __setitem__(self, key, value): subview: CachedConfigView = self[key] - for handle in subview.handles.values(): + for handle in subview.handles: handle.unset() _recursive_invalidate(subview, value) return super().__setitem__(key, value) @@ -776,13 +784,13 @@ def _recursive_invalidate(view: CachedConfigView, new_val): try: subval = new_val[subview.key] except (KeyError, IndexError, TypeError): - # the old key doesn't exist in the new value anymore; invalidate. - for handle in subview.handles.values(): + # the old key doesn't exist in the new value anymore- invalidate. + for handle in subview.handles: handle.invalidate() subval = None else: - # old key is present, possibly with a new value; unset. - for handle in subview.handles.values(): + # old key is present, possibly with a new value- unset. + for handle in subview.handles: handle.unset() _recursive_invalidate(subview, subval) @@ -798,7 +806,7 @@ def __getitem__(self, key): def __setitem__(self, key, value): subview: CachedConfigView = self[key] - for handle in subview.handles.values(): + for handle in subview.handles: handle.unset() _recursive_invalidate(subview, value) return RootView.__setitem__(self, key, value) From d6a37226b6a225c68d5fc7d8e1d68544ec152c21 Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 1 May 2022 22:51:05 +0530 Subject: [PATCH 07/14] core: Extract common code to CachedViewMixin --- confuse/core.py | 75 ++++++++++++++++++++----------------------------- 1 file changed, 31 insertions(+), 44 deletions(-) diff --git a/confuse/core.py b/confuse/core.py index c3ab0fd..47ea972 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -752,7 +752,35 @@ def invalidate(self): self.value = self._INVALID -class CachedConfigView(Subview): +class CachedViewMixin: + def __getitem__(self, key): + return self.subviews.setdefault(key, CachedConfigView(self, key)) + + def __setitem__(self, key, value): + subview: CachedConfigView = self[key] + for handle in subview.handles: + handle.unset() + subview._recursive_invalidate(value) + return super().__setitem__(key, value) + + def _recursive_invalidate(self, new_val): + """Invalidate the cached handles for (sub)keys that are not present in new_val""" + for subview in self.subviews.values(): + try: + subval = new_val[subview.key] + except (KeyError, IndexError, TypeError): + # the old key doesn't exist in the new value anymore- invalidate. + for handle in subview.handles: + handle.invalidate() + subval = None + else: + # old key is present, possibly with a new value- unset. + for handle in subview.handles: + handle.unset() + subview._recursive_invalidate(subval) + + +class CachedConfigView(CachedViewMixin, Subview): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.handles = [] # keep track of all the handles from this view @@ -767,55 +795,14 @@ def get(self, template=templates.REQUIRED): """You probably want to use get_handle instead.""" return super().get(template) - def __getitem__(self, key): - return self.subviews.setdefault(key, CachedConfigView(self, key)) - - def __setitem__(self, key, value): - subview: CachedConfigView = self[key] - for handle in subview.handles: - handle.unset() - _recursive_invalidate(subview, value) - return super().__setitem__(key, value) - -def _recursive_invalidate(view: CachedConfigView, new_val): - """Invalidate the cached handles for (sub)keys that are not present in new_val""" - for subview in view.subviews.values(): - try: - subval = new_val[subview.key] - except (KeyError, IndexError, TypeError): - # the old key doesn't exist in the new value anymore- invalidate. - for handle in subview.handles: - handle.invalidate() - subval = None - else: - # old key is present, possibly with a new value- unset. - for handle in subview.handles: - handle.unset() - _recursive_invalidate(subview, subval) - - -class CachedRootView(RootView): +class CachedRootView(CachedViewMixin, RootView): def __init__(self, *args, **kwargs): RootView.__init__(self, *args, **kwargs) self.subviews = {} - def __getitem__(self, key): - """Get a subview of this view.""" - return self.subviews.setdefault(key, CachedConfigView(self, key)) - - def __setitem__(self, key, value): - subview: CachedConfigView = self[key] - for handle in subview.handles: - handle.unset() - _recursive_invalidate(subview, value) - return RootView.__setitem__(self, key, value) - -class CachedConfiguration(Configuration): +class CachedConfiguration(CachedViewMixin, Configuration): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.subviews = {} - - __getitem__ = CachedRootView.__getitem__ - __setitem__ = CachedRootView.__setitem__ From 44cc6664be6279d59740e5d4edce5aa4fb55d99a Mon Sep 17 00:00:00 2001 From: iamkroot Date: Tue, 3 May 2022 21:40:12 +0530 Subject: [PATCH 08/14] cache: Also unset any handles on ancestors of a modified view --- confuse/core.py | 16 +++++++++++++--- test/test_cached_view.py | 12 +++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/confuse/core.py b/confuse/core.py index 47ea972..664330d 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -760,10 +760,20 @@ def __setitem__(self, key, value): subview: CachedConfigView = self[key] for handle in subview.handles: handle.unset() - subview._recursive_invalidate(value) + subview._invalidate_descendants(value) + self._invalidate_ancestors() + return super().__setitem__(key, value) - def _recursive_invalidate(self, new_val): + def _invalidate_ancestors(self): + """Invalidate the cached handles for all the views up the chain""" + parent = self + while parent.name != ROOT_NAME: + for handle in parent.handles: + handle.unset() + parent = parent.parent + + def _invalidate_descendants(self, new_val): """Invalidate the cached handles for (sub)keys that are not present in new_val""" for subview in self.subviews.values(): try: @@ -777,7 +787,7 @@ def _recursive_invalidate(self, new_val): # old key is present, possibly with a new value- unset. for handle in subview.handles: handle.unset() - subview._recursive_invalidate(subval) + subview._invalidate_descendants(subval) class CachedConfigView(CachedViewMixin, Subview): diff --git a/test/test_cached_view.py b/test/test_cached_view.py index 2c3fedc..e8bd09f 100644 --- a/test/test_cached_view.py +++ b/test/test_cached_view.py @@ -16,7 +16,7 @@ class CachedViewTest(unittest.TestCase): def setUp(self) -> None: self.config = CachedRootView([confuse.ConfigSource.of( {"a": ["b", "c"], - "x": {"y": [1, 2], "w": "z"}})]) + "x": {"y": [1, 2], "w": "z", "p": {"q": 3}}})]) return super().setUp() def test_basic(self): @@ -63,3 +63,13 @@ def test_root_update(self): def test_root_invalidated(self): pass + + def test_invalidate_then_set(self): + pass + + def test_parent_unset(self): + view: CachedConfigView = self.config['x']['p'] + handle = view.get_handle(dict) + self.assertEqual(handle.get(), {'q': 3}) + self.config['x']['p']['q'] = 4 + self.assertEqual(handle.get(), {'q': 4}) From a253275c91e2364c8be00a86eccd7f164515fcc8 Mon Sep 17 00:00:00 2001 From: iamkroot Date: Tue, 3 May 2022 22:35:01 +0530 Subject: [PATCH 09/14] Support get_handle on a CachedRootView --- confuse/core.py | 39 ++++++++++++++++++++++----------------- test/test_cached_view.py | 9 +++++---- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/confuse/core.py b/confuse/core.py index 664330d..6b3cfa1 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -753,8 +753,18 @@ def invalidate(self): class CachedViewMixin: - def __getitem__(self, key): - return self.subviews.setdefault(key, CachedConfigView(self, key)) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.handles = [] # keep track of all the handles from this view + self.subviews = {} + + def __getitem__(self, key) -> "CachedConfigView": + try: + return self.subviews[key] + except KeyError: + val = CachedConfigView(self, key) + self.subviews[key] = val + return val def __setitem__(self, key, value): subview: CachedConfigView = self[key] @@ -768,13 +778,15 @@ def __setitem__(self, key, value): def _invalidate_ancestors(self): """Invalidate the cached handles for all the views up the chain""" parent = self - while parent.name != ROOT_NAME: + while True: for handle in parent.handles: handle.unset() + if parent.name == ROOT_NAME: + break parent = parent.parent def _invalidate_descendants(self, new_val): - """Invalidate the cached handles for (sub)keys that are not present in new_val""" + """Invalidate the cached handles for (sub)keys that are absent in new_val""" for subview in self.subviews.values(): try: subval = new_val[subview.key] @@ -789,13 +801,6 @@ def _invalidate_descendants(self, new_val): handle.unset() subview._invalidate_descendants(subval) - -class CachedConfigView(CachedViewMixin, Subview): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.handles = [] # keep track of all the handles from this view - self.subviews = {} - def get_handle(self, template: templates.Template): handle = CachedHandle(self, template) self.handles.append(handle) @@ -806,13 +811,13 @@ def get(self, template=templates.REQUIRED): return super().get(template) +class CachedConfigView(CachedViewMixin, Subview): + pass + + class CachedRootView(CachedViewMixin, RootView): - def __init__(self, *args, **kwargs): - RootView.__init__(self, *args, **kwargs) - self.subviews = {} + pass class CachedConfiguration(CachedViewMixin, Configuration): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.subviews = {} + pass diff --git a/test/test_cached_view.py b/test/test_cached_view.py index e8bd09f..94b8e8f 100644 --- a/test/test_cached_view.py +++ b/test/test_cached_view.py @@ -59,10 +59,11 @@ def test_list_update(self): pass def test_root_update(self): - pass - - def test_root_invalidated(self): - pass + root = self.config + handle = self.config.get_handle({'a': Sequence(str)}) + self.assertDictEqual(handle.get(), {'a': ['b', 'c']}) + root['a'] = ['c', 'd'] + self.assertDictEqual(handle.get(), {'a': ['c', 'd']}) def test_invalidate_then_set(self): pass From 751537fdd5c34ea3fcb174f098f9b8c06d774d63 Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 15 May 2022 20:21:31 +0530 Subject: [PATCH 10/14] Move CachedViews to cache.py Also removes the ConfigHandleInvalidatedError since it is the same as NotFoundError. --- confuse/cache.py | 121 ++++++++++++++++++++ confuse/core.py | 101 +--------------- confuse/exceptions.py | 8 +- test/{test_cached_view.py => test_cache.py} | 16 +-- 4 files changed, 128 insertions(+), 118 deletions(-) create mode 100644 confuse/cache.py rename test/{test_cached_view.py => test_cache.py} (85%) diff --git a/confuse/cache.py b/confuse/cache.py new file mode 100644 index 0000000..1d6631c --- /dev/null +++ b/confuse/cache.py @@ -0,0 +1,121 @@ +from typing import Dict, List + +from . import templates +from .core import ROOT_NAME, Configuration, ConfigView, RootView, Subview +from .exceptions import NotFoundError + + +class CachedHandle(object): + """Handle for a cached value computed by applying a template on the view. + """ + # some sentinel objects + _INVALID = object() + _MISSING = object() + + def __init__(self, view: ConfigView, template=templates.REQUIRED) -> None: + self.value = self._INVALID + self.view = view + self.template = template + + def get(self): + """Retreive the cached value from the handle. + + Will re-compute the value using `view.get(template)` if it has been + invalidated. + + May raise a `NotFoundError` if the underlying view has been + invalidated. + """ + if self.value is self._MISSING: + raise NotFoundError("The cached handle doesn't have a valid view") + if self.value is self._INVALID: + self.value = self.view.get(self.template) + return self.value + + def _invalidate(self): + """Invalidate the cached value, will be repopulated on next `get()`. + """ + self.value = self._INVALID + + def _set_view_missing(self): + """Invalidate the handle, will raise `NotFoundError` on `get()`. + """ + self.value = self._MISSING + + +class CachedViewMixin: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # keep track of all the handles from this view + self.handles: List[CachedHandle] = [] + # need to cache the subviews to be able to access their handles + self.subviews: Dict[str, CachedConfigView] = {} + + def __getitem__(self, key) -> "CachedConfigView": + try: + return self.subviews[key] + except KeyError: + val = CachedConfigView(self, key) + self.subviews[key] = val + return val + + def __setitem__(self, key, value): + subview: CachedConfigView = self[key] + # invalidate the existing handles up and down the view tree + for handle in subview.handles: + handle._invalidate() + subview._invalidate_descendants(value) + self._invalidate_ancestors() + + return super().__setitem__(key, value) + + def _invalidate_ancestors(self): + """Invalidate the cached handles for all the views up the chain. + + This is to ensure that they aren't referring to stale values. + """ + parent = self + while True: + for handle in parent.handles: + handle._invalidate() + if parent.name == ROOT_NAME: + break + parent = parent.parent + + def _invalidate_descendants(self, new_val): + """Invalidate the handles for (sub)keys that were updated and + set_view_missing for keys that are absent in new_val. + """ + for subview in self.subviews.values(): + try: + subval = new_val[subview.key] + except (KeyError, IndexError, TypeError): + # the old key doesn't exist in the new value anymore- + # set view as missing for the handles. + for handle in subview.handles: + handle._set_view_missing() + subval = None + else: + # old key is present, possibly with a new value- invalidate. + for handle in subview.handles: + handle._invalidate() + subview._invalidate_descendants(subval) + + def get_handle(self, template=templates.REQUIRED): + """Retreive a `CachedHandle` for the current view and template. + """ + handle = CachedHandle(self, template) + self.handles.append(handle) + return handle + + +class CachedConfigView(CachedViewMixin, Subview): + pass + + +class CachedRootView(CachedViewMixin, RootView): + pass + + +class CachedConfiguration(CachedViewMixin, Configuration): + pass diff --git a/confuse/core.py b/confuse/core.py index 6b3cfa1..727a568 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -26,8 +26,7 @@ from . import templates from . import yaml_util from .sources import ConfigSource, EnvSource, YamlSource -from .exceptions import (ConfigTypeError, NotFoundError, ConfigError, - ConfigHandleInvalidatedError) +from .exceptions import ConfigTypeError, NotFoundError, ConfigError CONFIG_FILENAME = 'config.yaml' DEFAULT_FILENAME = 'config_default.yaml' @@ -723,101 +722,3 @@ def clear(self): # "Validated" configuration views: experimental! - - -class CachedHandle(object): - """Handle for a cached value computed by applying a template on the view""" - # some sentinel objects - _UNDEFINED = object() - _INVALID = object() - - def __init__(self, view: ConfigView, template: templates.Template) -> None: - self.value = self._UNDEFINED - self.view = view - self.template = template - - def get(self): - if self.value is self._INVALID: - raise ConfigHandleInvalidatedError() - if self.value is self._UNDEFINED: - self.value = self.view.get(self.template) - return self.value - - def unset(self): - """Unset the cached value, will be repopulated on next get()""" - self.value = self._UNDEFINED - - def invalidate(self): - """Invalidate the handle, will raise ConfigHandleInvalidatedError on get()""" - self.value = self._INVALID - - -class CachedViewMixin: - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.handles = [] # keep track of all the handles from this view - self.subviews = {} - - def __getitem__(self, key) -> "CachedConfigView": - try: - return self.subviews[key] - except KeyError: - val = CachedConfigView(self, key) - self.subviews[key] = val - return val - - def __setitem__(self, key, value): - subview: CachedConfigView = self[key] - for handle in subview.handles: - handle.unset() - subview._invalidate_descendants(value) - self._invalidate_ancestors() - - return super().__setitem__(key, value) - - def _invalidate_ancestors(self): - """Invalidate the cached handles for all the views up the chain""" - parent = self - while True: - for handle in parent.handles: - handle.unset() - if parent.name == ROOT_NAME: - break - parent = parent.parent - - def _invalidate_descendants(self, new_val): - """Invalidate the cached handles for (sub)keys that are absent in new_val""" - for subview in self.subviews.values(): - try: - subval = new_val[subview.key] - except (KeyError, IndexError, TypeError): - # the old key doesn't exist in the new value anymore- invalidate. - for handle in subview.handles: - handle.invalidate() - subval = None - else: - # old key is present, possibly with a new value- unset. - for handle in subview.handles: - handle.unset() - subview._invalidate_descendants(subval) - - def get_handle(self, template: templates.Template): - handle = CachedHandle(self, template) - self.handles.append(handle) - return handle - - def get(self, template=templates.REQUIRED): - """You probably want to use get_handle instead.""" - return super().get(template) - - -class CachedConfigView(CachedViewMixin, Subview): - pass - - -class CachedRootView(CachedViewMixin, RootView): - pass - - -class CachedConfiguration(CachedViewMixin, Configuration): - pass diff --git a/confuse/exceptions.py b/confuse/exceptions.py index e09d6ee..782260e 100644 --- a/confuse/exceptions.py +++ b/confuse/exceptions.py @@ -4,7 +4,7 @@ __all__ = [ 'ConfigError', 'NotFoundError', 'ConfigValueError', 'ConfigTypeError', - 'ConfigTemplateError', 'ConfigReadError', 'ConfigHandleInvalidatedError'] + 'ConfigTemplateError', 'ConfigReadError'] YAML_TAB_PROBLEM = "found character '\\t' that cannot start any token" @@ -54,9 +54,3 @@ def __init__(self, name, reason=None): message += u': {0}'.format(reason) super(ConfigReadError, self).__init__(message) - - -class ConfigHandleInvalidatedError(ConfigError): - """Cached handle has been invalidated""" - def __init__(self) -> None: - super().__init__("Cache has been invalidated") diff --git a/test/test_cached_view.py b/test/test_cache.py similarity index 85% rename from test/test_cached_view.py rename to test/test_cache.py index 94b8e8f..7aeab1b 100644 --- a/test/test_cached_view.py +++ b/test/test_cache.py @@ -1,18 +1,12 @@ -from __future__ import division, absolute_import, print_function - -import confuse -from confuse import CachedConfigView, CachedHandle, CachedRootView -import sys import unittest -from confuse.exceptions import ConfigHandleInvalidatedError +import confuse +from confuse.cache import CachedConfigView, CachedHandle, CachedRootView +from confuse.exceptions import NotFoundError from confuse.templates import Sequence -PY3 = sys.version_info[0] == 3 - class CachedViewTest(unittest.TestCase): - def setUp(self) -> None: self.config = CachedRootView([confuse.ConfigSource.of( {"a": ["b", "c"], @@ -42,7 +36,7 @@ def test_invalidation(self): self.config['x'] = {'p': [4, 5]} # new dict doesn't have a 'y' key - with self.assertRaises(ConfigHandleInvalidatedError): + with self.assertRaises(NotFoundError): handle.get() def test_multi_handle_invalidation(self): @@ -52,7 +46,7 @@ def test_multi_handle_invalidation(self): self.config['x'] = {'y': [4, 5]} # new dict doesn't have a 'w' key - with self.assertRaises(ConfigHandleInvalidatedError): + with self.assertRaises(NotFoundError): handle.get() def test_list_update(self): From 17ed5f94aa03af383e58fc4fe696da615cce25d1 Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 15 May 2022 20:31:43 +0530 Subject: [PATCH 11/14] cache: Update tests --- test/test_cache.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/test_cache.py b/test/test_cache.py index 7aeab1b..e489f67 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -30,7 +30,7 @@ def test_subview_update(self): self.config['x'] = {'y': [4, 5]} self.assertEqual(handle.get(), [4, 5]) - def test_invalidation(self): + def test_missing(self): view: CachedConfigView = self.config['x']['y'] handle: CachedHandle = view.get_handle(Sequence(int)) @@ -39,7 +39,7 @@ def test_invalidation(self): with self.assertRaises(NotFoundError): handle.get() - def test_multi_handle_invalidation(self): + def test_missing2(self): view: CachedConfigView = self.config['x']['w'] handle = view.get_handle(str) self.assertEqual(handle.get(), 'z') @@ -50,7 +50,11 @@ def test_multi_handle_invalidation(self): handle.get() def test_list_update(self): - pass + view: CachedConfigView = self.config['a'][1] + handle = view.get_handle(str) + self.assertEqual(handle.get(), 'c') + self.config['a'][1] = 'd' + self.assertEqual(handle.get(), 'd') def test_root_update(self): root = self.config @@ -59,10 +63,7 @@ def test_root_update(self): root['a'] = ['c', 'd'] self.assertDictEqual(handle.get(), {'a': ['c', 'd']}) - def test_invalidate_then_set(self): - pass - - def test_parent_unset(self): + def test_parent_invalidation(self): view: CachedConfigView = self.config['x']['p'] handle = view.get_handle(dict) self.assertEqual(handle.get(), {'q': 3}) From 98e4982e30bdb2d8384c4d6d027c7f0ca6080179 Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 15 May 2022 20:33:59 +0530 Subject: [PATCH 12/14] core: Remove a misplaced type annotation --- confuse/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confuse/core.py b/confuse/core.py index 727a568..6c6c4b0 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -424,7 +424,7 @@ def get_redactions(self): class Subview(ConfigView): """A subview accessed via a subscript of a parent view.""" - def __init__(self, parent: ConfigView, key): + def __init__(self, parent, key): """Make a subview of a parent view for a given subscript key. """ self.parent = parent From da6c5984065b07ffe7a7d39e59dadc5ddeb4ab77 Mon Sep 17 00:00:00 2001 From: iamkroot Date: Mon, 16 May 2022 20:16:58 +0530 Subject: [PATCH 13/14] cache: Call template.get_default_value when view is missing This matches the behaviour of view.get(template) --- confuse/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confuse/cache.py b/confuse/cache.py index 1d6631c..f61421a 100644 --- a/confuse/cache.py +++ b/confuse/cache.py @@ -2,7 +2,6 @@ from . import templates from .core import ROOT_NAME, Configuration, ConfigView, RootView, Subview -from .exceptions import NotFoundError class CachedHandle(object): @@ -27,7 +26,8 @@ def get(self): invalidated. """ if self.value is self._MISSING: - raise NotFoundError("The cached handle doesn't have a valid view") + # will raise a NotFoundError if no default value was provided + self.value = templates.as_template(self.template).get_default_value() if self.value is self._INVALID: self.value = self.view.get(self.template) return self.value From b5444ef190e9a60ea6ee40dbf2fbd7ec854a8c8b Mon Sep 17 00:00:00 2001 From: iamkroot Date: Sun, 22 May 2022 09:55:21 +0530 Subject: [PATCH 14/14] cache: Simplify invalidation logic to match existing behaviour --- confuse/cache.py | 39 ++++++++------------------------------- test/test_cache.py | 11 ++++------- 2 files changed, 12 insertions(+), 38 deletions(-) diff --git a/confuse/cache.py b/confuse/cache.py index f61421a..ded336f 100644 --- a/confuse/cache.py +++ b/confuse/cache.py @@ -7,9 +7,8 @@ class CachedHandle(object): """Handle for a cached value computed by applying a template on the view. """ - # some sentinel objects _INVALID = object() - _MISSING = object() + """Sentinel object to denote that the cached value is out-of-date.""" def __init__(self, view: ConfigView, template=templates.REQUIRED) -> None: self.value = self._INVALID @@ -22,12 +21,8 @@ def get(self): Will re-compute the value using `view.get(template)` if it has been invalidated. - May raise a `NotFoundError` if the underlying view has been - invalidated. + May raise a `NotFoundError` if the underlying view is missing. """ - if self.value is self._MISSING: - # will raise a NotFoundError if no default value was provided - self.value = templates.as_template(self.template).get_default_value() if self.value is self._INVALID: self.value = self.view.get(self.template) return self.value @@ -37,11 +32,6 @@ def _invalidate(self): """ self.value = self._INVALID - def _set_view_missing(self): - """Invalidate the handle, will raise `NotFoundError` on `get()`. - """ - self.value = self._MISSING - class CachedViewMixin: def __init__(self, *args, **kwargs): @@ -62,9 +52,7 @@ def __getitem__(self, key) -> "CachedConfigView": def __setitem__(self, key, value): subview: CachedConfigView = self[key] # invalidate the existing handles up and down the view tree - for handle in subview.handles: - handle._invalidate() - subview._invalidate_descendants(value) + subview._invalidate_descendants() self._invalidate_ancestors() return super().__setitem__(key, value) @@ -82,24 +70,13 @@ def _invalidate_ancestors(self): break parent = parent.parent - def _invalidate_descendants(self, new_val): - """Invalidate the handles for (sub)keys that were updated and - set_view_missing for keys that are absent in new_val. + def _invalidate_descendants(self): + """Invalidate the handles for (sub)keys that were updated. """ + for handle in self.handles: + handle._invalidate() for subview in self.subviews.values(): - try: - subval = new_val[subview.key] - except (KeyError, IndexError, TypeError): - # the old key doesn't exist in the new value anymore- - # set view as missing for the handles. - for handle in subview.handles: - handle._set_view_missing() - subval = None - else: - # old key is present, possibly with a new value- invalidate. - for handle in subview.handles: - handle._invalidate() - subview._invalidate_descendants(subval) + subview._invalidate_descendants() def get_handle(self, template=templates.REQUIRED): """Retreive a `CachedHandle` for the current view and template. diff --git a/test/test_cache.py b/test/test_cache.py index e489f67..189044d 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -2,7 +2,6 @@ import confuse from confuse.cache import CachedConfigView, CachedHandle, CachedRootView -from confuse.exceptions import NotFoundError from confuse.templates import Sequence @@ -35,9 +34,9 @@ def test_missing(self): handle: CachedHandle = view.get_handle(Sequence(int)) self.config['x'] = {'p': [4, 5]} - # new dict doesn't have a 'y' key - with self.assertRaises(NotFoundError): - handle.get() + # new dict doesn't have a 'y' key, but according to the view-theory, + # it will get the value from the older view that has been shadowed. + self.assertEqual(handle.get(), [1, 2]) def test_missing2(self): view: CachedConfigView = self.config['x']['w'] @@ -45,9 +44,7 @@ def test_missing2(self): self.assertEqual(handle.get(), 'z') self.config['x'] = {'y': [4, 5]} - # new dict doesn't have a 'w' key - with self.assertRaises(NotFoundError): - handle.get() + self.assertEqual(handle.get(), 'z') def test_list_update(self): view: CachedConfigView = self.config['a'][1]