Merge pull request #77 from bird-house/import-adapter

Import adapter

CHANGES.rst

@@ -4,6 +4,20 @@ Changes
 Unreleased
 ==========
 
+New Features:
+
+* Add `postgres` extra requirements for when it is used as database driver with `sqlalchemy`.
+
+Changes:
+
+* Use ``container`` instead of ``config`` for ``AdapterInterface.owsproxy_config`` to match real use cases.
+
+Fixes:
+
+* Improve the adapter import methodology to work with more use cases (https://github.com/Ouranosinc/Magpie/issues/182).
+* Fix incorrect setup for bump version within `Makefile`.
+* Fix Twitcher `main` including ``twitcher.<module>`` instead of ``.<module>``.
+
 0.5.0 (2019-05-22)
 ==================
 

Makefile

@@ -1,5 +1,5 @@
 # Configuration
-VERSION := VERSION := birdhouse/twitcher:0.5.0
+VERSION := 0.5.0
 APP_ROOT := $(abspath $(lastword $(MAKEFILE_LIST))/..)
 INI_FILE ?= development.ini
 

requirements.txt

@@ -7,6 +7,7 @@ argcomplete
 pytz
 lxml
 pyopenssl
+six
 # debug
 pyramid_debugtoolbar
 # deploy

setup.cfg

@@ -4,19 +4,19 @@ commit = True
 tag = True
 
 [bumpversion:file:CHANGES.rst]
-search = 
+search =
 	Unreleased
 	==========
-replace = 
+replace =
 	Unreleased
 	==========
-	
+
 	{new_version} ({now:%%Y-%%m-%%d})
 	==================
 
 [bumpversion:file:Makefile]
 search = VERSION := birdhouse/twitcher:{current_version}
-replace = VERSION := birdhouse/twitcher:{new_version}
+replace = {new_version}
 
 [bumpversion:file:twitcher/__version__.py]
 search = __version__ = '{current_version}'
@@ -30,18 +30,18 @@ replace = release = '{new_version}'
 description-file = README.rst
 
 [tool:pytest]
-addopts = 
+addopts =
 	--strict
 	--tb=native
 python_files = test_*.py
-markers = 
+markers =
 	online: mark test to need internet connection
 	slow: mark test to be slow
 
 [flake8]
 ignore = F401,E402
 max-line-length = 120
-exclude = 
+exclude =
 	.git,
 	__pycache__,
 	twitcher/alembic/versions,

setup.py

@@ -33,7 +33,10 @@
       zip_safe=False,
       test_suite='twitcher',
       install_requires=reqs,
-      extras_require={"dev": dev_reqs},  # pip install ".[dev]"
+      extras_require={
+          "dev": dev_reqs,              # pip install ".[dev]"
+          "postgres": ["psycopg2"],     # when using postgres database driver with sqlalchemy
+      },
       entry_points="""\
       [paste.app_factory]
       main = twitcher:main

tests/test_adapter.py

@@ -3,7 +3,11 @@
 from twitcher.adapter.default import DefaultAdapter
 from twitcher.store import ServiceStoreInterface
 from pyramid.testing import DummyRequest
+from pathlib import Path
 import pytest
+import shutil
+import site
+import os
 
 
 def test_import_adapter():
@@ -25,7 +29,7 @@ def test_adapter_factory_none_specified():
 
 
 # noinspection PyAbstractClass
-class TestAdapter(AdapterInterface):
+class DummyAdapter(AdapterInterface):
     def servicestore_factory(self, request):
         class DummyServiceStore(ServiceStoreInterface):
             def save_service(self, service): return True    # noqa: E704
@@ -38,20 +42,59 @@ def clear_services(self): pass                  # noqa: E704
 
 
 # noinspection PyPep8Naming
-def test_adapter_factory_TestAdapter_valid_import():
-    settings = {'twitcher.adapter': '{}.{}'.format(TestAdapter.__module__, TestAdapter.__name__)}
+def test_adapter_factory_DummyAdapter_valid_import_with_init():
+    settings = {'twitcher.adapter': DummyAdapter.name}
     adapter = get_adapter_factory(settings)
-    assert isinstance(adapter, TestAdapter), "Expect {!s}, but got {!s}".format(TestAdapter, type(adapter))
+    assert isinstance(adapter, DummyAdapter), "Expect {!s}, but got {!s}".format(DummyAdapter, type(adapter))
+
+
+def make_path(base, other='__init__.py'):
+    return str(Path(base) / Path(other))
+
+
+def test_adapter_factory_TmpAdapter_valid_import_installed_without_init():
+    """
+    Test a valid installed adapter import by setting although not located under same directory,
+    with much deeper structure, and without any '__init__.py' defining a python 'package'.
+    """
+
+    mod_pkg = 'test_package'
+    mod_base = site.getsitepackages()[0]
+    try:
+        mod_name = '{}.module.submodule.section'.format(mod_pkg)
+        mod_path = make_path(mod_base, mod_name.replace('.', '/'))
+        mod_file_name = 'file'
+        mod_file = make_path(mod_path, '{}.py'.format(mod_file_name))
+        mod_class = 'TmpAdapter'
+        os.makedirs(mod_path, exist_ok=True)
+        with open(mod_file, 'w') as f:
+            f.writelines([
+                "from twitcher.adapter.base import AdapterInterface\n\n",
+                "class {}(AdapterInterface):\n".format(mod_class),
+                "    pass\n"
+            ])
+        mod_import = '.'.join([mod_name, mod_file_name, mod_class])
+
+        settings = {'twitcher.adapter': mod_import}
+        adapter = get_adapter_factory(settings)
+        adapter_mod_name = '.'.join([adapter.__module__, type(adapter).__name__])
+        assert not isinstance(adapter, DefaultAdapter)
+        assert isinstance(adapter, AdapterInterface)
+        assert adapter_mod_name == mod_import
+    finally:
+        shutil.rmtree(make_path(mod_base, mod_pkg), ignore_errors=True)
 
 
 # noinspection PyAbstractClass
-class TestAdapterFake(object):
+class DummyAdapterFake(object):
     pass
 
 
 # noinspection PyPep8Naming
 def test_adapter_factory_TestAdapter_invalid_raised():
-    settings = {'twitcher.adapter': '{}.{}'.format(TestAdapterFake.__module__, TestAdapterFake.__name__)}
+    # fake adapter doesn't have 'name' property, provide at least same functionality
+    dummy_name = '{}.{}'.format(DummyAdapterFake.__module__, DummyAdapterFake.__name__)
+    settings = {'twitcher.adapter': dummy_name}
     with pytest.raises(TypeError) as err:
         get_adapter_factory(settings)
         pytest.fail(msg="Invalid adapter not inheriting from 'AdapterInterface' should raise on import.")
@@ -61,7 +104,7 @@ def test_adapter_factory_TestAdapter_invalid_raised():
 
 # noinspection PyTypeChecker
 def test_adapter_factory_call_servicestore_factory():
-    settings = {'twitcher.adapter': '{}.{}'.format(TestAdapter.__module__, TestAdapter.__name__)}
+    settings = {'twitcher.adapter': DummyAdapter.name}
     adapter = get_adapter_factory(settings)
     store = adapter.servicestore_factory(DummyRequest())
     assert isinstance(store, ServiceStoreInterface)

twitcher/__init__.py

@@ -8,10 +8,10 @@ def main(global_config, **settings):
     """
     with Configurator(settings=settings) as config:
         # include twitcher components
-        config.include('.models')
-        config.include('.frontpage')
-        config.include('.rpcinterface')
-        config.include('.owsproxy')
+        config.include('twitcher.models')
+        config.include('twitcher.frontpage')
+        config.include('twitcher.rpcinterface')
+        config.include('twitcher.owsproxy')
         # tweens/middleware
         # TODO: maybe add tween for exception handling or use unknown_failure view
         config.include('twitcher.tweens')

twitcher/adapter/__init__.py

@@ -1,6 +1,7 @@
-from twitcher.adapter.default import DefaultAdapter, AdapterInterface
+from twitcher.adapter.default import DefaultAdapter, TWITCHER_ADAPTER_DEFAULT
+from twitcher.adapter.base import AdapterInterface
 from twitcher.utils import get_settings
-
+from importlib import import_module
 from inspect import isclass
 from typing import TYPE_CHECKING
 
@@ -15,21 +16,20 @@
     from typing import AnyStr, Type, Union
 
 
-TWITCHER_ADAPTER_DEFAULT = 'default'
-
-
 def import_adapter(name):
     # type: (AnyStr) -> Type[AdapterInterface]
     """Attempts import of the class specified by python string ``package.module.class``."""
     components = name.split('.')
     mod_name = components[0]
-    mod = __import__(mod_name)
+    mod = import_module(mod_name)
     for comp in components[1:]:
         if not hasattr(mod, comp):
+            mod_from = mod_name
             mod_name = '{mod}.{sub}'.format(mod=mod_name, sub=comp)
-            mod = __import__(mod_name, fromlist=[mod_name])
+            mod = import_module(mod_name, package=mod_from)
             continue
         mod = getattr(mod, comp)
+        mod_name = mod.__name__
     if not isclass(mod) or not issubclass(mod, AdapterInterface):
         raise TypeError("Invalid reference is not of type '{}.{}'."
                         .format(AdapterInterface.__module__, AdapterInterface.__name__))

twitcher/adapter/base.py

@@ -1,6 +1,7 @@
 from twitcher.utils import get_settings
 
 from typing import TYPE_CHECKING
+import six
 if TYPE_CHECKING:
     from twitcher.typedefs import AnySettingsContainer, JSON
     from twitcher.store import AccessTokenStoreInterface, ServiceStoreInterface
@@ -9,14 +10,24 @@
     from pyramid.request import Request
 
 
-class AdapterInterface(object):
+class AdapterBase(type):
+    @property
+    def name(cls):
+        return '{}.{}'.format(cls.__module__, cls.__name__)
+
+
+class AdapterInterface(six.with_metaclass(AdapterBase)):
     """
     Common interface allowing functionality overriding using an adapter implementation.
     """
     def __init__(self, container):
         # type: (AnySettingsContainer) -> None
         self.settings = get_settings(container)
 
+    @classmethod
+    def name(cls):
+        return '{}.{}'.format(cls.__module__, cls.__name__)
+
     def describe_adapter(self):
         # type: () -> JSON
         """
@@ -52,7 +63,7 @@ def owssecurity_factory(self, request):
         """
         raise NotImplementedError
 
-    def owsproxy_config(self, config):
+    def owsproxy_config(self, container):
         # type: (AnySettingsContainer) -> None
         """
         Returns the 'owsproxy' implementation of the adapter.

twitcher/adapter/default.py

@@ -2,18 +2,27 @@
 Factories to create storage backends.
 """
 
-from twitcher.adapter.base import AdapterInterface
+from twitcher.adapter.base import AdapterInterface, AdapterBase
 from twitcher.store import AccessTokenStore, ServiceStore
 from twitcher.owssecurity import OWSSecurity
 from twitcher.utils import get_settings
 
 from pyramid.config import Configurator
+import six
 
+TWITCHER_ADAPTER_DEFAULT = 'default'
 
-class DefaultAdapter(AdapterInterface):
+
+class DefaultBase(AdapterBase):
+    @property
+    def name(cls):
+        return TWITCHER_ADAPTER_DEFAULT
+
+
+class DefaultAdapter(six.with_metaclass(DefaultBase, AdapterInterface)):
     def describe_adapter(self):
         from twitcher.__version__ import __version__
-        return {"name": "default", "version": str(__version__)}
+        return {"name": self.name, "version": str(__version__)}
 
     def configurator_factory(self, container):
         settings = get_settings(container)