Compatibility with Neo4J 4.0.4 (4.x in general should work) (#139)

* Compatibility with latest Neo4J 4.0.4

* Proxy connection is encrypted by default.

Also read an envvar to allow overriding. Need this to keep the
quick start sample working.

* New import form silences warning.

The neo4j.v1 import path raises a warning starting in the 1.7.x
driver series.

* Add a config parameter for SSL cert validation.

Author: davcamer

metadata_service/config.py

@@ -1,3 +1,4 @@
+import distutils.util
 import os
 from typing import List, Dict, Optional  # noqa: F401
 
@@ -6,6 +7,8 @@
 PROXY_PORT = 'PROXY_PORT'
 PROXY_USER = 'PROXY_USER'
 PROXY_PASSWORD = 'PROXY_PASSWORD'
+PROXY_ENCRYPTED = 'PROXY_ENCRYPTED'
+PROXY_VALIDATE_SSL = 'PROXY_VALIDATE_SSL'
 PROXY_CLIENT = 'PROXY_CLIENT'
 
 PROXY_CLIENTS = {
@@ -29,6 +32,10 @@ class Config:
 
     PROXY_USER = os.environ.get('CREDENTIALS_PROXY_USER', 'neo4j')
     PROXY_PASSWORD = os.environ.get('CREDENTIALS_PROXY_PASSWORD', 'test')
+    PROXY_ENCRYPTED = True
+    """Whether the connection to the proxy should use SSL/TLS encryption."""
+    PROXY_VALIDATE_SSL = True
+    """Whether the SSL/TLS certificate presented by the user should be validated against the system's trusted CAs."""
 
     IS_STATSD_ON = False
 
@@ -59,6 +66,8 @@ class LocalConfig(Config):
     PROXY_HOST = os.environ.get('PROXY_HOST', f'bolt://{LOCAL_HOST}')
     PROXY_PORT = os.environ.get('PROXY_PORT', 7687)
     PROXY_CLIENT = PROXY_CLIENTS[os.environ.get('PROXY_CLIENT', 'NEO4J')]
+    PROXY_ENCRYPTED = bool(distutils.util.strtobool(os.environ.get(PROXY_ENCRYPTED, 'True')))
+    PROXY_VALIDATE_SSL = bool(distutils.util.strtobool(os.environ.get(PROXY_VALIDATE_SSL, 'False')))
 
     JANUS_GRAPH_URL = None
 

metadata_service/proxy/__init__.py

@@ -29,8 +29,15 @@ def get_proxy_client() -> BaseProxy:
             port = current_app.config[config.PROXY_PORT]
             user = current_app.config[config.PROXY_USER]
             password = current_app.config[config.PROXY_PASSWORD]
+            encrypted = current_app.config[config.PROXY_ENCRYPTED]
+            validate_ssl = current_app.config[config.PROXY_VALIDATE_SSL]
 
             client = import_string(current_app.config[config.PROXY_CLIENT])
-            _proxy_client = client(host=host, port=port, user=user, password=password)
+            _proxy_client = client(host=host,
+                                   port=port,
+                                   user=user,
+                                   password=password,
+                                   encrypted=encrypted,
+                                   validate_ssl=validate_ssl)
 
     return _proxy_client

metadata_service/proxy/atlas_proxy.py

@@ -55,11 +55,19 @@ def __init__(self, *,
                  host: str,
                  port: int,
                  user: str = 'admin',
-                 password: str = '') -> None:
+                 password: str = '',
+                 encrypted: bool = False,
+                 validate_ssl: bool = False) -> None:
         """
         Initiate the Apache Atlas client with the provided credentials
         """
-        self._driver = Atlas(host=host, port=port, username=user, password=password)
+        protocol = 'https' if encrypted else 'http'
+        self._driver = Atlas(host=host,
+                             port=port,
+                             username=user,
+                             password=password,
+                             protocol=protocol,
+                             validate_ssl=validate_ssl)
 
     def _get_ids_from_basic_search(self, *, params: Dict) -> List[str]:
         """

metadata_service/proxy/neo4j_proxy.py

@@ -14,7 +14,8 @@
 from amundsen_common.models.user import User as UserEntity
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
-from neo4j.v1 import BoltStatementResult, Driver, GraphDatabase  # noqa: F401
+from neo4j import BoltStatementResult, Driver, GraphDatabase  # noqa: F401
+import neo4j
 
 from metadata_service.entity.dashboard_detail import DashboardDetail as DashboardDetailEntity
 from metadata_service.entity.dashboard_query import DashboardQuery as DashboardQueryEntity
@@ -45,7 +46,9 @@ def __init__(self, *,
                  user: str = 'neo4j',
                  password: str = '',
                  num_conns: int = 50,
-                 max_connection_lifetime_sec: int = 100) -> None:
+                 max_connection_lifetime_sec: int = 100,
+                 encrypted: bool = True,
+                 validate_ssl: bool = False) -> None:
         """
         There's currently no request timeout from client side where server
         side can be enforced via "dbms.transaction.timeout"
@@ -57,10 +60,13 @@ def __init__(self, *,
         value needs to be smaller than surrounding network environment's timeout.
         """
         endpoint = f'{host}:{port}'
+        trust = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES if validate_ssl else neo4j.TRUST_ALL_CERTIFICATES
         self._driver = GraphDatabase.driver(endpoint, max_connection_pool_size=num_conns,
                                             connection_timeout=10,
                                             max_connection_lifetime=max_connection_lifetime_sec,
-                                            auth=(user, password))  # type: Driver
+                                            auth=(user, password),
+                                            encrypted=encrypted,
+                                            trust=trust)  # type: Driver
 
     @timer_with_counter
     def get_table(self, *, table_uri: str) -> Table:

requirements.txt

@@ -62,8 +62,8 @@ pytz==2018.4
 six==1.11.0
 Werkzeug==0.15.3
 wheel==0.33.1
-neo4j-driver==1.6.0
-neotime==1.0.0
+neo4j==1.7.6
+neotime==1.7.1
 pytz==2018.4
 requests-aws4auth==0.9
 statsd==3.2.1

tests/unit/proxy/test_neo4j_proxy.py

@@ -10,7 +10,7 @@
                                           Watermark, ProgrammaticDescription)
 from amundsen_common.models.user import UserSchema
 from mock import MagicMock, patch
-from neo4j.v1 import GraphDatabase
+from neo4j import GraphDatabase
 
 from metadata_service import create_app
 from metadata_service.entity.dashboard_detail import DashboardDetail

tests/unit/proxy/test_statsd_utilities.py

@@ -7,7 +7,7 @@
 from flask import current_app
 
 from metadata_service import create_app
-from neo4j.v1 import GraphDatabase
+from neo4j import GraphDatabase
 from metadata_service.proxy.neo4j_proxy import Neo4jProxy