Add base proxy behind elastic search to support other search engines

search_service/api/search.py

@@ -2,7 +2,7 @@
 
 from flask_restful import Resource, fields, marshal_with, reqparse
 
-from search_service.proxy import elasticsearch
+from search_service.proxy import get_proxy_client
 
 table_fields = {
     "name": fields.String,
@@ -30,7 +30,7 @@ class SearchAPI(Resource):
     Search API
     """
     def __init__(self) -> None:
-        self.elasticsearch = elasticsearch.get_elasticsearch_proxy()
+        self.proxy = get_proxy_client()
 
         self.parser = reqparse.RequestParser(bundle_errors=True)
 
@@ -50,7 +50,7 @@ def get(self) -> Iterable[Any]:
 
         try:
 
-            results = self.elasticsearch.fetch_search_results(
+            results = self.proxy.fetch_search_results(
                 query_term=args['query_term'],
                 page_index=args['page_index']
             )
@@ -68,7 +68,7 @@ class SearchFieldAPI(Resource):
     Search API with explict field
     """
     def __init__(self) -> None:
-        self.elasticsearch = elasticsearch.get_elasticsearch_proxy()
+        self.proxy = get_proxy_client()
 
         self.parser = reqparse.RequestParser(bundle_errors=True)
 
@@ -91,7 +91,7 @@ def get(self, *, field_name: str,
         args = self.parser.parse_args(strict=True)
 
         try:
-            results = self.elasticsearch.fetch_search_results_with_field(
+            results = self.proxy.fetch_search_results_with_field(
                 query_term=args.get('query_term'),
                 field_name=field_name,
                 field_value=field_value,

search_service/config.py

@@ -8,9 +8,17 @@
 SEARCH_PAGE_SIZE_KEY = 'SEARCH_PAGE_SIZE'
 STATS_FEATURE_KEY = 'STATS'
 
+PROXY_ENDPOINT = 'PROXY_ENDPOINT'
+PROXY_USER = 'PROXY_USER'
+PROXY_PASSWORD = 'PROXY_PASSWORD'
+PROXY_CLIENT = 'PROXY_CLIENT'
+PROXY_CLIENTS = {
+    'ELASTICSEARCH': 'search_service.proxy.elasticsearch.ElasticsearchProxy',
+}
+
 
 class Config:
-    LOG_FORMAT = '%(asctime)s.%(msecs)03d [%(levelname)s] %(module)s.%(funcName)s:%(lineno)d (%(process)d:'\
+    LOG_FORMAT = '%(asctime)s.%(msecs)03d [%(levelname)s] %(module)s.%(funcName)s:%(lineno)d (%(process)d:' \
                  '%(threadName)s) - %(message)s'
     LOG_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S%z'
     LOG_LEVEL = 'INFO'
@@ -21,12 +29,12 @@ class LocalConfig(Config):
     TESTING = False
     STATS = False
     LOCAL_HOST = '0.0.0.0'
-    ELASTICSEARCH_PORT = '9200'
-    ELASTICSEARCH_ENDPOINT = os.environ.get('ELASTICSEARCH_ENDPOINT',
-                                            'http://{LOCAL_HOST}:{PORT}'.format(
-                                                LOCAL_HOST=LOCAL_HOST,
-                                                PORT=ELASTICSEARCH_PORT)
-                                            )
-    ELASTICSEARCH_INDEX = 'table_search_index'
-    ELASTICSEARCH_AUTH_USER = 'elastic'
-    ELASTICSEARCH_AUTH_PW = 'elastic'
+    PROXY_PORT = '9200'
+    PROXY_ENDPOINT = os.environ.get('PROXY_ENDPOINT',
+                                    'http://{LOCAL_HOST}:{PORT}'.format(
+                                        LOCAL_HOST=LOCAL_HOST,
+                                        PORT=PROXY_PORT)
+                                    )
+    PROXY_CLIENT = PROXY_CLIENTS[os.environ.get('PROXY_CLIENT', 'ELASTICSEARCH')]
+    PROXY_USER = os.environ.get('CREDENTIALS_PROXY_USER', 'elastic')
+    PROXY_PASSWORD = os.environ.get('CREDENTIALS_PROXY_PASSWORD', 'elastic')

search_service/proxy/__init__.py

@@ -0,0 +1,35 @@
+from threading import Lock
+
+from flask import current_app
+
+from search_service import config
+from search_service.proxy.base import BaseProxy
+from werkzeug.utils import import_string
+
+_proxy_client = None
+_proxy_client_lock = Lock()
+
+
+def get_proxy_client() -> BaseProxy:
+    """
+    Provides singleton proxy client based on the config
+    :return: Proxy instance of any subclass of BaseProxy
+    """
+    global _proxy_client
+
+    if _proxy_client:
+        return _proxy_client
+
+    with _proxy_client_lock:
+        if _proxy_client:
+            return _proxy_client
+        else:
+            # Gather all the configuration to create a Proxy Client
+            host = current_app.config[config.PROXY_ENDPOINT]
+            user = current_app.config[config.PROXY_USER]
+            password = current_app.config[config.PROXY_PASSWORD]
+
+            client = import_string(current_app.config[config.PROXY_CLIENT])
+            _proxy_client = client(host=host, index=None, user=user, password=password)
+
+    return _proxy_client

search_service/proxy/base.py

@@ -0,0 +1,26 @@
+from abc import ABCMeta, abstractmethod
+
+from typing import Union, List, Dict, Any
+
+from search_service.models.search_result import SearchResult
+
+
+class BaseProxy(metaclass=ABCMeta):
+    """
+    Base Proxy, which behaves like an interface for all
+    the proxy clients available in the amundsen search service
+    """
+
+    @abstractmethod
+    def fetch_search_results_with_field(self, *,
+                                        query_term: str,
+                                        field_name: str,
+                                        field_value: str,
+                                        page_index: int = 0) -> SearchResult:
+        pass
+
+    @abstractmethod
+    def fetch_search_results(self, *,
+                             query_term: str,
+                             page_index: int = 0) -> SearchResult:
+        pass

search_service/proxy/elasticsearch.py

@@ -11,6 +11,7 @@
 from search_service import config
 from search_service.models.search_result import SearchResult
 from search_service.models.table import Table
+from search_service.proxy.base import BaseProxy
 from search_service.proxy.statsd_utilities import timer_with_counter
 
 # Default Elasticsearch index to use, if none specified
@@ -22,7 +23,7 @@
 LOGGING = logging.getLogger(__name__)
 
 
-class ElasticsearchProxy:
+class ElasticsearchProxy(BaseProxy):
     """
     ElasticSearch connection handler
     """