From a10f56330134fd449b08bdd5968272fde138d12a Mon Sep 17 00:00:00 2001 From: Nanne Wielinga Date: Thu, 2 May 2019 14:05:15 +0200 Subject: [PATCH] Add base proxy behind elastic search to support other search engines --- search_service/api/search.py | 10 ++++---- search_service/config.py | 28 +++++++++++++-------- search_service/proxy/__init__.py | 35 +++++++++++++++++++++++++++ search_service/proxy/base.py | 26 ++++++++++++++++++++ search_service/proxy/elasticsearch.py | 3 ++- 5 files changed, 86 insertions(+), 16 deletions(-) create mode 100644 search_service/proxy/base.py diff --git a/search_service/api/search.py b/search_service/api/search.py index 25e4b411..47928cca 100644 --- a/search_service/api/search.py +++ b/search_service/api/search.py @@ -2,7 +2,7 @@ from flask_restful import Resource, fields, marshal_with, reqparse -from search_service.proxy import elasticsearch +from search_service.proxy import get_proxy_client table_fields = { "name": fields.String, @@ -30,7 +30,7 @@ class SearchAPI(Resource): Search API """ def __init__(self) -> None: - self.elasticsearch = elasticsearch.get_elasticsearch_proxy() + self.proxy = get_proxy_client() self.parser = reqparse.RequestParser(bundle_errors=True) @@ -50,7 +50,7 @@ def get(self) -> Iterable[Any]: try: - results = self.elasticsearch.fetch_search_results( + results = self.proxy.fetch_search_results( query_term=args['query_term'], page_index=args['page_index'] ) @@ -68,7 +68,7 @@ class SearchFieldAPI(Resource): Search API with explict field """ def __init__(self) -> None: - self.elasticsearch = elasticsearch.get_elasticsearch_proxy() + self.proxy = get_proxy_client() self.parser = reqparse.RequestParser(bundle_errors=True) @@ -91,7 +91,7 @@ def get(self, *, field_name: str, args = self.parser.parse_args(strict=True) try: - results = self.elasticsearch.fetch_search_results_with_field( + results = self.proxy.fetch_search_results_with_field( query_term=args.get('query_term'), field_name=field_name, field_value=field_value, diff --git a/search_service/config.py b/search_service/config.py index fb154263..a16c2e4c 100644 --- a/search_service/config.py +++ b/search_service/config.py @@ -8,9 +8,17 @@ SEARCH_PAGE_SIZE_KEY = 'SEARCH_PAGE_SIZE' STATS_FEATURE_KEY = 'STATS' +PROXY_ENDPOINT = 'PROXY_ENDPOINT' +PROXY_USER = 'PROXY_USER' +PROXY_PASSWORD = 'PROXY_PASSWORD' +PROXY_CLIENT = 'PROXY_CLIENT' +PROXY_CLIENTS = { + 'ELASTICSEARCH': 'search_service.proxy.elasticsearch.ElasticsearchProxy', +} + class Config: - LOG_FORMAT = '%(asctime)s.%(msecs)03d [%(levelname)s] %(module)s.%(funcName)s:%(lineno)d (%(process)d:'\ + LOG_FORMAT = '%(asctime)s.%(msecs)03d [%(levelname)s] %(module)s.%(funcName)s:%(lineno)d (%(process)d:' \ '%(threadName)s) - %(message)s' LOG_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S%z' LOG_LEVEL = 'INFO' @@ -21,12 +29,12 @@ class LocalConfig(Config): TESTING = False STATS = False LOCAL_HOST = '0.0.0.0' - ELASTICSEARCH_PORT = '9200' - ELASTICSEARCH_ENDPOINT = os.environ.get('ELASTICSEARCH_ENDPOINT', - 'http://{LOCAL_HOST}:{PORT}'.format( - LOCAL_HOST=LOCAL_HOST, - PORT=ELASTICSEARCH_PORT) - ) - ELASTICSEARCH_INDEX = 'table_search_index' - ELASTICSEARCH_AUTH_USER = 'elastic' - ELASTICSEARCH_AUTH_PW = 'elastic' + PROXY_PORT = '9200' + PROXY_ENDPOINT = os.environ.get('PROXY_ENDPOINT', + 'http://{LOCAL_HOST}:{PORT}'.format( + LOCAL_HOST=LOCAL_HOST, + PORT=PROXY_PORT) + ) + PROXY_CLIENT = PROXY_CLIENTS[os.environ.get('PROXY_CLIENT', 'ELASTICSEARCH')] + PROXY_USER = os.environ.get('CREDENTIALS_PROXY_USER', 'elastic') + PROXY_PASSWORD = os.environ.get('CREDENTIALS_PROXY_PASSWORD', 'elastic') diff --git a/search_service/proxy/__init__.py b/search_service/proxy/__init__.py index e69de29b..933b6ea2 100644 --- a/search_service/proxy/__init__.py +++ b/search_service/proxy/__init__.py @@ -0,0 +1,35 @@ +from threading import Lock + +from flask import current_app + +from search_service import config +from search_service.proxy.base import BaseProxy +from werkzeug.utils import import_string + +_proxy_client = None +_proxy_client_lock = Lock() + + +def get_proxy_client() -> BaseProxy: + """ + Provides singleton proxy client based on the config + :return: Proxy instance of any subclass of BaseProxy + """ + global _proxy_client + + if _proxy_client: + return _proxy_client + + with _proxy_client_lock: + if _proxy_client: + return _proxy_client + else: + # Gather all the configuration to create a Proxy Client + host = current_app.config[config.PROXY_ENDPOINT] + user = current_app.config[config.PROXY_USER] + password = current_app.config[config.PROXY_PASSWORD] + + client = import_string(current_app.config[config.PROXY_CLIENT]) + _proxy_client = client(host=host, index=None, user=user, password=password) + + return _proxy_client diff --git a/search_service/proxy/base.py b/search_service/proxy/base.py new file mode 100644 index 00000000..559adf2d --- /dev/null +++ b/search_service/proxy/base.py @@ -0,0 +1,26 @@ +from abc import ABCMeta, abstractmethod + +from typing import Union, List, Dict, Any + +from search_service.models.search_result import SearchResult + + +class BaseProxy(metaclass=ABCMeta): + """ + Base Proxy, which behaves like an interface for all + the proxy clients available in the amundsen search service + """ + + @abstractmethod + def fetch_search_results_with_field(self, *, + query_term: str, + field_name: str, + field_value: str, + page_index: int = 0) -> SearchResult: + pass + + @abstractmethod + def fetch_search_results(self, *, + query_term: str, + page_index: int = 0) -> SearchResult: + pass diff --git a/search_service/proxy/elasticsearch.py b/search_service/proxy/elasticsearch.py index 982929b7..9ce2e248 100644 --- a/search_service/proxy/elasticsearch.py +++ b/search_service/proxy/elasticsearch.py @@ -11,6 +11,7 @@ from search_service import config from search_service.models.search_result import SearchResult from search_service.models.table import Table +from search_service.proxy.base import BaseProxy from search_service.proxy.statsd_utilities import timer_with_counter # Default Elasticsearch index to use, if none specified @@ -22,7 +23,7 @@ LOGGING = logging.getLogger(__name__) -class ElasticsearchProxy: +class ElasticsearchProxy(BaseProxy): """ ElasticSearch connection handler """