Merge branch 'refactor' into 'master'

tlex · tlex · commit 9bce9d3d09f4 · 2020-12-06T15:02:08.000Z
Code refactor

See merge request ix.ai/csp!6
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,8 +1,10 @@
+---
 variables:
   DOCKERHUB_REPO_NAME: csp
   ENABLE_ARM64: 'true'
   ENABLE_ARMv7: 'true'
   ENABLE_ARMv6: 'true'
+  ENABLE_386: 'true'
 
 include:
   - project: 'ix.ai/ci-templates'
diff --git a/.pylintrc b/.pylintrc
@@ -60,7 +60,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use "--disable=all --enable=classes
 # --disable=W".
-disable=logging-format-interpolation,
+disable=logging-fstring-interpolation,
 #        too-few-public-methods,
         invalid-name,
         no-self-use,
diff --git a/Dockerfile b/Dockerfile
@@ -2,13 +2,14 @@ FROM alpine:latest
 LABEL maintainer="docker@ix.ai" \
       ai.ix.repository="ix.ai/csp"
 
-WORKDIR /app
-
-COPY src/ /app
+COPY csp/requirements.txt /csp/requirements.txt
 
 RUN apk add --no-cache python3 py3-pip py3-waitress py3-flask py3-prometheus-client && \
-    pip3 install --no-cache-dir -r requirements.txt
+    pip3 install --no-cache-dir -r /csp/requirements.txt
+    
+COPY csp/ /csp
+COPY csp.sh /usr/local/bin/csp.sh
 
 EXPOSE 9180
 
-ENTRYPOINT ["python3", "/app/csp.py"]
+ENTRYPOINT ["/usr/local/bin/csp.sh"]
diff --git a/README.md b/README.md
diff --git a/build.sh b/build.sh
@@ -1,4 +1,6 @@
 #!/usr/bin/env sh
 
-echo "Setting VERSION='${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}' in src/constants.py"
-echo "VERSION = '${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}'" >> src/constants.py
+echo "Setting VERSION"
+find . -name .git -type d -prune -o -type f -name constants.py -exec sed -i "s|^VERSION.*|VERSION\ =\ \'${CI_COMMIT_REF_NAME:-None}\'|g" {} + -exec grep VERSION {} +
+echo "Setting BUILD"
+find . -name .git -type d -prune -o -type f -name constants.py -exec sed -i "s|^BUILD.*|BUILD\ =\ \'${CI_PIPELINE_ID:-None}\'|g" {} + -exec grep BUILD {} +
diff --git a/csp.sh b/csp.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+exec python3 -m csp "$@"
diff --git a/csp/__init__.py b/csp/__init__.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+""" sets the logging defaults for csp """
+
+import os
+from .lib import log as logging
+from .lib import constants
+
+version = f'{constants.VERSION}-{constants.BUILD}'
+log = logging.setup_logger(
+    name='csp',
+    level=os.environ.get('LOGLEVEL', 'INFO'),
+    gelf_host=os.environ.get('GELF_HOST'),
+    gelf_port=int(os.environ.get('GELF_PORT', 12201)),
+    _ix_id=__package__,
+    _version=version,
+)
diff --git a/csp/__main__.py b/csp/__main__.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+""" processes the environment variables and starts csp """
+
+import logging
+from . import csp
+from .lib import helpers
+from .lib import constants
+
+log = logging.getLogger('csp')
+
+# The environ keys to use, each of them correlating to `int`, `list`, `string`, `boolean` or `filter`
+options = helpers.gather_environ({
+    'max_content_length': 'int',
+    'address': 'string',
+    'port': 'int',
+    'enable_healthz_version': 'boolean',
+    'csp_path': 'string',
+    'healthz_path': 'string',
+    'metrics_path': 'string',
+})
+c = csp.CSP(**options)
+
+version = f'{constants.VERSION}-{constants.BUILD}'
+log.warning(f"Starting **{__package__} {version}**. Listening on {c.get_address()}:{c.get_port()}")
+
+c.start()
diff --git a/csp/csp.py b/csp/csp.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+""" Web server that logs HTTP POST requests """
+
+import logging
+import os
+import json
+from flask import Flask
+from flask import request
+from waitress import serve
+from .lib.constants import VERSION, BUILD, W001, W002, W003
+log = logging.getLogger('csp')
+
+
+class CSP():
+    """ The main CSP class """
+
+    settings = {
+        'max_content_length': 32768,
+        'address': '*',
+        'port': 9180,
+        'enable_healthz_version': False,
+        'csp_path': '/csp',
+        'healthz_path': '/healthz',
+        'metrics_path': '/metrics',  # Placeholder
+    }
+
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            if k in self.settings:
+                self.settings[k] = v
+            else:
+                log.warning(f'{k} not found in settings. Ignoring.')
+        self.server = Flask(__name__)
+        self.server.secret_key = os.urandom(64).hex()
+        self.server.add_url_rule(self.settings['csp_path'], 'csp', self.log_csp, methods=['POST'])
+        self.server.add_url_rule(self.settings['healthz_path'], 'healthz', self.healthz, methods=['GET'])
+
+    def log_csp(self):
+        """ Logs the content posted """
+
+        result = ('OK', 200)
+
+        try:
+            if request.content_length == 0:
+                raise TypeError(W002)
+            if request.content_length > self.settings['max_content_length']:
+                log.warning(f'{W001} ({request.content_length}). Dropping.')
+                result = (W001, 413)
+        except TypeError:
+            log.warning(W002)
+            result = (W002, 422)
+
+        if result == ('OK', 200):
+            log.debug(f"{request.environ}")
+            content = request.get_data(as_text=True)
+            try:
+                json_content = json.loads(content)
+                log.info(f'{json.dumps(json_content)}')
+                # Placeholder: json_content can now be analysed. Ideally, with a function outside of the CSP class
+            except json.decoder.JSONDecodeError:
+                log.debug(f'{W003}: `{content}`')
+                result = (W003, 422)
+
+        return result
+
+    def healthz(self):
+        """ Healthcheck """
+        version_string = f'{__package__} {VERSION}-{BUILD}'
+
+        log.debug(f'Healthcheck {version_string}')
+
+        message = 'OK'
+        if self.settings['enable_healthz_version']:
+            message = version_string
+        return (message, 200)
+
+    def start(self):
+        """ Start the web server """
+        serve(
+            self.server,
+            host=self.settings['address'],
+            port=self.settings['port'],
+            ident=None,
+        )
+
+    def get_port(self) -> int:
+        """ returns the configured port from self.settings['port'] """
+        return self.settings['port']
+
+    def get_address(self) -> int:
+        """ returns the configured address from self.settings['port'] """
+        return self.settings['address']
diff --git a/csp/lib/constants.py b/csp/lib/constants.py
@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+""" Constants declarations """
+
+VERSION = None
+BUILD = None
+
+W001 = 'Content too large'
+W002 = 'Empty content received'
+W003 = 'Content is not JSON'
diff --git a/csp/lib/helpers.py b/csp/lib/helpers.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+""" Helper functions """
+
+import logging
+import os
+from distutils.util import strtobool
+
+log = logging.getLogger("csp")
+
+
+def gather_environ(keys) -> dict:
+    """
+    Return a dict of environment variables correlating to the keys dict
+    :param keys: The environ keys to use, each of them correlating to `int`, `list`, `string`, `boolean` or `filter`
+    :return: A dict of found environ values
+    """
+    environs = {}
+    for key, key_type in keys.items():
+        if os.environ.get(key.upper()):
+            environs.update({key: os.environ[key.upper()]})
+            if key_type == 'int':
+                try:
+                    environs[key] = int(environs[key])
+                except ValueError:
+                    log.warning(f"`{environs[key]}` not understood for {key.upper()}. Ignoring.")
+                    del environs[key]
+                    continue
+            if key_type == 'list':
+                environs[key] = environs[key].split(' ')
+            if key_type == 'boolean':
+                try:
+                    environs[key] = bool(strtobool(environs[key]))
+                except ValueError:
+                    log.warning(f"`{environs[key]}` not understood for {key.upper()}. Setting to False.")
+                    environs[key] = False
+                    continue
+            if key_type == 'filter':
+                filters = environs[key].split('=', 1)
+                environs[key] = {filters[0]: filters[1]}
+            log.info(f'{key.upper()} is set')
+    return environs
diff --git a/csp/lib/log.py b/csp/lib/log.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+""" Global logging configuration """
+
+import logging
+import pygelf
+
+
+def setup_logger(name, level='INFO', gelf_host=None, gelf_port=None, **kwargs):
+    """ sets up the logger """
+    logging.basicConfig(handlers=[logging.NullHandler()])
+    formatter = logging.Formatter(
+        fmt='%(asctime)s.%(msecs)03d %(levelname)s [%(module)s.%(funcName)s] %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S',
+    )
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    if gelf_host and gelf_port:
+        handler = pygelf.GelfUdpHandler(
+            host=gelf_host,
+            port=gelf_port,
+            debug=True,
+            include_extra_fields=True,
+            **kwargs
+        )
+        logger.addHandler(handler)
+
+    ix_logger = logging.getLogger('ix_notifiers')
+    ix_logger.setLevel(level)
+    ix_logger.addHandler(handler)
+
+    return logger
diff --git a/csp/requirements.txt b/csp/requirements.txt
@@ -0,0 +1,4 @@
+waitress==1.4.4
+flask==1.1.2
+prometheus_client==0.9.0
+pygelf==0.3.6
diff --git a/src/__init__.py b/src/__init__.py
diff --git a/src/constants.py b/src/constants.py
diff --git a/src/csp.py b/src/csp.py
diff --git a/src/requirements.txt b/src/requirements.txt

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+#!/usr/bin/env sh`
	`2`	`+`
	`3`	`+exec python3 -m csp "$@"`