Merge pull request #20 from UKPLab/logging

Centrialzed Logging

.gitignore

@@ -4,3 +4,7 @@
 __pycache__/
 *.py[cod]
 .venv
+.pytest_cache
+.es-data
+.DS_Store
+*.egg-info

elk-json-formatter/ElkJsonFormatter/ElkJsonFormatter.py

@@ -0,0 +1,10 @@
+from pythonjsonlogger import jsonlogger
+from datetime import datetime;
+
+class ElkJsonFormatter(jsonlogger.JsonFormatter):
+    def add_fields(self, log_record, record, message_dict):
+        super(ElkJsonFormatter, self).add_fields(log_record, record, message_dict)
+        log_record['@timestamp'] = datetime.now().isoformat()
+        log_record['level'] = record.levelname
+        log_record['logger'] = record.name
+        log_record['application'] = 'SQUARE'

elk-json-formatter/ElkJsonFormatter/__init__.py

@@ -0,0 +1 @@
+from .ElkJsonFormatter import ElkJsonFormatter

elk-json-formatter/README.md

@@ -0,0 +1,5 @@
+This package is for formatting logs as JSON.  
+
+To build run `python setup.py sdist`.  
+
+When changing the code, please update the version in `setup.py` and the default version in the bash `distribute_tar_to_services.sh` script. Then run the script to distribute the tar ball to the listed services in the script (update if necessary) to make the new version installable.  

elk-json-formatter/distribute_tar_to_services.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Distributes tar ball of ElkJsonFormatter to parent directories listed in $services
+# Please update version when creating new ElkJsonFormatter 
+version=${1:-"0.0.2"}
+services=(square-backend)
+for service in $services; do
+    cp dist/ElkJsonFormatter-${version}.tar.gz ../$service/ElkJsonFormatter.tar.gz
+done

elk-json-formatter/requirements.txt

@@ -0,0 +1 @@
+python-json-logger==2.0.1

elk-json-formatter/setup.py

@@ -0,0 +1,8 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="ElkJsonFormatter",
+    version="0.0.2",
+    packages=find_packages(include=['ElkJsonFormatter']),
+    install_requires=["python-json-logger==2.0.1"]
+)

logging/config/elasticsearch.yml

@@ -0,0 +1,2 @@
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0

logging/config/filebeat.yml

@@ -0,0 +1,19 @@
+filebeat.inputs:
+- type: container
+  paths: 
+    - '/var/lib/docker/containers/*/*.log'
+
+processors:
+# - add_docker_metadata:
+#     host: "unix:///var/run/docker.sock"
+
+- decode_json_fields:
+    fields: ["message"]
+    target: "square"
+    overwrite_keys: true
+
+output.logstash:
+  hosts: ["logstash:5044"]
+
+logging.json: true
+logging.metrics.enabled: false

logging/config/kibana.yml

@@ -0,0 +1,9 @@
+server.name: kibana
+server.host: 0.0.0.0
+elasticsearch.hosts: [ "http://elasticsearch:9200" ]
+monitoring.ui.container.elasticsearch.enabled: true
+
+## X-Pack security credentials
+#
+elasticsearch.username: elastic
+elasticsearch.password: changeme

logging/docker-compose.yml

@@ -0,0 +1,79 @@
+version: "3.2"
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
+    volumes:
+      - type: bind
+        source: ./config/elasticsearch.yml
+        target: /usr/share/elasticsearch/config/elasticsearch.yml
+        read_only: true
+      - type: volume
+        source: logging-es-data
+        target: /usr/share/elasticsearch/data
+    environment:
+      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+      ELASTIC_PASSWORD: changeme
+      discovery.type: single-node
+    networks:
+      - elk
+  logstash:
+    image: docker.elastic.co/logstash/logstash:7.13.2
+    volumes:
+      - type: bind
+        source: ./config/logstash.yml
+        target: /usr/share/logstash/config/logstash.yml
+        read_only: true
+      - type: bind
+        source: ./logstash/pipeline
+        target: /usr/share/logstash/pipeline
+        read_only: true
+    ports:
+      - "5044:5044"
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+  kibana:
+    image: docker.elastic.co/kibana/kibana:7.13.2
+    volumes:
+      - type: bind
+        source: ./config/kibana.yml
+        target: /usr/share/kibana/config/kibana.yml
+        read_only: true
+    ports:
+      - "5601:5601"
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+
+  filebeat:
+    image: "docker.elastic.co/beats/filebeat:7.13.2"
+    command: -e --strict.perms=false
+    user: root
+    volumes:
+        - type: bind
+          source: ./config/filebeat.yml
+          target: /usr/share/filebeat/filebeat.yml
+          read_only: true
+        - type: bind
+          source: /var/lib/docker
+          target: /var/lib/docker
+          read_only: true
+        - type: bind
+          source: /var/run/docker.sock
+          target: /var/run/docker.sock
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+
+networks:
+  elk:
+    driver: bridge
+
+volumes:
+    logging-es-data:

logging/logstash/pipeline/logstash.conf

@@ -0,0 +1,22 @@
+input {
+	beats {
+		port => 5044
+	}
+}
+
+filter {
+	if [square][application] != "SQUARE" {
+		drop { }
+	}
+}
+
+output {
+	elasticsearch {
+		hosts => "elasticsearch:9200"
+		user => "elastic"
+		password => "changeme"
+		ecs_compatibility => disabled
+		index => "square-logs"
+	}
+}
+

square-backend/Dockerfile

@@ -4,6 +4,7 @@ RUN pip install --upgrade pip
 
 WORKDIR /app
 
+COPY ElkJsonFormatter.tar.gz ./ElkJsonFormatter.tar.gz
 COPY requirements.txt requirements.txt
 RUN pip install -r requirements.txt
 # not in requirements because we use SQLite for dev server
@@ -31,5 +32,7 @@ RUN pytest \
 
 FROM base as build
 
+COPY logging.conf logging.conf
+
 RUN ["chmod", "+x", "./entry.sh"]
 ENTRYPOINT ["./entry.sh"]

square-backend/docker/config.yaml

@@ -1,6 +1,6 @@
 ---
 DEBUG: false
-SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://square:dbpw@square-core_db_1:5432/square
+SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://square:dbpw@db:5432/square
 SQLALCHEMY_TRACK_MODIFICATIONS: false
 JWT_IDENTITY_CLAIM: sub
 JWT_SECRET_KEY: change this key to something else
@@ -9,10 +9,10 @@ skillSelector:
   selectors:
     - name: Base
       config: {}
-    - name: Elasticsearch
-      config:
-        k: 50
-        weighting: score
-        elasticsearch:
-          url: "http://square-core_es_1:9200"
-          index: square_selector_data
+    # - name: Elasticsearch
+    #   config:
+    #     k: 50
+    #     weighting: score
+    #     elasticsearch:
+    #       url: "http://square-core_es_1:9200"
+    #       index: square_selector_data

square-backend/docker/entry.sh

@@ -10,4 +10,4 @@ python flask-manage.py db migrate --directory=migration
 python flask-manage.py db upgrade --directory=migration
 
 echo "Starting the server"
-python main.py --host "0.0.0.0"
+python main.py --host "0.0.0.0" --logging_conf logging.conf

square-backend/logging.conf

@@ -0,0 +1,21 @@
+[loggers]
+keys = root
+
+[logger_root]
+level = DEBUG
+handlers = root
+
+[handlers]
+keys = root
+
+[handler_root]
+class = StreamHandler
+level = DEBUG
+formatter = json
+
+[formatters]
+keys = json
+
+[formatter_json]
+class = ElkJsonFormatter.ElkJsonFormatter
+

square-backend/main.py

@@ -3,14 +3,17 @@
 from squareapi.app import create_app
 import argparse
 import logging
+import logging.config
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--host", type=str, default="127.0.0.1")
     parser.add_argument("--config", type=str, default="./config.yaml")
-    parser.add_argument("--logging_config", type=str, default="./logging_config.yaml")
+    parser.add_argument("--logging_config", type=str, default="./logging.conf")
     args = parser.parse_args()
 
-    socketio, app = create_app(config_path=args.config, logging_config_path=args.logging_config)
+    logging.config.fileConfig(args.logging_config, disable_existing_loggers=False)
+
+    socketio, app = create_app(config_path=args.config)
 
     socketio.run(app, host=args.host, log_output=True, log=logging.getLogger("server"))

square-backend/requirements.txt

@@ -1,6 +1,6 @@
 Flask
 SQLAlchemy
-Flask-Migrate
+Flask-Migrate==2.7.0
 Flask-Script
 Flask-CORS
 flask-jwt-extended==3.25.1
@@ -10,3 +10,4 @@ flasgger
 eventlet
 pyyaml
 elasticsearch>=7.0.0
+ElkJsonFormatter.tar.gz

square-backend/squareapi/app.py

@@ -1,5 +1,4 @@
 import logging
-import logging.config
 import yaml
 from flask import Flask
 from flask_cors import CORS
@@ -10,8 +9,7 @@
 logger = logging.getLogger(__name__)
 
 
-def create_app(app_name="SQUARE_API", config_path="./config.yaml", logging_config_path="./logging_config.yaml"):
-    logging.config.dictConfig(yaml.load(open(logging_config_path)))
+def create_app(app_name="SQUARE_API", config_path="./config.yaml"):
 
     logger.info("Creating Flask App")
     config = yaml.load(open(config_path))