From 57427e4d30fc571db400ef7eca96d96b98df188c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 13:25:19 +0200 Subject: [PATCH 01/16] . --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 8cb0e948e..19c5929eb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ __pycache__/ *.py[cod] .venv +.pytest_cache +.es-data +.DS_Store From e5d16c5ebf9ec110baa48eb18d10b9daaeee42fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 13:31:00 +0200 Subject: [PATCH 02/16] egg-info --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 19c5929eb..745abbbc1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ __pycache__/ .pytest_cache .es-data .DS_Store +*.egg-info From f02e4a9155524198935297f1a4f2e35c46d4af1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 13:31:49 +0200 Subject: [PATCH 03/16] elk json formatter --- .../ElkJsonFormatter/ElkJsonFormatter.py | 10 ++++++++++ elk-json-formatter/ElkJsonFormatter/__init__.py | 1 + elk-json-formatter/README.md | 5 +++++ .../dist/ElkJsonFormatter-0.0.1.tar.gz | Bin 0 -> 1415 bytes elk-json-formatter/distribute_tar_to_services.sh | 8 ++++++++ elk-json-formatter/requirements.txt | 1 + elk-json-formatter/setup.py | 8 ++++++++ square-backend/ElkJsonFormatter.tar.gz | Bin 0 -> 1415 bytes 8 files changed, 33 insertions(+) create mode 100644 elk-json-formatter/ElkJsonFormatter/ElkJsonFormatter.py create mode 100644 elk-json-formatter/ElkJsonFormatter/__init__.py create mode 100644 elk-json-formatter/README.md create mode 100644 elk-json-formatter/dist/ElkJsonFormatter-0.0.1.tar.gz create mode 100644 elk-json-formatter/distribute_tar_to_services.sh create mode 100644 elk-json-formatter/requirements.txt create mode 100644 elk-json-formatter/setup.py create mode 100644 square-backend/ElkJsonFormatter.tar.gz diff --git a/elk-json-formatter/ElkJsonFormatter/ElkJsonFormatter.py b/elk-json-formatter/ElkJsonFormatter/ElkJsonFormatter.py new file mode 100644 index 000000000..46016811c --- /dev/null +++ b/elk-json-formatter/ElkJsonFormatter/ElkJsonFormatter.py @@ -0,0 +1,10 @@ +from pythonjsonlogger import jsonlogger +from datetime import datetime; + +class ElkJsonFormatter(jsonlogger.JsonFormatter): + def add_fields(self, log_record, record, message_dict): + super(ElkJsonFormatter, self).add_fields(log_record, record, message_dict) + log_record['@timestamp'] = datetime.now().isoformat() + log_record['level'] = record.levelname + log_record['logger'] = record.name + log_record['application'] = 'MY-APPLICATION' diff --git a/elk-json-formatter/ElkJsonFormatter/__init__.py b/elk-json-formatter/ElkJsonFormatter/__init__.py new file mode 100644 index 000000000..33628f592 --- /dev/null +++ b/elk-json-formatter/ElkJsonFormatter/__init__.py @@ -0,0 +1 @@ +from .ElkJsonFormatter import ElkJsonFormatter diff --git a/elk-json-formatter/README.md b/elk-json-formatter/README.md new file mode 100644 index 000000000..6742c0a67 --- /dev/null +++ b/elk-json-formatter/README.md @@ -0,0 +1,5 @@ +This package is for formatting logs as JSON. + +To build run `python setup.py sdist`. + +When changing the code, please update the version in `setup.py` and the default version in the bash `distribute_tar_to_services.sh` script. Then run the script to distribute the tar ball to the listed services in the script (update if necessary) to make the new version installable. diff --git a/elk-json-formatter/dist/ElkJsonFormatter-0.0.1.tar.gz b/elk-json-formatter/dist/ElkJsonFormatter-0.0.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f8c2541be653cb98d37136fdce9244b1f9469a90 GIT binary patch literal 1415 zcmV;21$g=&iwFojc+g-1|72-%bT370Yf5u(Zbol%ZDDkDWpXVrE-)@JE_7jX0PS2` zZ<|OI&NIJalo#8Qjk)8^%2J~?-KK4vs7be4MG=|6V9$ozGD8~w`{iPM!A=|}UYd>1 z_kuZ`Ie_7OA7>aw?26rYXZZdT!48S{{u4+A9$HG0s?L6Ac3IYRc@N$1)qsR?9H+bA z_DTLvpqR+#)F*w}P&CEVB}o$vRW?mi*aN(M9C^2&Sr~i_V;^%);*KOrqTJnj{+qg9 zERA9*DcWMrFXX>w=*Aw>>&*W&2_E%J-CXzW<-hnU_+MM*e+Brz7XI(8;D1@t6x9F$ z*j@gwn!7Jejg9zUkz~mP{@)q?SBz!;SAqX);eTa;|FxcGiV`qC@PCE>s|GO|ZR0)Q`fLqnSm>{T$3mMi%Av za{YHfuszHes@^5dSp;!mL+e05DIDS=>^PQ7iRZ8;Bc9tvslOEyJB*z+DwjTC3=fIr zP@7NMW?)H_?xeccHp&9E#JR|8V@^ZPJTDvXvXe_MydO2LP=9*5Vi5jpwnWN8H{WG* zb!(5@5if68Sj5~AV4tjZ&;O=*{|8-h6nWIfoQ6T}-#GcJb9i?4_4xha#qsH&u>pUB z{)7I5{)7In(|^mN0p*spiS9pD|0Uh5=zj*#|Jw9ltX}R?;H!2ZhPMXj|AqeFrT$;V z?8&MM$qxEIE%(WA*r9%9v>ObKB2l@Z(%O9QN z!N=1r)_;mw?f+FwNrn2a4xA8<9n5j(PZBd41_ww^^M8PSaiMgLwroVne{8RNxX(3q@q<6&gwZ)rEVy+Qv$ z|3Uu&yP^N9i1&s1FG+Gm|8+xy@n5w7QXT4l(0|Z>(0|bXC+YuuCAFpcUs9Ed{%f+S zf&SNlV(51#8>ihVMj-e5O5VEVy8* zuKzIq)34M2Rr5b7z20vy{Ck)6{~4qFp0EE)wlshx&i|{kH-2 zKj?pVIxMUX{nw3Z{u{Ce^?xn67*U2IY~QA;jnbc~zM^7Q6HbF+HtPsOn4!<-rvnio zLAVIfO+q~f#Yup!i?;`;96E@YLm9u$9e<2Sfb0wH5RAr0$!*`{pWOE(h-l@ZH5Pue#pXFy#P5IdVnCLXu4 zrarzc;sxaA+)=0ZkLTeVk7WN2=s$e_{YvzI)%V}3s_BvrPXW8D{{;`*?ywI1m$W7Q z*Gw4yR|_u5aA;*OcCG}DhgPBOeUz`{`qoF1089Y&Kh*!bQUCXJsQ*F#%Q|dp`Tv;y ztBPLLf7OKk|5}jG|CV2E9)=z(Cw|K#r-Rx1@r{~00000 V00000006)u{smtI(n> literal 0 HcmV?d00001 diff --git a/elk-json-formatter/distribute_tar_to_services.sh b/elk-json-formatter/distribute_tar_to_services.sh new file mode 100644 index 000000000..69e658cd7 --- /dev/null +++ b/elk-json-formatter/distribute_tar_to_services.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Distributes tar ball of ElkJsonFormatter to parent directories listed in $services +# Please update version when creating new ElkJsonFormatter +version=${1:-"0.0.1"} +services=(square-backend) +for service in $services; do + cp dist/ElkJsonFormatter-${version}.tar.gz ../$service/ElkJsonFormatter.tar.gz +done diff --git a/elk-json-formatter/requirements.txt b/elk-json-formatter/requirements.txt new file mode 100644 index 000000000..c7fabe6ab --- /dev/null +++ b/elk-json-formatter/requirements.txt @@ -0,0 +1 @@ +python-json-logger==2.0.1 diff --git a/elk-json-formatter/setup.py b/elk-json-formatter/setup.py new file mode 100644 index 000000000..cd5d2414b --- /dev/null +++ b/elk-json-formatter/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup, find_packages + +setup( + name="ElkJsonFormatter", + version="0.0.1", + packages=find_packages(include=['ElkJsonFormatter']), + install_requires=["python-json-logger==2.0.1"] +) diff --git a/square-backend/ElkJsonFormatter.tar.gz b/square-backend/ElkJsonFormatter.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f8c2541be653cb98d37136fdce9244b1f9469a90 GIT binary patch literal 1415 zcmV;21$g=&iwFojc+g-1|72-%bT370Yf5u(Zbol%ZDDkDWpXVrE-)@JE_7jX0PS2` zZ<|OI&NIJalo#8Qjk)8^%2J~?-KK4vs7be4MG=|6V9$ozGD8~w`{iPM!A=|}UYd>1 z_kuZ`Ie_7OA7>aw?26rYXZZdT!48S{{u4+A9$HG0s?L6Ac3IYRc@N$1)qsR?9H+bA z_DTLvpqR+#)F*w}P&CEVB}o$vRW?mi*aN(M9C^2&Sr~i_V;^%);*KOrqTJnj{+qg9 zERA9*DcWMrFXX>w=*Aw>>&*W&2_E%J-CXzW<-hnU_+MM*e+Brz7XI(8;D1@t6x9F$ z*j@gwn!7Jejg9zUkz~mP{@)q?SBz!;SAqX);eTa;|FxcGiV`qC@PCE>s|GO|ZR0)Q`fLqnSm>{T$3mMi%Av za{YHfuszHes@^5dSp;!mL+e05DIDS=>^PQ7iRZ8;Bc9tvslOEyJB*z+DwjTC3=fIr zP@7NMW?)H_?xeccHp&9E#JR|8V@^ZPJTDvXvXe_MydO2LP=9*5Vi5jpwnWN8H{WG* zb!(5@5if68Sj5~AV4tjZ&;O=*{|8-h6nWIfoQ6T}-#GcJb9i?4_4xha#qsH&u>pUB z{)7I5{)7In(|^mN0p*spiS9pD|0Uh5=zj*#|Jw9ltX}R?;H!2ZhPMXj|AqeFrT$;V z?8&MM$qxEIE%(WA*r9%9v>ObKB2l@Z(%O9QN z!N=1r)_;mw?f+FwNrn2a4xA8<9n5j(PZBd41_ww^^M8PSaiMgLwroVne{8RNxX(3q@q<6&gwZ)rEVy+Qv$ z|3Uu&yP^N9i1&s1FG+Gm|8+xy@n5w7QXT4l(0|Z>(0|bXC+YuuCAFpcUs9Ed{%f+S zf&SNlV(51#8>ihVMj-e5O5VEVy8* zuKzIq)34M2Rr5b7z20vy{Ck)6{~4qFp0EE)wlshx&i|{kH-2 zKj?pVIxMUX{nw3Z{u{Ce^?xn67*U2IY~QA;jnbc~zM^7Q6HbF+HtPsOn4!<-rvnio zLAVIfO+q~f#Yup!i?;`;96E@YLm9u$9e<2Sfb0wH5RAr0$!*`{pWOE(h-l@ZH5Pue#pXFy#P5IdVnCLXu4 zrarzc;sxaA+)=0ZkLTeVk7WN2=s$e_{YvzI)%V}3s_BvrPXW8D{{;`*?ywI1m$W7Q z*Gw4yR|_u5aA;*OcCG}DhgPBOeUz`{`qoF1089Y&Kh*!bQUCXJsQ*F#%Q|dp`Tv;y ztBPLLf7OKk|5}jG|CV2E9)=z(Cw|K#r-Rx1@r{~00000 V00000006)u{smtI(n> literal 0 HcmV?d00001 From 58fe20d20adb486d0d500a111a5f643afcd10666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 13:40:12 +0200 Subject: [PATCH 04/16] logging --- logging/config/elasticsearch.yml | 2 + logging/config/filebeat.yml | 19 ++++++ logging/config/kibana.yml | 9 +++ logging/config/logstash.yml | 0 logging/docker-compose.yml | 80 +++++++++++++++++++++++++ logging/logstash/pipeline/logstash.conf | 22 +++++++ 6 files changed, 132 insertions(+) create mode 100644 logging/config/elasticsearch.yml create mode 100644 logging/config/filebeat.yml create mode 100644 logging/config/kibana.yml create mode 100644 logging/config/logstash.yml create mode 100644 logging/docker-compose.yml create mode 100644 logging/logstash/pipeline/logstash.conf diff --git a/logging/config/elasticsearch.yml b/logging/config/elasticsearch.yml new file mode 100644 index 000000000..50b154702 --- /dev/null +++ b/logging/config/elasticsearch.yml @@ -0,0 +1,2 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 diff --git a/logging/config/filebeat.yml b/logging/config/filebeat.yml new file mode 100644 index 000000000..f6bc3ff86 --- /dev/null +++ b/logging/config/filebeat.yml @@ -0,0 +1,19 @@ +filebeat.inputs: +- type: container + paths: + - '/var/lib/docker/containers/*/*.log' + +processors: +# - add_docker_metadata: +# host: "unix:///var/run/docker.sock" + +- decode_json_fields: + fields: ["message"] + target: "myapp" + overwrite_keys: true + +output.logstash: + hosts: ["logstash:5044"] + +logging.json: true +logging.metrics.enabled: false diff --git a/logging/config/kibana.yml b/logging/config/kibana.yml new file mode 100644 index 000000000..be462313f --- /dev/null +++ b/logging/config/kibana.yml @@ -0,0 +1,9 @@ +server.name: kibana +server.host: 0.0.0.0 +elasticsearch.hosts: [ "http://elasticsearch:9200" ] +monitoring.ui.container.elasticsearch.enabled: true + +## X-Pack security credentials +# +elasticsearch.username: elastic +elasticsearch.password: changeme diff --git a/logging/config/logstash.yml b/logging/config/logstash.yml new file mode 100644 index 000000000..e69de29bb diff --git a/logging/docker-compose.yml b/logging/docker-compose.yml new file mode 100644 index 000000000..c2a190672 --- /dev/null +++ b/logging/docker-compose.yml @@ -0,0 +1,80 @@ +version: "3.2" + +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2 + volumes: + - type: bind + source: ./config/elasticsearch.yml + target: /usr/share/elasticsearch/config/elasticsearch.yml + read_only: true + - type: volume + source: es-data + target: /usr/share/elasticsearch/data + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + ELASTIC_PASSWORD: changeme + discovery.type: single-node + networks: + - elk + logstash: + image: docker.elastic.co/logstash/logstash:7.13.2 + volumes: + - type: bind + source: ./config/logstash.yml + target: /usr/share/logstash/config/logstash.yml + read_only: true + - type: bind + source: ./logstash/pipeline + target: /usr/share/logstash/pipeline + read_only: true + ports: + - "5044:5044" + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + kibana: + image: docker.elastic.co/kibana/kibana:7.13.2 + volumes: + - type: bind + source: ./config/kibana.yml + target: /usr/share/kibana/config/kibana.yml + read_only: true + ports: + - "5601:5601" + networks: + - elk + depends_on: + - elasticsearch + + filebeat: + image: "docker.elastic.co/beats/filebeat:7.13.2" + user: root + volumes: + - type: bind + source: ./config/filebeat.yml + target: /usr/share/filebeat/filebeat.yml + read_only: true + - type: bind + source: /var/lib/docker + target: /var/lib/docker + read_only: true + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + networks: + - elk + depends_on: + - elasticsearch + +networks: + elk: + driver: bridge + +volumes: + es-data: + driver_opts: + device: ./es-data diff --git a/logging/logstash/pipeline/logstash.conf b/logging/logstash/pipeline/logstash.conf new file mode 100644 index 000000000..25086ce58 --- /dev/null +++ b/logging/logstash/pipeline/logstash.conf @@ -0,0 +1,22 @@ +input { + beats { + port => 5044 + } +} + +filter { + if [myapp][application] != "MY-APPLICATION" { + drop { } + } +} + +output { + elasticsearch { + hosts => "elasticsearch:9200" + user => "elastic" + password => "changeme" + ecs_compatibility => disabled + index => "my-application-logs-4" + } +} + From e25b4119ee8b35b7a11fe1f76369e2956ccec58a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 14:08:54 +0200 Subject: [PATCH 05/16] set application to square --- .../ElkJsonFormatter/ElkJsonFormatter.py | 2 +- .../distribute_tar_to_services.sh | 2 +- elk-json-formatter/setup.py | 2 +- logging/logstash/pipeline/logstash.conf | 4 ++-- square-backend/ElkJsonFormatter.tar.gz | Bin 1415 -> 1409 bytes 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elk-json-formatter/ElkJsonFormatter/ElkJsonFormatter.py b/elk-json-formatter/ElkJsonFormatter/ElkJsonFormatter.py index 46016811c..feb8eff7e 100644 --- a/elk-json-formatter/ElkJsonFormatter/ElkJsonFormatter.py +++ b/elk-json-formatter/ElkJsonFormatter/ElkJsonFormatter.py @@ -7,4 +7,4 @@ def add_fields(self, log_record, record, message_dict): log_record['@timestamp'] = datetime.now().isoformat() log_record['level'] = record.levelname log_record['logger'] = record.name - log_record['application'] = 'MY-APPLICATION' + log_record['application'] = 'SQUARE' diff --git a/elk-json-formatter/distribute_tar_to_services.sh b/elk-json-formatter/distribute_tar_to_services.sh index 69e658cd7..7ec103dac 100644 --- a/elk-json-formatter/distribute_tar_to_services.sh +++ b/elk-json-formatter/distribute_tar_to_services.sh @@ -1,7 +1,7 @@ #!/bin/bash # Distributes tar ball of ElkJsonFormatter to parent directories listed in $services # Please update version when creating new ElkJsonFormatter -version=${1:-"0.0.1"} +version=${1:-"0.0.2"} services=(square-backend) for service in $services; do cp dist/ElkJsonFormatter-${version}.tar.gz ../$service/ElkJsonFormatter.tar.gz diff --git a/elk-json-formatter/setup.py b/elk-json-formatter/setup.py index cd5d2414b..2e2c53dda 100644 --- a/elk-json-formatter/setup.py +++ b/elk-json-formatter/setup.py @@ -2,7 +2,7 @@ setup( name="ElkJsonFormatter", - version="0.0.1", + version="0.0.2", packages=find_packages(include=['ElkJsonFormatter']), install_requires=["python-json-logger==2.0.1"] ) diff --git a/logging/logstash/pipeline/logstash.conf b/logging/logstash/pipeline/logstash.conf index 25086ce58..6b6be512a 100644 --- a/logging/logstash/pipeline/logstash.conf +++ b/logging/logstash/pipeline/logstash.conf @@ -5,7 +5,7 @@ input { } filter { - if [myapp][application] != "MY-APPLICATION" { + if [myapp][application] != "SQUARE" { drop { } } } @@ -16,7 +16,7 @@ output { user => "elastic" password => "changeme" ecs_compatibility => disabled - index => "my-application-logs-4" + index => "square-logs" } } diff --git a/square-backend/ElkJsonFormatter.tar.gz b/square-backend/ElkJsonFormatter.tar.gz index f8c2541be653cb98d37136fdce9244b1f9469a90..7f910ebb746e84ebcd7c934df0de163798f538f4 100644 GIT binary patch delta 1398 zcmV-+1&R8H3xNv;ABzYGGl7u?9)B_}bYXG;?VMY4qc#-B^USBvJVRP{ue;#p&05k^d8E7795H$xt;lU4OGIZFg7J z`vY^JOIx7#uY=(F3yY&qaT;RIN!nFpMb>(2_kYVU%B@*$6-{6E`6d6?O~cwkMw9+8 zoZ!K7qL}BsKmV5@{zLq)$A4{LSt`W;4UYeHbMLjT@vQi-i3p%T{NI@PubHdyzYp=h zk@&AI$A4Ap%L;^ki2t?tUw=2q$>dqCVe0z5@!wKw{;%~dRXqPIP5S?a#(yu3LzGPT zIF9}n_cw@rpQMO}Nt|*tw@5`f7ju#ai|Tl_|BEC!0cH%A{>&tix)rBV)=WE z;v@;EgE@_(qI~P*&$IpGgVqb;Ht-+#zrcTepj*Iy;QtE$ZJS1v+xBy~|5W}fhE?Og zP#^f;82{z^?XCb{w|_%mcyD?e|EKkJ(**uEq2q)7kKYdD(3KeBSt2JB=_&92rH5V+NRK}~N>A!w z50r9ALL!$$&VR%XXmrgUDV~ylGMbXdit#wHi?_7T>E6J9;6Lymv>E)rEAYOO|B9m4 z_-~ko0{m}6;Odb71OI{lz<=QXllZ@wNv$dWSNd9w|GH`!!2c#xPW|rY^R&C=4CLWZ z6I}rQ{s;UA{saF(!2j0{kv__Qt)BnueGT~kQ|>>wK7a7PasI>n@1Mv2JLi8}yZS(b zP_ViDf59xjm-ByR{-0`CrV9LTLNNai_z(F%HcU=*!9BNJdDKPGfYH(ZEG%7NN`X?E$K$ z4kG4K#(yu1g5Sp^Le3aRe%=lr6Xe7$>7XPam=TmE`D!YK{ib}XKvXnXwRDMa;k;&PL_9|G z+Qk9Uf@n4f@3L(|KD={KiCx?;VEEq`CrCi$Mc)uzoM`3-%w!wUn4pvzHjF*b}l52eY<4) z5EVNG--bw$AQC|S5BdKlaKp+qZ1OkCTAkeb@1%y^4VE}jl E0QlAJ+W-In delta 1404 zcmV-?1%vv53x^8^ABzYGJa~}?9)B?|bYXG;?Oa=Ln@AMSGrwY#7u%AJx#P{sQlmEA zrfr<4Nw-=>5t+bX&xYGFLmL14w=*Aw>>&*W& z2_E%J-CXzW<-hnU_+MM*e+Brz7XI(8;D1@t6x9F$*j@gwn!7Jejg9zUkz~mP{@)q? zSBz!;SAqX);eTa;|FxcGihmL?Kk$Er|EmTu8gG;uhNeBp|7JD+71fke{+H^^{~hIj zHx7Lijrk}H{!Xv27Y>IcM%0hOn4_6R$o(A5Nk$gs_HzAqL9ji{7^>bS%~=F-VMFUc zKq(yJBJ4PpONr;OCL^BPMybCQ6FZEZHY%4sVGIw67cOze}n#m{)7I5{;$)2%c240mbHoQ zKUM!F-K^+;2GIZ7^nYKhUhY!ht9BrUw+86{h5p~A{$IuH$*Ky;4*EYW_sMYBp@AEA zpP3q)_5aF-Bt!r2&hS6ff3Kzf%lRMbKj8ld`Ty+8AD!dD$I~s=e~MY{|5Z#$h5D}! zoDhy3%yH*W5;GbG2S`rye}H{*uw4C3ee#<`I8S^Z$KwO^eSh#}aQb5)diMgLwroVne{8RNxxxGRELH|Mj0lT69tBCi7`Y%ax zMgMg}gYjRr08$<5f6#x>f6#x>|0n7Hd?mG|`d?C&ivDY|se%62fnw-)CmW~TDMldo z`%2!s54i#Q5Bd-K4*>mtSr_SA{a32>zosgn|G(t=Lx1Xn{@1SmF#pr9)BjcTKPkQ5 zZ!r9Om-YV{qx_z)|4ZZlWZg6*(EmCBVKJqN`}fUb+T z2dEr6h<}(v8NbdQe~d_g>=6!zSsXqh$POLSMv+G_BPfZo*;I1-UGY?bXd1ALbd7M} z6?gUYYp@vk|(^b{Er|gcB>hiWUzoM&i5lVS+6d_x)`4eNQ?rfl6yHxmby5z9SK+BYX>K!05n5IdVnCLXu4rarzc;sxaA+)=0Z zkLTeVk7WN2=s$e_{YvzI)%V}3s_BvrPXW8D{{;`*?ywI1m$W7Q*Gw4yR|_u5aA;*O zcCG}DhgPBOeUz`{`qoF1089Y&Kh*!bQUCXJsQ*F#%Q|dp`Tv;ytBPLLf7OKk|5}jG z|816EZ61amD<^)-Ep6n|z_BL1157AB5p3pDl4f(0_V=rI&we}KIH`L3`}xf@F_vNa z3qhMSu)V|~{mVu*SmUZyw44;rRyp*vf4TqQu*Ll=p#}5>00000000000002MBK`$m KJ4%ZHcmM!DNaf%F From bd7fd66ca8dd1d43869799bbdf46e85b1cf6f429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 14:09:22 +0200 Subject: [PATCH 06/16] add ElkJsonFormatter tar --- square-backend/Dockerfile | 3 +++ square-backend/requirements.txt | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/square-backend/Dockerfile b/square-backend/Dockerfile index d70aad4fb..83a5e6e91 100644 --- a/square-backend/Dockerfile +++ b/square-backend/Dockerfile @@ -4,6 +4,7 @@ RUN pip install --upgrade pip WORKDIR /app +COPY ElkJsonFormatter.tar.gz ./ElkJsonFormatter.tar.gz COPY requirements.txt requirements.txt RUN pip install -r requirements.txt # not in requirements because we use SQLite for dev server @@ -31,5 +32,7 @@ RUN pytest \ FROM base as build +COPY logging.conf logging.conf + RUN ["chmod", "+x", "./entry.sh"] ENTRYPOINT ["./entry.sh"] diff --git a/square-backend/requirements.txt b/square-backend/requirements.txt index 8a000f709..54fe19d77 100644 --- a/square-backend/requirements.txt +++ b/square-backend/requirements.txt @@ -1,6 +1,6 @@ Flask SQLAlchemy -Flask-Migrate +Flask-Migrate==2.7.0 Flask-Script Flask-CORS flask-jwt-extended==3.25.1 @@ -10,3 +10,4 @@ flasgger eventlet pyyaml elasticsearch>=7.0.0 +ElkJsonFormatter.tar.gz From 28b708591fe43cdd3d0bd80192ff357bfb7eac9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 14:10:09 +0200 Subject: [PATCH 07/16] use file config for logging --- square-backend/docker/entry.sh | 2 +- square-backend/logging.conf | 21 ++++++++++++++++ square-backend/logging_config.yaml | 40 ------------------------------ square-backend/main.py | 7 ++++-- square-backend/squareapi/app.py | 4 +-- 5 files changed, 28 insertions(+), 46 deletions(-) create mode 100644 square-backend/logging.conf delete mode 100644 square-backend/logging_config.yaml diff --git a/square-backend/docker/entry.sh b/square-backend/docker/entry.sh index 0cc59d1b2..e44446947 100644 --- a/square-backend/docker/entry.sh +++ b/square-backend/docker/entry.sh @@ -10,4 +10,4 @@ python flask-manage.py db migrate --directory=migration python flask-manage.py db upgrade --directory=migration echo "Starting the server" -python main.py --host "0.0.0.0" +python main.py --host "0.0.0.0" --logging_conf logging.conf diff --git a/square-backend/logging.conf b/square-backend/logging.conf new file mode 100644 index 000000000..d62db8187 --- /dev/null +++ b/square-backend/logging.conf @@ -0,0 +1,21 @@ +[loggers] +keys = root + +[logger_root] +level = DEBUG +handlers = root + +[handlers] +keys = root + +[handler_root] +class = StreamHandler +level = DEBUG +formatter = json + +[formatters] +keys = json + +[formatter_json] +class = ElkJsonFormatter.ElkJsonFormatter + diff --git a/square-backend/logging_config.yaml b/square-backend/logging_config.yaml deleted file mode 100644 index 4a975fcab..000000000 --- a/square-backend/logging_config.yaml +++ /dev/null @@ -1,40 +0,0 @@ -version: 1 -formatters: - simple: - format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' -handlers: - console: - class: logging.StreamHandler - level: DEBUG - formatter: simple - stream: ext://sys.stdout - file: - class: logging.handlers.RotatingFileHandler - level: INFO - formatter: simple - filename: logs/app.log - maxBytes: 1000000 - backupCount: 10 - access_file: - class: logging.handlers.RotatingFileHandler - level: INFO - filename: logs/access.log - maxBytes: 1000000 - backupCount: 5 -loggers: - console: - level: INFO - handlers: [console] - propagate: no - file: - level: INFO - handlers: [file] - propagate: no - server: - level: INFO - handlers: [console] - propagate: no -root: - level: INFO - handlers: [console] -disable_existing_loggers: False \ No newline at end of file diff --git a/square-backend/main.py b/square-backend/main.py index 4fdfa2f51..ac4bcd533 100644 --- a/square-backend/main.py +++ b/square-backend/main.py @@ -3,14 +3,17 @@ from squareapi.app import create_app import argparse import logging +import logging.config if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--host", type=str, default="127.0.0.1") parser.add_argument("--config", type=str, default="./config.yaml") - parser.add_argument("--logging_config", type=str, default="./logging_config.yaml") + parser.add_argument("--logging_config", type=str, default="./logging.conf") args = parser.parse_args() - socketio, app = create_app(config_path=args.config, logging_config_path=args.logging_config) + logging.config.fileConfig(args.logging_config) + + socketio, app = create_app(config_path=args.config) socketio.run(app, host=args.host, log_output=True, log=logging.getLogger("server")) diff --git a/square-backend/squareapi/app.py b/square-backend/squareapi/app.py index 39206b88a..7d9432e33 100644 --- a/square-backend/squareapi/app.py +++ b/square-backend/squareapi/app.py @@ -1,5 +1,4 @@ import logging -import logging.config import yaml from flask import Flask from flask_cors import CORS @@ -10,8 +9,7 @@ logger = logging.getLogger(__name__) -def create_app(app_name="SQUARE_API", config_path="./config.yaml", logging_config_path="./logging_config.yaml"): - logging.config.dictConfig(yaml.load(open(logging_config_path))) +def create_app(app_name="SQUARE_API", config_path="./config.yaml"): logger.info("Creating Flask App") config = yaml.load(open(config_path)) From d91de0ec2bff42b811b9b338155acca835897225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 14:19:12 +0200 Subject: [PATCH 08/16] driver type added --- logging/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/logging/docker-compose.yml b/logging/docker-compose.yml index c2a190672..ddd1212a7 100644 --- a/logging/docker-compose.yml +++ b/logging/docker-compose.yml @@ -76,5 +76,6 @@ networks: volumes: es-data: + type: local driver_opts: device: ./es-data From e3b08656c6bfd565963e4bf094d07a7a1ba01d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 15:06:42 +0200 Subject: [PATCH 09/16] set target to square --- logging/config/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logging/config/filebeat.yml b/logging/config/filebeat.yml index f6bc3ff86..aa430fb9e 100644 --- a/logging/config/filebeat.yml +++ b/logging/config/filebeat.yml @@ -9,7 +9,7 @@ processors: - decode_json_fields: fields: ["message"] - target: "myapp" + target: "square" overwrite_keys: true output.logstash: From d6aeec070e724e4ffb6b3c64cd18304d79cd0029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 15:06:55 +0200 Subject: [PATCH 10/16] remove docker logging file --- square-backend/docker/logging_config.yaml | 40 ----------------------- 1 file changed, 40 deletions(-) delete mode 100644 square-backend/docker/logging_config.yaml diff --git a/square-backend/docker/logging_config.yaml b/square-backend/docker/logging_config.yaml deleted file mode 100644 index c8853ae58..000000000 --- a/square-backend/docker/logging_config.yaml +++ /dev/null @@ -1,40 +0,0 @@ -version: 1 -formatters: - simple: - format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' -handlers: - console: - class: logging.StreamHandler - level: WARNING - formatter: simple - stream: ext://sys.stdout - file: - class: logging.handlers.RotatingFileHandler - level: INFO - formatter: simple - filename: logs/app.log - maxBytes: 1000000 - backupCount: 10 - access_file: - class: logging.handlers.RotatingFileHandler - level: INFO - filename: logs/access.log - maxBytes: 1000000 - backupCount: 5 -loggers: - console: - level: DEBUG - handlers: [console] - propagate: no - file: - level: DEBUG - handlers: [file] - propagate: no - server: - level: INFO - handlers: [console,access_file] - propagate: no -root: - level: DEBUG - handlers: [console,file] -disable_existing_loggers: False \ No newline at end of file From 7e59293ab4b25e0f30da31f94ef625f9f4f86a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 15:07:14 +0200 Subject: [PATCH 11/16] do not disable existing loggers --- square-backend/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/square-backend/main.py b/square-backend/main.py index ac4bcd533..3f9be2f6e 100644 --- a/square-backend/main.py +++ b/square-backend/main.py @@ -12,7 +12,7 @@ parser.add_argument("--logging_config", type=str, default="./logging.conf") args = parser.parse_args() - logging.config.fileConfig(args.logging_config) + logging.config.fileConfig(args.logging_config, disable_existing_loggers=False) socketio, app = create_app(config_path=args.config) From 27e449910050485795b3a625e0fb98494e3f360e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 15:08:03 +0200 Subject: [PATCH 12/16] filebeat.yml ignore permissions --- logging/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/logging/docker-compose.yml b/logging/docker-compose.yml index ddd1212a7..613f4fbc8 100644 --- a/logging/docker-compose.yml +++ b/logging/docker-compose.yml @@ -52,6 +52,7 @@ services: filebeat: image: "docker.elastic.co/beats/filebeat:7.13.2" + command: --strict.perms=false user: root volumes: - type: bind From 73be6b10099e1351fb4867f818b4f33384c67636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 15:08:14 +0200 Subject: [PATCH 13/16] logging-es-data volume --- logging/docker-compose.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/logging/docker-compose.yml b/logging/docker-compose.yml index 613f4fbc8..cd63b03eb 100644 --- a/logging/docker-compose.yml +++ b/logging/docker-compose.yml @@ -9,7 +9,7 @@ services: target: /usr/share/elasticsearch/config/elasticsearch.yml read_only: true - type: volume - source: es-data + source: logging-es-data target: /usr/share/elasticsearch/data environment: ES_JAVA_OPTS: "-Xmx256m -Xms256m" @@ -76,7 +76,4 @@ networks: driver: bridge volumes: - es-data: - type: local - driver_opts: - device: ./es-data + logging-es-data: From 6e48d95543f7184db6092759b87779d98bf8a5da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 15:47:49 +0200 Subject: [PATCH 14/16] logstash pipeline replace myapp with square --- logging/logstash/pipeline/logstash.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logging/logstash/pipeline/logstash.conf b/logging/logstash/pipeline/logstash.conf index 6b6be512a..57fc022c4 100644 --- a/logging/logstash/pipeline/logstash.conf +++ b/logging/logstash/pipeline/logstash.conf @@ -5,7 +5,7 @@ input { } filter { - if [myapp][application] != "SQUARE" { + if [square][application] != "SQUARE" { drop { } } } From 7e7f682544fe2d046de93baaeec48dc90e2716b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 15:49:21 +0200 Subject: [PATCH 15/16] change to db --- square-backend/docker/config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/square-backend/docker/config.yaml b/square-backend/docker/config.yaml index 4295075b0..3e42891ac 100644 --- a/square-backend/docker/config.yaml +++ b/square-backend/docker/config.yaml @@ -1,6 +1,6 @@ --- DEBUG: false -SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://square:dbpw@square-core_db_1:5432/square +SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://square:dbpw@db:5432/square SQLALCHEMY_TRACK_MODIFICATIONS: false JWT_IDENTITY_CLAIM: sub JWT_SECRET_KEY: change this key to something else @@ -9,10 +9,10 @@ skillSelector: selectors: - name: Base config: {} - - name: Elasticsearch - config: - k: 50 - weighting: score - elasticsearch: - url: "http://square-core_es_1:9200" - index: square_selector_data \ No newline at end of file + # - name: Elasticsearch + # config: + # k: 50 + # weighting: score + # elasticsearch: + # url: "http://square-core_es_1:9200" + # index: square_selector_data From aa537efe758ce48c3c8fdd2920e334a960c24a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Baumg=C3=A4rtner?= Date: Mon, 21 Jun 2021 15:58:58 +0200 Subject: [PATCH 16/16] add -e --- logging/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logging/docker-compose.yml b/logging/docker-compose.yml index cd63b03eb..9f09e1851 100644 --- a/logging/docker-compose.yml +++ b/logging/docker-compose.yml @@ -52,7 +52,7 @@ services: filebeat: image: "docker.elastic.co/beats/filebeat:7.13.2" - command: --strict.perms=false + command: -e --strict.perms=false user: root volumes: - type: bind