Dev/elk k8s setup

dev/docker-compose.yml

@@ -98,7 +98,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5020"
+#        syslog-address: "tcp://192.168.38.128:30520"
 
   exareme2_global:
     image: madgik/exareme2_worker:${EXAREME2}
@@ -129,7 +129,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5020"
+#        syslog-address: "tcp://192.168.38.128:30520"
 
   exareme2_controller:
     image: madgik/exareme2_controller:${EXAREME2}
@@ -157,7 +157,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5020"
+#        syslog-address: "tcp://192.168.38.128:30520"
 
   portalbackend_db:
     image: postgres:11.20-alpine
@@ -219,7 +219,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5020"
+#        syslog-address: "tcp://192.168.38.128:30520"
 
   gateway-db:
     image: postgres

elk_stack/docker-compose.yml

@@ -22,8 +22,8 @@ services:
     image: docker.elastic.co/logstash/logstash:8.15.0
     container_name: logstash
     volumes:
-      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
-      - ./logstash/pipeline:/usr/share/logstash/pipeline
+      - ./kubernetes/files/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
+      - ./kubernetes/files/logstash/pipeline:/usr/share/logstash/pipeline
     environment:
       - "xpack.security.enabled=false"  # Disable security
     ports:

elk_stack/kubernetes/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: elk
+description: A helm chart for Kubernetes deployment of the ELK stack
+version: 0.0.1
+type: application

elk_stack/kubernetes/README.md

@@ -0,0 +1 @@
+# ELK Deployment

elk_stack/kubernetes/files/logstash/config/logstash.yml

@@ -0,0 +1,2 @@
+http.host: "0.0.0.0"  # Limit access to localhost or use a specific interface
+xpack.monitoring.enabled: false # Enable monitoring to track performance in production

elk_stack/kubernetes/templates/elastic.yaml

@@ -0,0 +1,106 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: elasticsearch-vol
+  namespace: {{ .Values.namespace }}
+  labels:
+    storage: elasticsearch-storage
+spec:
+  capacity:
+    storage: {{ .Values.elasticsearch.storage.data_size }}
+  volumeMode: Filesystem
+  accessModes:
+  - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: k8s-local-storage
+  hostPath:
+    path: {{ .Values.elasticsearch.storage.data_path }}
+    type: DirectoryOrCreate
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: master
+          operator: In
+          values:
+          - "true"
+
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: elasticsearch-storage-claim
+  namespace: {{ .Values.namespace }}
+  labels:
+    storage: elasticsearch-storage
+spec:
+  selector:
+    matchLabels:
+      storage: elasticsearch-storage
+  storageClassName: k8s-local-storage
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.elasticsearch.storage.data_size }}
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elasticsearch
+  namespace: {{ .Values.namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      initContainers:
+      - name: fix-permissions
+        image: busybox
+        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data && chmod -R 775 /usr/share/elasticsearch/data"]
+        volumeMounts:
+        - name: elasticsearch-data
+          mountPath: /usr/share/elasticsearch/data
+      containers:
+      - name: elasticsearch
+        image: docker.elastic.co/elasticsearch/elasticsearch:{{ .Values.elk_version }}
+        ports:
+        - containerPort: 9200
+        env:
+        - name: discovery.type
+          value: single-node
+        - name: xpack.security.enabled
+          value: "{{ .Values.security.enabled }}"
+        - name: xpack.security.http.ssl.enabled
+          value: "{{ .Values.security.ssl_enabled }}"
+        - name: ES_JAVA_OPTS 
+          value: {{ .Values.elasticsearch.java_opts }}
+        volumeMounts:
+        - name: elasticsearch-data
+          mountPath: /usr/share/elasticsearch/data
+      volumes:
+      - name: elasticsearch-data
+        persistentVolumeClaim:
+          claimName: elasticsearch-storage-claim
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  namespace: {{ .Values.namespace }}
+spec:
+  selector:
+    app: elasticsearch
+  ports:
+  - port: 9200
+    targetPort: 9200

elk_stack/kubernetes/templates/kibana.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kibana
+  namespace: {{ .Values.namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kibana
+  template:
+    metadata:
+      labels:
+        app: kibana
+    spec:
+      containers:
+      - name: kibana
+        image: docker.elastic.co/kibana/kibana:{{ .Values.elk_version }}
+        ports:
+        - containerPort: 5601
+        env:
+        - name: xpack.security.enabled
+          value: "{{ .Values.security.enabled }}"
+        - name: ELASTICSEARCH_HOSTS
+          value: http://elasticsearch:9200
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana
+  namespace: {{ .Values.namespace }}
+spec:
+  type: LoadBalancer
+  selector:
+    app: kibana
+  ports:
+  - protocol: TCP
+    port: 5601
+    targetPort: 5601
+    nodePort: 31000

elk_stack/kubernetes/templates/logstash.yaml

@@ -0,0 +1,107 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logstash-pipeline-config
+  namespace: {{ .Values.namespace }}
+data:
+  logstash.conf: |
+{{ .Files.Get "files/logstash/pipeline/logstash.conf" | indent 4 }}
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logstash-config
+  namespace: {{ .Values.namespace }}
+data:
+  logstash.yml: |
+{{ .Files.Get "files/logstash/config/logstash.yml" | indent 4 }}
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logstash
+  namespace: {{ .Values.namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: logstash
+  template:
+    metadata:
+      labels:
+        app: logstash
+    spec:
+      containers:
+      - name: logstash
+        image: docker.elastic.co/logstash/logstash:{{ .Values.elk_version }}
+        ports:
+        - containerPort: 5044
+        - containerPort: 5010     # Consume k8s logs (TCP)
+        - containerPort: 5020     # Consume docker logs (TCP)
+        {{- if .Values.logstash.expose_monitoring_port }}
+        - containerPort: 9600
+        {{- end }}
+        env:
+        - name: xpack.security.enabled
+          value: "{{ .Values.security.enabled }}"
+        volumeMounts:
+        - name: logstash-pipeline-config
+          mountPath: /usr/share/logstash/pipeline/logstash.conf
+          subPath: logstash.conf
+          readOnly: true
+        - name: logstash-config
+          mountPath: /usr/share/logstash/config/logstash.yml
+          subPath: logstash.yml
+          readOnly: true
+      volumes:
+      - name: logstash-pipeline-config
+        configMap:
+          name: logstash-pipeline-config
+      - name: logstash-config
+        configMap:
+          name: logstash-config
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: logstash
+  namespace: {{ .Values.namespace }}
+spec:
+  selector:
+    app: logstash
+  ports:
+  - name: logstash
+    port: 5044
+    targetPort: 5044
+  - name: logstash-monitoring
+    port: 9600
+    targetPort: 9600
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: logstash-log-consumer
+  namespace: {{ .Values.namespace }}
+spec:
+  type: LoadBalancer
+  selector:
+    app: logstash
+  ports:
+  - name: k8s-logs-consumer
+    protocol: TCP
+    port: 5010
+    targetPort: 5010
+    nodePort: {{ .Values.logstash.k8s_logs_consumer_port }}
+  - name: docker-logs-consumer
+    protocol: TCP
+    port: 5020
+    targetPort: 5020
+    nodePort: {{ .Values.logstash.docker_logs_consumer_port }}

elk_stack/kubernetes/templates/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ .Values.namespace }}

elk_stack/kubernetes/values.yaml

@@ -0,0 +1,18 @@
+namespace: elk
+
+elk_version: 8.15.0
+
+security:
+  enabled: false      # Enable security
+  ssl_enabled: false  # Enable https
+
+elasticsearch:
+  java_opts: '-Xms1g -Xmx1g'    # Define memory for elastisearch jvm
+  storage:
+    data_path: /opt/mip-deployment/.stored_data/elasticsearch
+    data_size: 10Gi
+
+logstash:
+  expose_monitoring_port: false   # Expose only for monitoring logstash performance
+  k8s_logs_consumer_port: 30510
+  docker_logs_consumer_port: 30520