Push alerts to elastic search (#108) (#118)

* add elasticsearch stack to github actions * added kind config to map nodeports and removed portforwarding remove node settings added authentication added auth * push alerts to elastic search push alerts to elastic search elastic env in action nit echo nit test test lint fix * push metrics to elastic search metric structure change added upload_metric * push telemetry to elastic + refactoring remove warning fixed model error added module export from elastic models namespace minor checks minor checks changed method name fix tests changed check on metric type reduced elastic log level float casting fixed telemetry model minor nits test fix poetry.lock * export modules lint * taint constructor update * metrics model refactoring lint * elastic library log level * downgraded elastic search to 7.x fix --------- Signed-off-by: Tullio Sebastiani <[email protected]>
krkn-chaos · Aug 14, 2024 · 32dfb12 · 32dfb12
1 parent aceeef4
commit 32dfb12
Show file tree

Hide file tree

Showing 14 changed files with 953 additions and 115 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -31,8 +31,9 @@ jobs:
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo add stable https://charts.helm.sh/stable
           helm repo update
-      - name: Deploy prometheus & Port Forwarding
+      - name: Deploy prometheus
         run: |
+          # nodePort mapping 30000 -> http://localhost:9090
           kubectl create namespace monitoring
           helm install \
           --wait --timeout 360s \
@@ -47,11 +48,36 @@ jobs:
           --set alertmanager.service.type=NodePort \
           --set prometheus-node-exporter.service.nodePort=32001 \
           --set prometheus-node-exporter.service.type=NodePort
+      - name: Deploy Elasticsearch
 
-          SELECTOR=`kubectl -n monitoring get service kind-prometheus-kube-prome-prometheus -o wide --no-headers=true | awk '{ print $7 }'`
-          POD_NAME=`kubectl -n monitoring get pods --selector="$SELECTOR" --no-headers=true | awk '{ print $1 }'`
-          kubectl -n monitoring port-forward $POD_NAME 9090:9090 &
-          sleep 5
+        env:
+          ELASTIC_URL: ${{ vars.ELASTIC_URL }}
+          ELASTIC_PORT: ${{ vars.ELASTIC_PORT }}
+          ELASTIC_USER: ${{ vars.ELASTIC_USER }}
+          ELASTIC_PASSWORD: ${{ vars.ELASTIC_PASSWORD }}
+        run: |
+          echo "ELASTIC_URL: ${ELASTIC_URL}"
+          echo "ELASTIC_PORT: ${ELASTIC_PORT}"
+          echo "ELASTIC_USER: ${ELASTIC_USER}"
+          echo "ELASTIC_PASSWORD:${ELASTIC_PASSWORD}"
+          
+          
+          # nodePort mapping 32766 -> http://localhost:9091
+          helm install \
+          --wait --timeout 360s \
+          elasticsearch \
+          oci://registry-1.docker.io/bitnamicharts/elasticsearch \
+          --set master.masterOnly=false \
+          --set master.replicaCount=1 \
+          --set data.replicaCount=0 \
+          --set coordinating.replicaCount=0 \
+          --set ingest.replicaCount=0 \
+          --set service.type=NodePort \
+          --set service.nodePorts.restAPI=32766 \
+          --set security.elasticPassword=test \
+          --set security.enabled=true \
+          --set image.tag=7.17.23-debian-12-r0 \
+          --set security.tls.autoGenerated=true
       - name: Check out code
         uses: actions/checkout@v3
       - name: Update version number
@@ -78,7 +104,11 @@ jobs:
           BUCKET_NAME: ${{ secrets.BUCKET_NAME }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          ES_SERVER: ${{ secrets.ES_SERVER }}
+          ELASTIC_URL: ${{ vars.ELASTIC_URL }}
+          ELASTIC_PORT: ${{ vars.ELASTIC_PORT }}
+          ELASTIC_USER: ${{ vars.ELASTIC_USER }}
+          ELASTIC_PASSWORD: ${{ vars.ELASTIC_PASSWORD }}
+
         run: |
           export TEST_WORKDIR=`pwd`/`date +%s`
           mkdir $TEST_WORKDIR

diff --git a/kind-config.yml b/kind-config.yml
@@ -0,0 +1,9 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraPortMappings:
+      - containerPort: 30000
+        hostPort: 9090
+      - containerPort: 32766
+        hostPort: 9091
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -21,7 +21,8 @@ tzlocal = "5.1"
 pytz = "^2023.3"
 PyYAML = "6.0.1"
 prometheus-api-client = "^0.5.4"
-elasticsearch = "8.14.0"
+elasticsearch = "7.13.4"
+elasticsearch-dsl = "7.4.1"
 wheel = "^0.42.0"
 cython = "3.0"
 

diff --git a/src/krkn_lib/elastic/__init__.py b/src/krkn_lib/elastic/__init__.py
@@ -0,0 +1 @@
+from .krkn_elastic import *  # NOQA