From b66b4c5da3ec75369d6e6841afd79b8568469451 Mon Sep 17 00:00:00 2001
From: Kumar Shivendu <kshivendu1@gmail.com>
Date: Tue, 11 Jun 2024 21:33:37 +0530
Subject: [PATCH] feat: Add 15m timeout for CI benchmarks (#157)

* feat: Add 15m timeout for CI benchmarks

* feat: Add Slack alert for CI benchmarks (#158)

* feat: Add Slack alert for CI benchmarks

* Update continuous-benchmark.yaml

* fix: Use quotes on func name to declare they arent commands

* feat: Separate failure reasons

* fix: YAML error
---
 .github/workflows/continuous-benchmark.yaml | 32 +++++++++++++++++++--
 tools/run_ci.sh                             | 13 +++++++++
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/continuous-benchmark.yaml b/.github/workflows/continuous-benchmark.yaml
index 370b8197..a559d378 100644
--- a/.github/workflows/continuous-benchmark.yaml
+++ b/.github/workflows/continuous-benchmark.yaml
@@ -16,6 +16,7 @@ jobs:
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
       - name: Benches
+        id: benches
         run: |
             export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
             export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
@@ -28,15 +29,42 @@ jobs:
             DATASET_TO_ENGINE["h-and-m-2048-angular-filters"]="qdrant-continuous-benchmark"
             DATASET_TO_ENGINE["dbpedia-openai-100K-1536-angular"]="qdrant-bq-continuous-benchmark"
 
+            set +e
+
             for dataset in "${!DATASET_TO_ENGINE[@]}"; do
               export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]}
               export DATASETS=$dataset
 
               # Benchmark the dev branch:
               export QDRANT_VERSION=ghcr/dev
-              bash -x tools/run_ci.sh
+              timeout 15m bash -x tools/run_ci.sh
 
               # Benchmark the master branch:
               export QDRANT_VERSION=docker/master
-              bash -x tools/run_ci.sh
+              timeout 15m bash -x tools/run_ci.sh
             done
+
+            set -e
+      - name: Fail job if any of the benches failed
+        if: steps.benches.outputs.failed == 'error' || steps.benches.outputs.failed == 'timeout'
+        run: exit 1
+      - name: Send Notification
+        if: failure() || cancelled()
+        uses: slackapi/slack-github-action@v1.26.0
+        with:
+          payload: |
+            {
+              "text": "CI benchmarks run status: ${{ job.status }}",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "CI benchmarks failed because of ${{ steps.benches.outputs.failed }}.\nView the results <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CI_ALERTS_CHANNEL_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
diff --git a/tools/run_ci.sh b/tools/run_ci.sh
index 3168bd77..cc054041 100644
--- a/tools/run_ci.sh
+++ b/tools/run_ci.sh
@@ -2,6 +2,19 @@
 
 set -e
 
+function handle_error() {
+  echo "Error occured ${QDRANT_VERSION@A} ${ENGINE_NAME@A} ${DATASETS@A}"
+  echo "{failed}={error}" >> $GITHUB_OUTPUT
+}
+
+function handle_term() {
+  echo "Timeout occured ${QDRANT_VERSION@A} ${ENGINE_NAME@A} ${DATASETS@A}"
+  echo "{failed}={timeout}" >> $GITHUB_OUTPUT
+}
+
+trap 'handle_err' ERR
+trap 'handle_term' TERM
+
 # Script, that runs benchmark within the GitHub Actions CI environment
 
 SCRIPT=$(realpath "$0")