chore: Added Datadog heartbeat metrics

syedimranhassan · syedimranhassan · commit 23b5090f1726 · 2024-09-04T11:30:58.000+05:00
diff --git a/dataeng/resources/opsgenie-enable-heartbeat.sh b/dataeng/resources/opsgenie-enable-heartbeat.sh
@@ -13,6 +13,7 @@
 
 
 OPSGENIE_HEARTBEAT_API_URL="https://api.opsgenie.com/v2/heartbeats"
+PAGERDUTY_HEARTBEAT_API_URL=""
 
 if [ -n "$OPSGENIE_HEARTBEAT_NAME" ] && \
    [ -n "$OPSGENIE_HEARTBEAT_CONFIG_KEY" ] &&  \
diff --git a/devops/jobs/CheckASGLifeCycleHooks.groovy b/devops/jobs/CheckASGLifeCycleHooks.groovy
@@ -56,6 +56,7 @@ class CheckASGLifeCycleHooks {
                         credentialsBinding {
                             string('ROLE_ARN', "check-lifecycle-hooks-${deployment}-role-arn")
                             string("GENIE_KEY", "opsgenie_heartbeat_key")
+                            string("DD_KEY", "datadog_heartbeat_key")
                         }
                     }
 
@@ -104,6 +105,21 @@ class CheckASGLifeCycleHooks {
                         if (opsgenie_heartbeat_name) {
                              shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')
                         }
+                        String datadog_heartbeat_name = configuration.get('DATADOG_HEARTBEAT_NAME', '')
+                        if (datadog_heartbeat_name) {
+                            String DD_JSON = """
+                                {
+                                    "series": [{
+                                        "metric": "${datadog_heartbeat_name}",
+                                        "points": [['\$(date +%s)', 1]],
+                                        "type": "gauge",
+                                        "tags": ["env:${deployment}"]
+                                    }]
+                                }
+                                """
+
+                                shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')
+                        }
                     }
                 }
             }
diff --git a/devops/jobs/CheckTableSize.groovy b/devops/jobs/CheckTableSize.groovy
@@ -25,6 +25,7 @@ class CheckTableSize {
                             def variable = "${deployment}-table-size-monitoring"
                             string("ROLE_ARN", variable)
                             string("GENIE_KEY", "opsgenie_heartbeat_key")
+                            string("DD_KEY", "datadog_heartbeat_key")
                         }
                     }
 
diff --git a/devops/jobs/CreateSandboxCI.groovy b/devops/jobs/CreateSandboxCI.groovy
@@ -32,6 +32,7 @@ class CreateSandboxCI {
         wrappers {
             credentialsBinding {
                 string("GENIE_KEY", "opsgenie_heartbeat_key")
+                string("DD_KEY", "datadog_heartbeat_key")
             }
         }
 
@@ -124,6 +125,18 @@ class CreateSandboxCI {
             }
           }
           shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/SandboxCI'+type+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')
+          def metricName = "SandboxCI${type}.heartbeat"
+          String DD_JSON = """
+                          {
+                              "series": [{
+                                  "metric": "${metricName}",
+                                  "points": [['\$(date +%s)', 1]],
+                                  "type": "gauge"
+                              }]
+                          }
+                          """
+
+          shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')
         }
 
         triggers {
diff --git a/devops/jobs/Janitor.groovy b/devops/jobs/Janitor.groovy
@@ -42,6 +42,7 @@ class Janitor {
                     credentialsBinding {
                         string('ROLE_ARN', "tools-jenkins-janitor-${deployment}-role-arn")
                         string("GENIE_KEY", "opsgenie_heartbeat_key")
+                        string("DD_KEY", "datadog_heartbeat_key")
                     }
                 }
 
@@ -102,6 +103,7 @@ class Janitor {
                     env('AWS_CLEANER', configuration.get('aws_cleaner'))
                     env('DENY_LIST', configuration.get('deny_list'))
                     env('NOOP', extraVars.get('NOOP', false))
+                    env('DEPLOYMENT', deployment)
                 }
 
 
diff --git a/devops/jobs/JenkinsHeartbeat.groovy b/devops/jobs/JenkinsHeartbeat.groovy
@@ -19,6 +19,7 @@ class JenkinsHeartbeat{
             wrappers {
                 credentialsBinding {
                     string("GENIE_KEY", "opsgenie_heartbeat_key")
+                    string("DD_KEY", "datadog_heartbeat_key")
                 }
             }
 
@@ -30,6 +31,20 @@ class JenkinsHeartbeat{
                 if (opsgenie_heartbeat_name) {
                     shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')
                 }
+                String datadog_heartbeat_name = extraVars.get('DATADOG_HEARTBEAT_NAME', '')
+                if (datadog_heartbeat_name) {
+                    String DD_JSON = """
+                        {
+                            "series": [{
+                                "metric": "${datadog_heartbeat_name}",
+                                "points": [['\$(date +%s)', 1]],
+                                "type": "gauge"
+                            }]
+                        }
+                        """
+
+                        shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')
+                    }
             }
 
         }
diff --git a/devops/jobs/ProspectusJanitor.groovy b/devops/jobs/ProspectusJanitor.groovy
@@ -41,6 +41,7 @@ class ProspectusJanitor {
                     credentialsBinding {
                         string('ROLE_ARN', "tools-jenkins-janitor-${deployment}-role-arn")
                         string("GENIE_KEY", "opsgenie_heartbeat_key")
+                        string("DD_KEY", "datadog_heartbeat_key")
                     }
                 }
 
@@ -100,6 +101,7 @@ class ProspectusJanitor {
                     env('AWS_REGION', configuration.get('aws_region', 'us-east-1'))
                     env('AWS_CLEANER', configuration.get('aws_cleaner'))
                     env('NOOP', extraVars.get('NOOP', false))
+                    env('DEPLOYMENT', deployment)
                 }
 
 
diff --git a/devops/jobs/SandboxTermination.groovy b/devops/jobs/SandboxTermination.groovy
@@ -37,6 +37,7 @@ class SandboxTermination{
                     string('ROLE_ARN', "launch-sandboxes-role-arn")
                     string('EDX_GIT_BOT_TOKEN', "edx_git_bot_token")
                     string("GENIE_KEY", "opsgenie_heartbeat_key")
+                    string("DD_KEY", "datadog_heartbeat_key")
                 }
             }
 
@@ -102,6 +103,20 @@ class SandboxTermination{
                 if (opsgenie_heartbeat_name) {
                     shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')
                 }
+                String datadog_heartbeat_name = extraVars.get('DATADOG_HEARTBEAT_NAME', '')
+                if (datadog_heartbeat_name) {
+                    String DD_JSON = """
+                        {
+                            "series": [{
+                                "metric": "${datadog_heartbeat_name}",
+                                "points": [['\$(date +%s)', 1]],
+                                "type": "gauge"
+                            }]
+                        }
+                        """
+
+                        shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')
+                    }
             }
 
             if (extraVars.get('NOTIFY_ON_FAILURE')){
diff --git a/devops/resources/janitor.sh b/devops/resources/janitor.sh
@@ -43,3 +43,13 @@ else
 fi
 
 curl -X GET 'https://api.opsgenie.com/v2/heartbeats/'${JOB_NAME##*/}'/ping' -H 'Authorization: GenieKey '${GENIE_KEY}
+curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" \
+-H "Content-Type: application/json" \
+-d '{
+      "series" : [{
+          "metric": '${JOB_NAME##*/}".heartbeat"',
+          "points": [['"$(date +%s)"', 1]],
+          "type": "gauge",
+          "tags": ["env:'${DEPLOYMENT}'"]
+      }]
+  }'
diff --git a/devops/resources/table-size-monitoring.sh b/devops/resources/table-size-monitoring.sh
@@ -32,3 +32,13 @@ fi
 python check_table_size.py  --threshold ${THRESHOLD} ${RDSTHRESHOLD} ${RDSIGNORE}
 
 curl -X GET 'https://api.opsgenie.com/v2/heartbeats/table-size-monitoring-'${DEPLOYMENT}'/ping' -H 'Authorization: GenieKey '${GENIE_KEY}
+curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" \
+-H "Content-Type: application/json" \
+-d '{
+      "series" : [{
+          "metric": "table-size-monitoring-'${DEPLOYMENT}'.heartbeat",
+          "points": [['"$(date +%s)"', 1]],
+          "type": "gauge",
+          "tags": ["env:'${DEPLOYMENT}'"]
+      }]
+  }'

Original file line number	Diff line number	Diff line change
`@@ -56,6 +56,7 @@ class CheckASGLifeCycleHooks {`
`56`	`56`	`credentialsBinding {`
`57`	`57`	`string('ROLE_ARN', "check-lifecycle-hooks-${deployment}-role-arn")`
`58`	`58`	`string("GENIE_KEY", "opsgenie_heartbeat_key")`
	`59`	`+ string("DD_KEY", "datadog_heartbeat_key")`
`59`	`60`	`}`
`60`	`61`	`}`
`61`	`62`
`@@ -104,6 +105,21 @@ class CheckASGLifeCycleHooks {`
`104`	`105`	`if (opsgenie_heartbeat_name) {`
`105`	`106`	`shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')`
`106`	`107`	`}`
	`108`	`+ String datadog_heartbeat_name = configuration.get('DATADOG_HEARTBEAT_NAME', '')`
	`109`	`+ if (datadog_heartbeat_name) {`
	`110`	`+ String DD_JSON = """`
	`111`	`+ {`
	`112`	`+ "series": [{`
	`113`	`+ "metric": "${datadog_heartbeat_name}",`
	`114`	`+ "points": [['\$(date +%s)', 1]],`
	`115`	`+ "type": "gauge",`
	`116`	`+ "tags": ["env:${deployment}"]`
	`117`	`+ }]`
	`118`	`+ }`
	`119`	`+ """`
	`120`	`+`
	`121`	`+ shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')`
	`122`	`+ }`
`107`	`123`	`}`
`108`	`124`	`}`
`109`	`125`	`}`
Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ class CheckTableSize {`
`25`	`25`	`def variable = "${deployment}-table-size-monitoring"`
`26`	`26`	`string("ROLE_ARN", variable)`
`27`	`27`	`string("GENIE_KEY", "opsgenie_heartbeat_key")`
	`28`	`+ string("DD_KEY", "datadog_heartbeat_key")`
`28`	`29`	`}`
`29`	`30`	`}`
`30`	`31`
Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@ class CreateSandboxCI {`
`32`	`32`	`wrappers {`
`33`	`33`	`credentialsBinding {`
`34`	`34`	`string("GENIE_KEY", "opsgenie_heartbeat_key")`
	`35`	`+ string("DD_KEY", "datadog_heartbeat_key")`
`35`	`36`	`}`
`36`	`37`	`}`
`37`	`38`
`@@ -124,6 +125,18 @@ class CreateSandboxCI {`
`124`	`125`	`}`
`125`	`126`	`}`
`126`	`127`	`shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/SandboxCI'+type+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')`
	`128`	`+ def metricName = "SandboxCI${type}.heartbeat"`
	`129`	`+ String DD_JSON = """`
	`130`	`+ {`
	`131`	`+ "series": [{`
	`132`	`+ "metric": "${metricName}",`
	`133`	`+ "points": [['\$(date +%s)', 1]],`
	`134`	`+ "type": "gauge"`
	`135`	`+ }]`
	`136`	`+ }`
	`137`	`+ """`
	`138`	`+`
	`139`	`+ shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')`
`127`	`140`	`}`
`128`	`141`
`129`	`142`	`triggers {`
Original file line number	Diff line number	Diff line change
`@@ -42,6 +42,7 @@ class Janitor {`
`42`	`42`	`credentialsBinding {`
`43`	`43`	`string('ROLE_ARN', "tools-jenkins-janitor-${deployment}-role-arn")`
`44`	`44`	`string("GENIE_KEY", "opsgenie_heartbeat_key")`
	`45`	`+ string("DD_KEY", "datadog_heartbeat_key")`
`45`	`46`	`}`
`46`	`47`	`}`
`47`	`48`
`@@ -102,6 +103,7 @@ class Janitor {`
`102`	`103`	`env('AWS_CLEANER', configuration.get('aws_cleaner'))`
`103`	`104`	`env('DENY_LIST', configuration.get('deny_list'))`
`104`	`105`	`env('NOOP', extraVars.get('NOOP', false))`
	`106`	`+ env('DEPLOYMENT', deployment)`
`105`	`107`	`}`
`106`	`108`
`107`	`109`
Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,7 @@ class JenkinsHeartbeat{`
`19`	`19`	`wrappers {`
`20`	`20`	`credentialsBinding {`
`21`	`21`	`string("GENIE_KEY", "opsgenie_heartbeat_key")`
	`22`	`+ string("DD_KEY", "datadog_heartbeat_key")`
`22`	`23`	`}`
`23`	`24`	`}`
`24`	`25`
`@@ -30,6 +31,20 @@ class JenkinsHeartbeat{`
`30`	`31`	`if (opsgenie_heartbeat_name) {`
`31`	`32`	`shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')`
`32`	`33`	`}`
	`34`	`+ String datadog_heartbeat_name = extraVars.get('DATADOG_HEARTBEAT_NAME', '')`
	`35`	`+ if (datadog_heartbeat_name) {`
	`36`	`+ String DD_JSON = """`
	`37`	`+ {`
	`38`	`+ "series": [{`
	`39`	`+ "metric": "${datadog_heartbeat_name}",`
	`40`	`+ "points": [['\$(date +%s)', 1]],`
	`41`	`+ "type": "gauge"`
	`42`	`+ }]`
	`43`	`+ }`
	`44`	`+ """`
	`45`	`+`
	`46`	`+ shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')`
	`47`	`+ }`
`33`	`48`	`}`
`34`	`49`
`35`	`50`	`}`
Original file line number	Diff line number	Diff line change
`@@ -41,6 +41,7 @@ class ProspectusJanitor {`
`41`	`41`	`credentialsBinding {`
`42`	`42`	`string('ROLE_ARN', "tools-jenkins-janitor-${deployment}-role-arn")`
`43`	`43`	`string("GENIE_KEY", "opsgenie_heartbeat_key")`
	`44`	`+ string("DD_KEY", "datadog_heartbeat_key")`
`44`	`45`	`}`
`45`	`46`	`}`
`46`	`47`
`@@ -100,6 +101,7 @@ class ProspectusJanitor {`
`100`	`101`	`env('AWS_REGION', configuration.get('aws_region', 'us-east-1'))`
`101`	`102`	`env('AWS_CLEANER', configuration.get('aws_cleaner'))`
`102`	`103`	`env('NOOP', extraVars.get('NOOP', false))`
	`104`	`+ env('DEPLOYMENT', deployment)`
`103`	`105`	`}`
`104`	`106`
`105`	`107`