Update RDS CloudWatch alarm parameters for more robust monitoring

Author: TimDiekmann

infra/terraform/hash/postgres/alerts.tf

@@ -29,7 +29,8 @@ resource "aws_cloudwatch_metric_alarm" "rds_free_storage_space" {
   namespace           = "AWS/RDS"
   statistic           = "Minimum"
   period              = 300                     # 5 minutes
-  evaluation_periods  = 2                       # Must be low for 10 minutes total
+  evaluation_periods  = 2                       # 10 minutes total
+  datapoints_to_alarm = 2                       # Both datapoints must be low
   threshold           = 10 * 1024 * 1024 * 1024 # 10GB in bytes
   comparison_operator = "LessThanThreshold"
   treat_missing_data  = "breaching"
@@ -58,7 +59,8 @@ resource "aws_cloudwatch_metric_alarm" "rds_cpu_utilization_high" {
   namespace           = "AWS/RDS"
   statistic           = "Average"
   period              = 300 # 5 minutes
-  evaluation_periods  = 2   # Must be high for 10 minutes total
+  evaluation_periods  = 5   # 25 minutes total
+  datapoints_to_alarm = 3   # 3 of 5 datapoints must be high (grace for spikes)
   threshold           = 80  # 80%
   comparison_operator = "GreaterThanThreshold"
   treat_missing_data  = "notBreaching"
@@ -87,7 +89,8 @@ resource "aws_cloudwatch_metric_alarm" "rds_freeable_memory_low" {
   namespace           = "AWS/RDS"
   statistic           = "Minimum"
   period              = 300               # 5 minutes
-  evaluation_periods  = 2                 # Must be low for 10 minutes total
+  evaluation_periods  = 3                 # 15 minutes total
+  datapoints_to_alarm = 2                 # 2 of 3 datapoints must be low (moderate grace)
   threshold           = 256 * 1024 * 1024 # 256MB in bytes
   comparison_operator = "LessThanThreshold"
   treat_missing_data  = "breaching"