Add FreeStorageSpace CloudWatch alert for RDS

Author: TimDiekmann

infra/terraform/hash/main.tf

@@ -84,18 +84,19 @@ module "bastion" {
 }
 
 module "postgres" {
-  depends_on            = [module.networking]
-  source                = "./postgres"
-  prefix                = local.prefix
-  subnets               = module.networking.snpriv
-  vpc_id                = module.networking.vpc.id
-  vpc_cidr_block        = module.networking.vpc.cidr_block
-  env                   = local.env
-  region                = local.region
-  pg_port               = 5432
-  instance_class        = "db.t3.small"
-  pg_superuser_username = "superuser"
-  pg_superuser_password = sensitive(data.vault_kv_secret_v2.secrets.data["pg_superuser_password"])
+  depends_on                                  = [module.networking]
+  source                                      = "./postgres"
+  prefix                                      = local.prefix
+  subnets                                     = module.networking.snpriv
+  vpc_id                                      = module.networking.vpc.id
+  vpc_cidr_block                              = module.networking.vpc.cidr_block
+  env                                         = local.env
+  region                                      = local.region
+  pg_port                                     = 5432
+  instance_class                              = "db.t3.small"
+  pg_superuser_username                       = "superuser"
+  pg_superuser_password                       = sensitive(data.vault_kv_secret_v2.secrets.data["pg_superuser_password"])
+  pagerduty_main_database_aws_integration_key = sensitive(data.vault_kv_secret_v2.secrets.data["pagerduty_main_database_aws_integration_key"])
 }
 
 module "temporal" {

infra/terraform/hash/postgres/alerts.tf

@@ -0,0 +1,49 @@
+# CloudWatch alerts for RDS database monitoring
+# Provides alerting for critical database metrics
+
+# SNS Topic for database alerts
+resource "aws_sns_topic" "database_alerts" {
+  name = "${var.prefix}-database-alerts"
+
+  tags = {
+    Name    = "${var.prefix}-database-alerts"
+    Service = "postgres"
+    Purpose = "RDS database health and performance alerts"
+  }
+}
+
+# PagerDuty subscription for database alerts
+resource "aws_sns_topic_subscription" "pagerduty" {
+  topic_arn = aws_sns_topic.database_alerts.arn
+  protocol  = "https"
+  endpoint  = "https://events.pagerduty.com/integration/${var.pagerduty_main_database_aws_integration_key}/enqueue"
+}
+
+# CloudWatch Alarm for RDS free storage space
+resource "aws_cloudwatch_metric_alarm" "rds_free_storage_space" {
+  alarm_name        = "${var.prefix}-rds-free-storage-space-low"
+  alarm_description = "CRITICAL: RDS instance ${aws_db_instance.postgres.identifier} has low free storage space. Storage: ${aws_db_instance.postgres.allocated_storage}GB total."
+
+  # RDS storage metrics
+  metric_name         = "FreeStorageSpace"
+  namespace           = "AWS/RDS"
+  statistic           = "Minimum"
+  period              = 300                     # 5 minutes
+  evaluation_periods  = 2                       # Must be low for 10 minutes total
+  threshold           = 10 * 1024 * 1024 * 1024 # 10GB in bytes
+  comparison_operator = "LessThanThreshold"
+  treat_missing_data  = "breaching"
+
+  dimensions = {
+    DBInstanceIdentifier = aws_db_instance.postgres.identifier
+  }
+
+  alarm_actions = [aws_sns_topic.database_alerts.arn]
+  ok_actions    = [aws_sns_topic.database_alerts.arn]
+
+  tags = {
+    Name     = "${var.prefix}-rds-free-storage-space-low-alarm"
+    Severity = "CRITICAL"
+    Purpose  = "Alert when RDS free storage space is critically low"
+  }
+}

infra/terraform/hash/postgres/variables.tf

@@ -50,3 +50,9 @@ variable "pg_superuser_password" {
   sensitive   = true
   description = "Password for the 'superuser' user in the Postgres instance"
 }
+
+variable "pagerduty_main_database_aws_integration_key" {
+  type        = string
+  sensitive   = true
+  description = "PagerDuty integration key for main database AWS alerts"
+}

infra/terraform/hash/temporal/task_definitions.tf

@@ -25,7 +25,7 @@ locals {
 
       logConfiguration = {
         logDriver = "awslogs"
-        options   = {
+        options = {
           "awslogs-create-group"  = "true"
           "awslogs-group"         = local.log_group_name
           "awslogs-stream-prefix" = local.migrate_service_name
@@ -34,13 +34,13 @@ locals {
       }
     },
     {
-      essential   = true
-      name        = "${local.prefix}-${local.temporal_service_name}"
-      image       = "temporalio/server:${var.temporal_version}"
-      cpu         = 0 # let ECS divvy up the available CPU
-      dependsOn   = [{ condition = "SUCCESS", containerName = "${local.prefix}-${local.migrate_service_name}" }]
+      essential = true
+      name      = "${local.prefix}-${local.temporal_service_name}"
+      image     = "temporalio/server:${var.temporal_version}"
+      cpu       = 0 # let ECS divvy up the available CPU
+      dependsOn = [{ condition = "SUCCESS", containerName = "${local.prefix}-${local.migrate_service_name}" }]
       healthCheck = {
-        command     = [
+        command = [
           "CMD", "/bin/sh", "-c", "temporal operator cluster health --address $(hostname):7233 | grep -q SERVING"
         ]
         startPeriod = 10
@@ -67,7 +67,7 @@ locals {
 
       logConfiguration = {
         logDriver = "awslogs"
-        options   = {
+        options = {
           "awslogs-create-group"  = "true"
           "awslogs-group"         = local.log_group_name
           "awslogs-stream-prefix" = local.temporal_service_name
@@ -103,7 +103,7 @@ locals {
 
       logConfiguration = {
         logDriver = "awslogs"
-        options   = {
+        options = {
           "awslogs-create-group"  = "true"
           "awslogs-group"         = local.log_group_name
           "awslogs-stream-prefix" = local.setup_service_name
@@ -116,7 +116,7 @@ locals {
 
       essential = false
       name      = "${local.prefix}-${local.ui_service_name}"
-      image       = "temporalio/ui:${var.temporal_ui_version}"
+      image     = "temporalio/ui:${var.temporal_ui_version}"
       cpu       = 0 # let ECS divvy up the available CPU
       dependsOn = [
         { condition = "HEALTHY", containerName = "${local.prefix}-${local.temporal_service_name}" },
@@ -145,7 +145,7 @@ locals {
 
       logConfiguration = {
         logDriver = "awslogs"
-        options   = {
+        options = {
           "awslogs-create-group"  = "true"
           "awslogs-group"         = local.log_group_name
           "awslogs-stream-prefix" = local.ui_service_name

infra/terraform/hash/tls.tf

@@ -47,7 +47,7 @@ resource "cloudflare_dns_record" "caa_hash_ai" {
     value = "amazon.com"
   }
 
-  ttl     = 1
+  ttl = 1
 
   tags = ["terraform"]
 }