Merge branch 'feature/PI-357-automatic_snapshots_of_etl' into release/2024-07-22

Author: megan-bower4

infrastructure/terraform/per_account/int/main.tf

@@ -107,10 +107,37 @@ module "snapshot_bucket" {
   version                               = "3.15.2"
   bucket                                = "${local.project}--${replace(terraform.workspace, "_", "-")}--snapshot"
   attach_deny_insecure_transport_policy = true
+  attach_access_log_delivery_policy     = true
   versioning = {
     enabled = true
   }
   tags = {
     Name = "${local.project}--${replace(terraform.workspace, "_", "-")}--snapshot"
   }
 }
+
+data "aws_s3_bucket_policy" "existing_policy" {
+  bucket = module.snapshot_bucket.s3_bucket_id
+}
+
+resource "aws_s3_bucket_policy" "snapshot_bucket_policy" {
+  bucket = module.snapshot_bucket.s3_bucket_id
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = concat(
+      jsondecode(data.aws_s3_bucket_policy.existing_policy.policy)["Statement"], [
+        {
+          Sid       = "AllowDynamoDBExport",
+          Effect    = "Allow",
+          Principal = { Service = "dynamodb.amazonaws.com" },
+          Action = [
+            "s3:PutObject",
+            "s3:AbortMultipartUpload",
+            "s3:ListMultipartUploadParts"
+          ],
+          Resource = "${module.snapshot_bucket.s3_bucket_arn}/*"
+        }
+    ])
+  })
+}

infrastructure/terraform/per_account/prod/main.tf

@@ -105,10 +105,37 @@ module "snapshot_bucket" {
   version                               = "3.15.2"
   bucket                                = "${local.project}--${replace(terraform.workspace, "_", "-")}--snapshot"
   attach_deny_insecure_transport_policy = true
+  attach_access_log_delivery_policy     = true
   versioning = {
     enabled = true
   }
   tags = {
     Name = "${local.project}--${replace(terraform.workspace, "_", "-")}--snapshot"
   }
 }
+
+data "aws_s3_bucket_policy" "existing_policy" {
+  bucket = module.snapshot_bucket.s3_bucket_id
+}
+
+resource "aws_s3_bucket_policy" "snapshot_bucket_policy" {
+  bucket = module.snapshot_bucket.s3_bucket_id
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = concat(
+      jsondecode(data.aws_s3_bucket_policy.existing_policy.policy)["Statement"], [
+        {
+          Sid       = "AllowDynamoDBExport",
+          Effect    = "Allow",
+          Principal = { Service = "dynamodb.amazonaws.com" },
+          Action = [
+            "s3:PutObject",
+            "s3:AbortMultipartUpload",
+            "s3:ListMultipartUploadParts"
+          ],
+          Resource = "${module.snapshot_bucket.s3_bucket_arn}/*"
+        }
+    ])
+  })
+}

infrastructure/terraform/per_workspace/locals.tf

@@ -10,5 +10,6 @@ locals {
   # e.g. api.cpm.dev.national.nhs.uk
   zone = var.domain
 
-  domain = "${terraform.workspace}.${var.domain}"
+  domain              = "${terraform.workspace}.${var.domain}"
+  etl_snapshot_bucket = contains(["int", "prod"], var.environment) ? "${local.project}--${replace(var.environment, "_", "-")}--snapshot" : "snapshot_not_required"
 }

infrastructure/terraform/per_workspace/main.tf

@@ -192,5 +192,6 @@ module "sds_etl" {
   table_arn                        = module.table.dynamodb_table_arn
   is_persistent                    = var.workspace_type == "PERSISTENT"
   truststore_bucket                = data.aws_s3_bucket.truststore_bucket
+  etl_snapshot_bucket              = local.etl_snapshot_bucket
   environment                      = var.environment
 }

infrastructure/terraform/per_workspace/modules/etl/sds/etl-diagram.asl.json

@@ -262,7 +262,38 @@
           "Next": "fail"
         }
       ],
-      "Default": "delete-state-lock"
+      "Default": "set-snapshot-bucket"
+    },
+    "set-snapshot-bucket": {
+      "Type": "Pass",
+      "Next": "check-snapshot-bucket",
+      "Result": {
+        "snapshot_bucket": "${etl_snapshot_bucket}"
+      }
+    },
+    "check-snapshot-bucket": {
+      "Type": "Choice",
+      "Choices": [
+        {
+          "Variable": "$.snapshot_bucket",
+          "StringEquals": "snapshot_not_required",
+          "Next": "skip-export"
+        }
+      ],
+      "Default": "export-dynamodb-to-s3"
+    },
+    "export-dynamodb-to-s3": {
+      "Type": "Task",
+      "Parameters": {
+        "S3Bucket": "${etl_snapshot_bucket}",
+        "TableArn": "${table_arn}"
+      },
+      "Resource": "arn:aws:states:::aws-sdk:dynamodb:exportTableToPointInTime",
+      "Next": "delete-state-lock"
+    },
+    "skip-export": {
+      "Type": "Pass",
+      "Next": "delete-state-lock"
     },
     "delete-state-lock": {
       "Type": "Task",

infrastructure/terraform/per_workspace/modules/etl/sds/main.tf

@@ -303,6 +303,8 @@ resource "aws_sfn_state_machine" "state_machine" {
       etl_update_state_machine_arn = module.update_transform_and_load_step_function.state_machine_arn
       etl_bulk_state_machine_arn   = module.bulk_transform_and_load_step_function.state_machine_arn
       etl_state_lock_key           = var.etl_state_lock_key
+      etl_snapshot_bucket          = var.etl_snapshot_bucket
+      table_arn                    = var.table_arn
     }
   )
   logging_configuration {

infrastructure/terraform/per_workspace/modules/etl/sds/step_function_role.tf

@@ -48,11 +48,14 @@ locals {
         "s3:ListBucket",
         "s3:ListBucketMultipartUploads",
         "s3:PutObjectVersionTagging",
-        "s3:DeleteObject"
+        "s3:DeleteObject",
+        "s3:PutBucketPolicy"
       ]
       resources = [
         "${module.bucket.s3_bucket_arn}",
-        "${module.bucket.s3_bucket_arn}/*"
+        "${module.bucket.s3_bucket_arn}/*",
+        "arn:aws:s3:::${var.etl_snapshot_bucket}/*",
+        "arn:aws:s3:::${var.etl_snapshot_bucket}"
       ]
     }
 
@@ -69,6 +72,19 @@ locals {
       ]
       resources = ["*"]
     }
+
+    dynamodb = {
+      actions   = ["dynamodb:ExportTableToPointInTime"]
+      resources = ["arn:aws:dynamodb:eu-west-2:${var.assume_account}:table/${var.table_name}"]
+    }
+
+    kms = {
+      actions = [
+        "kms:Decrypt",
+        "kms:DescribeKey"
+      ]
+      resources = [data.aws_kms_key.dynamodb_kms_key.arn]
+    }
   }
 
   depends_on = [module.bucket, module.update_transform_and_load_step_function, module.bulk_transform_and_load_step_function]
@@ -106,6 +122,10 @@ data "aws_iam_policy_document" "service" {
   }
 }
 
+data "aws_kms_key" "dynamodb_kms_key" {
+  key_id = "alias/${var.table_name}"
+}
+
 resource "aws_iam_policy" "service" {
   for_each = { for k, v in local.service_integrations : k => v }
   name     = "${local.name}--${each.key}"

infrastructure/terraform/per_workspace/modules/etl/sds/vars.tf

@@ -23,6 +23,7 @@ variable "table_name" {}
 variable "table_arn" {}
 variable "is_persistent" {}
 variable "truststore_bucket" {}
+variable "etl_snapshot_bucket" {}
 variable "environment" {
 
 }

scripts/infrastructure/policies/deployment2-policy.json

@@ -146,8 +146,6 @@
         "s3:PutBucketOwnershipControls",
         "s3:PutBucketPublicAccessBlock",
         "s3:PutBucketAcl",
-        "s3:PutBucketPolicy",
-        "s3:PutBucketLogging",
         "s3:DeleteBucketPolicy",
         "s3:DeleteBucket",
         "s3:DeleteObject",