Merge pull request #120 from NASA-PDS/replication

PDS Nucleus Archive Replication
NASA-PDS · Sep 6, 2024 · 58997ef · 58997ef
2 parents d9c5863 + a9acb1c
commit 58997ef
Show file tree

Hide file tree

Showing 16 changed files with 270 additions and 104 deletions.
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -135,7 +135,8 @@
         "venv",
         "dist",
         "build",
-        ".*\\.egg-info"
+        ".*\\.egg-info",
+        "\u2018*.tfstate \\\n    --exclude-files \u2018\\*.tfvars"
       ]
     }
   ],
@@ -184,16 +185,6 @@
         "is_secret": false
       }
     ],
-    "terraform/terraform-modules/ecs-ecr/docker/deploy-ecr-images.sh": [
-      {
-        "type": "AWS Sensitive Information (Experimental Plugin)",
-        "filename": "terraform/terraform-modules/ecs-ecr/docker/deploy-ecr-images.sh",
-        "hashed_secret": "9ad897024d8c36c541d7fe84084c4e9f4df00b2a",
-        "is_verified": false,
-        "line_number": 4,
-        "is_secret": false
-      }
-    ],
     "terraform/terraform-modules/ecs-ecr/docker/template-deploy-ecr-images.sh": [
       {
         "type": "AWS Sensitive Information (Experimental Plugin)",
@@ -218,35 +209,15 @@
         "filename": "terraform/terraform-modules/ecs-ecr/ecs_ecr.tf",
         "hashed_secret": "957580e87fca1bd3e2acdfbae2a6c6e24a1d4ade",
         "is_verified": false,
-        "line_number": 185,
+        "line_number": 197,
         "is_secret": false
       },
       {
         "type": "Secret Keyword",
         "filename": "terraform/terraform-modules/ecs-ecr/ecs_ecr.tf",
         "hashed_secret": "227f2d989bdd935539c4e9bd92b8c4a5965505ac",
         "is_verified": false,
-        "line_number": 199,
-        "is_secret": false
-      }
-    ],
-    "terraform/terraform-modules/ecs-ecr/ecs_task_execution_role_iam_policy.json": [
-      {
-        "type": "AWS Sensitive Information (Experimental Plugin)",
-        "filename": "terraform/terraform-modules/ecs-ecr/ecs_task_execution_role_iam_policy.json",
-        "hashed_secret": "9ad897024d8c36c541d7fe84084c4e9f4df00b2a",
-        "is_verified": false,
-        "line_number": 11,
-        "is_secret": false
-      }
-    ],
-    "terraform/terraform-modules/ecs-ecr/ecs_task_role_iam_policy.json": [
-      {
-        "type": "AWS Sensitive Information (Experimental Plugin)",
-        "filename": "terraform/terraform-modules/ecs-ecr/ecs_task_role_iam_policy.json",
-        "hashed_secret": "9ad897024d8c36c541d7fe84084c4e9f4df00b2a",
-        "is_verified": false,
-        "line_number": 11,
+        "line_number": 211,
         "is_secret": false
       }
     ],
@@ -280,16 +251,6 @@
         "is_secret": false
       }
     ],
-    "terraform/terraform-modules/mwaa-env/mwaa_execution_role_iam_policy.json": [
-      {
-        "type": "AWS Sensitive Information (Experimental Plugin)",
-        "filename": "terraform/terraform-modules/mwaa-env/mwaa_execution_role_iam_policy.json",
-        "hashed_secret": "9ad897024d8c36c541d7fe84084c4e9f4df00b2a",
-        "is_verified": false,
-        "line_number": 8,
-        "is_secret": false
-      }
-    ],
     "terraform/terraform-modules/mwaa-env/template_mwaa_execution_role_iam_policy.json": [
       {
         "type": "AWS Sensitive Information (Experimental Plugin)",
@@ -300,16 +261,6 @@
         "is_secret": false
       }
     ],
-    "terraform/terraform-modules/product-copy-completion-checker/lambda_inline_policy.json": [
-      {
-        "type": "AWS Sensitive Information (Experimental Plugin)",
-        "filename": "terraform/terraform-modules/product-copy-completion-checker/lambda_inline_policy.json",
-        "hashed_secret": "9ad897024d8c36c541d7fe84084c4e9f4df00b2a",
-        "is_verified": false,
-        "line_number": 11,
-        "is_secret": false
-      }
-    ],
     "terraform/terraform-modules/product-copy-completion-checker/product-copy-completion-checker.tf": [
       {
         "type": "AWS Sensitive Information (Experimental Plugin)",
@@ -331,5 +282,5 @@
       }
     ]
   },
-  "generated_at": "2024-08-15T22:17:21Z"
+  "generated_at": "2024-09-06T02:11:28Z"
 }
diff --git a/terraform/README.md b/terraform/README.md
@@ -119,7 +119,12 @@ pds_nucleus_config_bucket_name          = "pds-nucleus-config-mcp-dev"
 pds_nucleus_default_airflow_dag_id = "pds-basic-registry-load-use-case"
 ```
 
-5. Initialize Terraform working directory.
+
+5. Make sure to have an S3 bucket available in the AWS account to keep Terraform remote state. 
+The name of the S3 bucket should match with the bucket name in the `terraform/backend.tf` file. 
+If a bucket to keep the Terraform remote state is not available, please create a new bucket.
+
+6. Initialize Terraform working directory.
 
 ```shell
 terraform init

diff --git a/terraform/backend.tf b/terraform/backend.tf
@@ -0,0 +1,7 @@
+terraform {
+  backend "s3" {
+    bucket = "pds-nucleus-tf-state"
+    key    = "dev/nucleus_infra.tfstate"
+    region = "us-west-2"
+  }
+}
diff --git a/terraform/main.tf b/terraform/main.tf
@@ -7,13 +7,28 @@ module "common" {
   mwaa_dag_s3_bucket_name = var.mwaa_dag_s3_bucket_name
 }
 
-# Terraform module to create archive for PDS Nucleus
+# Terraform module to create primary archive for PDS Nucleus
 module "archive" {
   source                                       = "./terraform-modules/archive"
   pds_node_names                               = var.pds_node_names
   depends_on                                   = [module.common, module.ecs_ecr]
   pds_nucleus_hot_archive_bucket_name_postfix  = var.pds_nucleus_hot_archive_bucket_name_postfix
   pds_nucleus_cold_archive_bucket_name_postfix = var.pds_nucleus_cold_archive_bucket_name_postfix
+  pds_nucleus_cold_archive_buckets             = module.archive-secondary.pds_nucleus_cold_archive_buckets
+  permission_boundary_for_iam_roles            = var.permission_boundary_for_iam_roles
+  pds_nucleus_cold_archive_storage_class       = var.pds_nucleus_cold_archive_storage_class
+}
+
+# Terraform module to create secondary archive for PDS Nucleus
+module "archive-secondary" {
+  source                                       = "./terraform-modules/archive-secondary"
+  pds_node_names                               = var.pds_node_names
+  depends_on                                   = [module.common, module.ecs_ecr]
+  pds_nucleus_cold_archive_bucket_name_postfix = var.pds_nucleus_cold_archive_bucket_name_postfix
+
+  providers = {
+    aws = aws.secondary
+  }
 }
 
 # The Terraform module to create the PDS Nucleus Baseline System (without any project specific components)

diff --git a/terraform/providers.tf b/terraform/providers.tf
@@ -3,12 +3,23 @@ terraform {
     aws = {
       source  = "hashicorp/aws"
       version = "~> 5.0"
+      configuration_aliases = [ aws.secondary ]
     }
   }
 }
 
 provider "aws" {
   region = var.region
+  default_tags {
+    tags = {
+      product     = "PDS Nucleus"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.region_secondary
+  alias = "secondary"
 
   default_tags {
     tags = {

diff --git a/terraform/terraform-modules/archive-secondary/archive-secondary.tf b/terraform/terraform-modules/archive-secondary/archive-secondary.tf
@@ -0,0 +1,20 @@
+# Terraform script to create the PDS archive related resources
+
+# Create a cold archive for each PDS Node
+resource "aws_s3_bucket" "pds_nucleus_cold_archive" {
+  count = length(var.pds_node_names)
+  # convert PDS node name to S3 bucket name compatible format
+  bucket = "${lower(replace(var.pds_node_names[count.index], "_", "-"))}-${var.pds_nucleus_cold_archive_bucket_name_postfix}"
+}
+
+resource "aws_s3_bucket_versioning" "pds_nucleus_cold_archive" {
+  count = length(var.pds_node_names)
+  bucket = aws_s3_bucket.pds_nucleus_cold_archive[count.index].id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+output "pds_nucleus_cold_archive_buckets" {
+  value = aws_s3_bucket.pds_nucleus_cold_archive
+}
diff --git a/terraform/terraform-modules/archive-secondary/variables.tf b/terraform/terraform-modules/archive-secondary/variables.tf
@@ -0,0 +1,12 @@
+variable "pds_node_names" {
+  description = "List of PDS Node Names"
+  type        = list(string)
+  sensitive   = true
+}
+
+variable "pds_nucleus_cold_archive_bucket_name_postfix" {
+  description = "The postfix of the name of the cold archive s3 bucket"
+  default     = "cold-archive-<venue-name>"
+  type        = string
+  sensitive   = true
+}
diff --git a/terraform/terraform-modules/archive/archive.tf b/terraform/terraform-modules/archive/archive.tf
@@ -7,9 +7,128 @@ resource "aws_s3_bucket" "pds_nucleus_hot_archive" {
   bucket = "${lower(replace(var.pds_node_names[count.index], "_", "-"))}-${var.pds_nucleus_hot_archive_bucket_name_postfix}"
 }
 
-# Create a cold archive for each PDS Node
-resource "aws_s3_bucket" "pds_nucleus_cold_archive" {
+resource "aws_s3_bucket_versioning" "pds_nucleus_hot_archive" {
   count = length(var.pds_node_names)
-  # convert PDS node name to S3 bucket name compatible format
-  bucket = "${lower(replace(var.pds_node_names[count.index], "_", "-"))}-${var.pds_nucleus_cold_archive_bucket_name_postfix}"
+
+  bucket = aws_s3_bucket.pds_nucleus_hot_archive[count.index].id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+data "aws_iam_policy_document" "pds_nucleus_archive_replication_assume_role" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["s3.amazonaws.com"]
+    }
+
+    actions = ["sts:AssumeRole"]
+  }
+}
+
+# The Policy for Permission Boundary
+data "aws_iam_policy" "mcp_operator_policy" {
+  name = var.permission_boundary_for_iam_roles
+}
+
+resource "aws_iam_role" "pds_nucleus_archive_replication_role" {
+  name               = "pds-nucleus-archive-replication-role"
+  assume_role_policy = data.aws_iam_policy_document.pds_nucleus_archive_replication_assume_role.json
+  permissions_boundary = data.aws_iam_policy.mcp_operator_policy.arn
+}
+
+data "aws_iam_policy_document" "pds_nucleus_archive_replication_policy" {
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3:GetReplicationConfiguration",
+      "s3:ListBucket",
+    ]
+
+    resources = ["arn:aws:s3:::pds-*-archive-*"]
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3:GetObjectVersionForReplication",
+      "s3:GetObjectVersionAcl",
+      "s3:GetObjectVersionTagging",
+    ]
+
+    resources = ["arn:aws:s3:::pds-*-archive-*"]
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3:ReplicateObject",
+      "s3:ReplicateDelete",
+      "s3:ReplicateTags",
+    ]
+
+    resources = ["arn:aws:s3:::pds-*-archive-*"]
+  }
+}
+
+resource "aws_iam_policy" "pds_nucleus_archive_replication_policy" {
+  name   = "pds-nucleus-archive-replication-policy"
+  policy = data.aws_iam_policy_document.pds_nucleus_archive_replication_policy.json
+}
+
+resource "aws_iam_role_policy_attachment" "pds_nucleus_archive_replication_policy_document" {
+  role       = aws_iam_role.pds_nucleus_archive_replication_role.name
+  policy_arn = aws_iam_policy.pds_nucleus_archive_replication_policy.arn
+}
+
+resource "aws_s3_bucket_replication_configuration" "pds_nucleus_s3_bucket_replication_configuration" {
+
+  count = length(var.pds_node_names)
+
+  role   = aws_iam_role.pds_nucleus_archive_replication_role.arn
+  bucket = aws_s3_bucket.pds_nucleus_hot_archive[count.index].id
+
+  rule {
+
+    filter {
+    }
+
+    id = "pds-nucleus-replication-rule-${var.pds_node_names[count.index]}"
+
+    status = "Enabled"
+
+    delete_marker_replication {
+      status = "Disabled"
+    }
+
+    destination {
+      bucket        = var.pds_nucleus_cold_archive_buckets[count.index].arn
+      storage_class = var.pds_nucleus_cold_archive_storage_class
+
+      metrics {
+        event_threshold {
+          minutes = 15
+        }
+        status = "Enabled"
+      }
+
+
+      replication_time {
+        status = "Enabled"
+        time {
+          minutes = 15
+        }
+      }
+
+    }
+  }
+
+  depends_on = [aws_s3_bucket_versioning.pds_nucleus_hot_archive]
 }
diff --git a/terraform/terraform-modules/archive/variables.tf b/terraform/terraform-modules/archive/variables.tf
@@ -17,3 +17,20 @@ variable "pds_nucleus_cold_archive_bucket_name_postfix" {
   type        = string
   sensitive   = true
 }
+
+variable "pds_nucleus_cold_archive_buckets" {
+  description = "The list of the names of the cold archive s3 buckets"
+  type        = list
+  sensitive   = true
+}
+
+variable "pds_nucleus_cold_archive_storage_class" {
+  description = "The storage class of the cold archive s3 buckets"
+  type        = string
+}
+
+variable "permission_boundary_for_iam_roles" {
+  description = "Permission boundary for IAM roles"
+  type      = string
+  sensitive = true
+}
diff --git a/terraform/terraform-modules/ecs-ecr/docker/deploy-ecr-images.sh b/terraform/terraform-modules/ecs-ecr/docker/deploy-ecr-images.sh