FIM Caching Fixes (#608)

There were a small handful of Terraform Syntax and Logic errors that I needed to fix before deploying all the new Redshift resources. Additionally, I included a bugfix that hopefully makes the image-based Lambdas update correctly when their underlying code changes.
NOAA-OWP · Jan 3, 2024 · 30fd62b · 30fd62b
1 parent d5fac78
commit 30fd62b
Show file tree

Hide file tree

Showing 9 changed files with 33 additions and 20 deletions.
diff --git a/Core/EC2/RDSBastion/main.tf b/Core/EC2/RDSBastion/main.tf
@@ -80,7 +80,7 @@ variable "viz_redshift_name" {
   type = string
 }
 
-variable "viz_redshift_iam_role" {
+variable "viz_redshift_iam_role_arn" {
   type = string
 }
 
@@ -378,6 +378,7 @@ data "cloudinit_config" "startup" {
       viz_redshift_master_password      = local.dbs["viz_redshift"]["db_password"]
       viz_redshift_user_username        = jsondecode(var.viz_redshift_user_secret_string)["username"]
       viz_redshift_user_password        = jsondecode(var.viz_redshift_user_secret_string)["password"]
+      viz_redshift_iam_role_arn         = var.viz_redshift_iam_role_arn
       viz_db_name                       = local.dbs["viz"]["db_name"]
       viz_db_host                       = local.dbs["viz"]["db_host"]
       viz_db_port                       = local.dbs["viz"]["db_port"]

diff --git a/Core/EC2/RDSBastion/scripts/viz/redshift_setup.sh.tftpl b/Core/EC2/RDSBastion/scripts/viz/redshift_setup.sh.tftpl
@@ -9,15 +9,15 @@ echo "---- SETTING UP VIZ REDSHIFT DB ----"
 
 
 # Setting up Viz Redshift DB
-export PGPASSWORD=${viz_db_password}
+export PGPASSWORD=${viz_redshift_master_password}
 
 # Adding users to Viz Redshift DB
 echo "Adding viz redshift user..."
-psql -h "${viz_redshift_host}" -U "${viz_redshift_username}" -p ${viz_redshift_port} -d "${viz_redshift_name}" -qtAc "CREATE USER ${viz_redshift_user_username} WITH PASSWORD '${viz_redshift_user_password}';"
+psql -h "${viz_redshift_host}" -U "${viz_redshift_master_username}" -p ${viz_redshift_port} -d "${viz_redshift_name}" -qtAc "CREATE USER ${viz_redshift_user_username} WITH PASSWORD '${viz_redshift_user_password}';"
 
 # Setup external schemas - linked to viz processing rds datbase - this could be abstracted as done with the viz processing foreign schemas, but I'm not doing that now since it is only one schema we need.
 echo "Adding external schema link to viz ingest ..."
-psql -h "${viz_redshift_host}" -U "${viz_redshift_username}" -p ${viz_redshift_port} -d "${viz_redshift_name}" \
+psql -h "${viz_redshift_host}" -U "${viz_redshift_master_username}" -p ${viz_redshift_port} -d "${viz_redshift_name}" \
     -tAc "DROP SCHEMA IF EXISTS external_viz_ingest;
           CREATE EXTERNAL SCHEMA external_viz_ingest
           FROM POSTGRES

diff --git a/Core/IAM/Roles/redshift.json.tftpl b/Core/IAM/Roles/redshift.json.tftpl
@@ -2,7 +2,7 @@
     "Version": "2012-10-17",
     "Statement": [
         {
-            "Sid": "s3_access",
+            "Sid": "s3Access",
             "Action": [
                 "s3:PutObject",
                 "s3:Get*",
@@ -14,7 +14,7 @@
             ] 
         },
         {
-            "Sid": "secret_for_external_data_schema",
+            "Sid": "secretForExternalDataSchema",
             "Action": [
                 "secretsmanager:DescribeSecret",
                 "secretsmanager:GetSecretValue"

diff --git a/Core/LAMBDA/viz_functions/image_based/main.tf b/Core/LAMBDA/viz_functions/image_based/main.tf
@@ -192,6 +192,8 @@ resource "null_resource" "viz_raster_processing_cluster" {
     source_hash = data.archive_file.raster_processing_zip.output_md5
   }
 
+  depends_on = [ aws_s3_object.raster_processing_zip_upload ]
+
   provisioner "local-exec" {
     command = "aws codebuild start-build --project-name ${aws_codebuild_project.viz_raster_processing_lambda.name} --profile ${var.environment} --region ${var.region}"
   }
@@ -314,6 +316,8 @@ resource "null_resource" "viz_optimize_rasters_cluster" {
     source_hash = data.archive_file.optimize_rasters_zip.output_md5
   }
 
+  depends_on = [ aws_s3_object.optimize_rasters_zip_upload ]
+
   provisioner "local-exec" {
     command = "aws codebuild start-build --project-name ${aws_codebuild_project.viz_optimize_raster_lambda.name} --profile ${var.environment} --region ${var.region}"
   }
@@ -455,6 +459,8 @@ resource "null_resource" "viz_hand_fim_processing_cluster" {
     fim_version = var.fim_version
   }
 
+  depends_on = [ aws_s3_object.hand_fim_processing_zip_upload ]
+
   provisioner "local-exec" {
     command = "aws codebuild start-build --project-name ${aws_codebuild_project.viz_hand_fim_processing_lambda.name} --profile ${var.environment} --region ${var.region}"
   }
@@ -484,7 +490,7 @@ data "aws_lambda_function" "viz_hand_fim_processing" {
 
 data "archive_file" "schism_processing_zip" {
   type = "zip"
-  output_path = "${path.module}/temp/viz_schism_fim_processing__${var.environment}_${var.region}.zip"
+  output_path = "${path.module}/temp/viz_schism_fim_processing_${var.environment}_${var.region}.zip"
 
   dynamic "source" {
     for_each = fileset("${path.module}/viz_schism_fim_processing", "**")
@@ -593,6 +599,8 @@ resource "null_resource" "viz_schism_fim_processing_cluster" {
     source_hash = data.archive_file.schism_processing_zip.output_md5
   }
 
+  depends_on = [ aws_s3_object.schism_processing_zip_upload ]
+
   provisioner "local-exec" {
     command = "aws codebuild start-build --project-name ${aws_codebuild_project.viz_schism_fim_processing_lambda.name} --profile ${var.environment} --region ${var.region}"
   }

diff --git a/Core/LAMBDA/viz_functions/viz_publish_service/lambda_function.py b/Core/LAMBDA/viz_functions/viz_publish_service/lambda_function.py
@@ -48,7 +48,7 @@ def lambda_handler(event, context):
 
     # Check to see if the service already exists and a publish flag is present or not.
     time.sleep(30)
-    matching_services = [service for service in publish_server.services.list(folder=folder) if service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish]  # noqa: E501
+    matching_services = [service for service in publish_server.services.list(folder=folder) if 'serviceName' in service.properties and (service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish)]  # noqa: E501
     publish_flag = s3_file(publish_flag_bucket, publish_flag_key).check_existence()
     if len(matching_services) > 0 and publish_flag is True:
         print(f"{matching_services[0].properties['serviceName']} is already online.")
@@ -77,15 +77,15 @@ def lambda_handler(event, context):
             print(f"---> Published {sd_s3_path}")
 
             # Ensuring that the description for the service matches the iteminfo
-            matching_service = [service for service in publish_server.services.list(folder=folder) if service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish][0]
+            matching_service = [service for service in publish_server.services.list(folder=folder) if 'serviceName' in service.properties and (service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish)][0]
             if not matching_service.properties['description']:
                 print("Updating service property description to match iteminfo")
                 service_properties = matching_service.properties
                 service_properties['description'] = matching_service.iteminformation.properties['description']
                 try:
                     matching_service.edit(dict(service_properties))
                 except:
-                    matching_service = [service for service in publish_server.services.list(folder=folder) if service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish][0]
+                    matching_service = [service for service in publish_server.services.list(folder=folder) if 'serviceName' in service.properties and (service.properties['serviceName'] == service_name or service.properties['serviceName'] == service_name_publish)][0]
                     if not matching_service.properties['description']:
                         raise Exception("Failed to update the map service description")
 

diff --git a/Core/Redshift/viz/main.tf b/Core/Redshift/viz/main.tf
@@ -62,9 +62,13 @@ resource "aws_route53_record" "viz_redshift_data_warehouse" {
   name    = "redshift-viz.${var.private_route_53_zone.name}"
   type    = "CNAME"
   ttl     = 300
-  records = [aws_redshift_cluster.viz_redshift_data_warehouse.address]
+  records = [aws_redshift_cluster.viz_redshift_data_warehouse.dns_name]
 }
 
 output "dns_name" {
   value = aws_route53_record.viz_redshift_data_warehouse.name
+}
+
+output "port" {
+  value = aws_redshift_cluster.viz_redshift_data_warehouse.port
 }
diff --git a/Core/SecretsManager/main.tf b/Core/SecretsManager/main.tf
@@ -19,6 +19,6 @@ output "secret_strings" {
   value = { for name in keys(var.names_and_users) : name => module.secret[name].secret_string }
 }
 
-output "arns" {
+output "secret_arns" {
   value = { for name in keys(var.names_and_users) : name => module.secret[name].secret_arn }
 }
diff --git a/Core/SecurityGroups/main.tf b/Core/SecurityGroups/main.tf
@@ -158,7 +158,7 @@ resource "aws_security_group" "redshift" {
       protocol         = "tcp"
       security_groups  = []
       self             = false
-      to_port          = 22
+      to_port          = 5439
     }
   ]
 

diff --git a/Core/main.tf b/Core/main.tf
@@ -111,8 +111,8 @@ module "secrets-manager" {
     "viz-processing-pg-rdssecret"         = { "username" : "postgres" }
     "viz-proc-admin-rw-user"              = { "username" : "viz_proc_admin_rw_user" }
     "viz-proc-dev-rw-user"                = { "username" : "viz_proc_dev_rw_user" }
-    "viz_redshift_master"                 = { "username" : "viz_redshift_master" }
-    "viz_redshift_user"                   = { "username" : "viz_redshift_user" }
+    "viz-redshift-master"                 = { "username" : "viz_redshift_master" }
+    "viz-redshift-user"                   = { "username" : "viz_redshift_user" }
     "ingest-pg-rdssecret"                 = { "username" : "postgres" }
     "ingest-mqsecret"                     = { "username" : "rabbit_admin" }
     "rds-rfc-fcst"                        = { "username" : "rfc_fcst" }
@@ -352,7 +352,7 @@ module "redshift-viz" {
   db_viz_redshift_user_secret_string      = module.secrets-manager.secret_strings["viz-redshift-user"]
   db_viz_redshift_security_groups         = [module.security-groups.redshift.id]
   viz_redshift_db_name                    = local.env.viz_redshift_db_name
-  role_viz_redshift_arn                   = module.iam-roles.redshift.arn
+  role_viz_redshift_arn                   = module.iam-roles.role_redshift.arn
 
   private_route_53_zone = module.private-route53.zone
 }
@@ -416,12 +416,12 @@ module "rds-bastion" {
   viz_db_address                    = module.rds-viz.instance.address
   viz_db_port                       = module.rds-viz.instance.port
   viz_db_name                       = local.env.viz_db_name
-  viz_redshift_master_secret_string = module.secrets-manager.secret_strings["viz_redshift_master"]
-  viz_redshift_user_secret_string   = module.secrets-manager.secret_strings["viz_redshift_user"]
+  viz_redshift_master_secret_string = module.secrets-manager.secret_strings["viz-redshift-master"]
+  viz_redshift_user_secret_string   = module.secrets-manager.secret_strings["viz-redshift-user"]
   viz_redshift_address              = module.redshift-viz.dns_name
   viz_redshift_port                 = module.redshift-viz.port
   viz_redshift_name                 = local.env.viz_redshift_db_name
-  viz_redshift_iam_role             = module.iam-roles.redshift.arn
+  viz_redshift_iam_role_arn         = module.iam-roles.role_redshift.arn
   egis_db_master_secret_string      = module.secrets-manager.secret_strings["egis-master-pg-rds-secret"]
   egis_db_secret_string             = module.secrets-manager.secret_strings["egis-pg-rds-secret"]
   egis_db_address                   = module.rds-egis.dns_name
@@ -642,7 +642,7 @@ module "viz-lambda-functions" {
   egis_portal_password              = local.env.viz_ec2_hydrovis_egis_pass
   viz_redshift_host                 = module.redshift-viz.dns_name
   viz_redshift_db_name              = local.env.viz_redshift_db_name
-  viz_redshift_user_secret_string   = module.secrets-manager.secret_strings["viz_redshift_user"]
+  viz_redshift_user_secret_string   = module.secrets-manager.secret_strings["viz-redshift-user"]
   dataservices_host                 = module.data-services.dns_name
   viz_pipeline_step_function_arn    = module.step-functions.viz_pipeline_step_function.arn
   default_tags                      = local.env.tags