Merge pull request #143 from wri/develop

Merge private subnet changes up

Author: dmannarino

src/Dockerfile

@@ -15,7 +15,7 @@ RUN pip install . -t python
 # to change the hash of the file and get TF to realize it needs to be
 # redeployed. Ticket for a better solution:
 # https://gfw.atlassian.net/browse/GTC-1250
-# change 2
+# change 3
 
 RUN yum install -y zip geos-devel
 

src/datapump/globals.py

@@ -30,8 +30,8 @@ class Globals(EnvSettings):
     s3_bucket_data_lake: str = Field(env="S3_BUCKET_DATA_LAKE")
     s3_glad_path: Optional[str] = Field(env="S3_GLAD_PATH")
     ec2_key_name: Optional[str] = Field("", env="EC2_KEY_NAME")
-    public_subnet_ids: List[str] = Field(
-        json.loads(os.environ.get("PUBLIC_SUBNET_IDS", b"[]"))
+    subnet_ids: List[str] = Field(
+        json.loads(os.environ.get("SUBNET_IDS", b"[]"))
     )
     emr_instance_profile: Optional[str] = Field("", env="EMR_INSTANCE_PROFILE")
     emr_service_role: Optional[str] = Field("", env="EMR_SERVICE_ROLE")

src/datapump/jobs/geotrellis.py

@@ -908,8 +908,8 @@ def _instances(worker_count: int) -> Dict[str, Any]:
         if GLOBALS.ec2_key_name:
             instances["Ec2KeyName"] = GLOBALS.ec2_key_name
 
-        if GLOBALS.public_subnet_ids:
-            instances["Ec2SubnetIds"] = GLOBALS.public_subnet_ids
+        if GLOBALS.subnet_ids:
+            instances["Ec2SubnetIds"] = GLOBALS.subnet_ids
 
         return instances
 

terraform/main.tf

@@ -54,7 +54,7 @@ module "datapump" {
   glad_path = local.glad_path
   emr_instance_profile_name = data.terraform_remote_state.core.outputs.emr_instance_profile_name
   emr_service_role_name = data.terraform_remote_state.core.outputs.emr_service_role_name
-  public_subnet_ids = data.terraform_remote_state.core.outputs.public_subnet_ids
+  subnet_ids = data.terraform_remote_state.core.outputs.private_subnet_ids
   ec2_key_name = data.terraform_remote_state.core.outputs.key_pair_jterry_gfw
   gcs_secret_arn = data.terraform_remote_state.core.outputs.secrets_read-gfw-gee-export_arn
   read_gfw_api_secrets_policy = data.terraform_remote_state.core.outputs.secrets_read-gfw-api-token_policy_arn

terraform/modules/datapump/cloudwatch.tf

@@ -5,6 +5,13 @@ resource "aws_cloudwatch_event_rule" "everyday-11-pm-est" {
   tags                = local.tags
 }
 
+resource "aws_cloudwatch_event_rule" "everyday-10-pm-est" {
+  name                = substr("everyday-10-pm-est${local.name_suffix}", 0, 64)
+  description         = "Run everyday at 10 pm EST"
+  schedule_expression = "cron(0 6 ? * * *)"
+  tags                = local.tags
+}
+
 resource "aws_cloudwatch_event_rule" "everyday-7-pm-est" {
   name                = substr("everyday-7-pm-est${local.name_suffix}", 0, 64)
   description         = "Run everyday at 7 pm EST"
@@ -38,7 +45,7 @@ resource "aws_cloudwatch_event_target" "sync-deforestation-alerts" {
 }
 
 resource "aws_cloudwatch_event_target" "sync-glad" {
-  rule      = aws_cloudwatch_event_rule.everyday-11-pm-est.name
+  rule      = aws_cloudwatch_event_rule.everyday-10-pm-est.name
   target_id = substr("${local.project}-sync-glad${local.name_suffix}", 0, 64)
   arn       = aws_sfn_state_machine.datapump.id
   input    = "{\"command\": \"sync\", \"parameters\": {\"types\": [\"glad\"]}}"

terraform/modules/datapump/lambdas.tf

@@ -45,7 +45,7 @@ resource "aws_lambda_function" "executor" {
       S3_BUCKET_PIPELINE             = var.pipelines_bucket
       S3_BUCKET_DATA_LAKE            = var.data_lake_bucket
       GEOTRELLIS_JAR_PATH            = var.geotrellis_jar_path
-      PUBLIC_SUBNET_IDS              = jsonencode(var.public_subnet_ids)
+      SUBNET_IDS                     = jsonencode(var.subnet_ids)
       EC2_KEY_NAME                   = var.ec2_key_name
       EMR_SERVICE_ROLE               = var.emr_service_role_name
       EMR_INSTANCE_PROFILE           = var.emr_instance_profile_name

terraform/modules/datapump/variables.tf

@@ -65,10 +65,10 @@ variable "ec2_key_name" {
   description = "Key pair to use for SSHing into EC2"
 }
 
-variable "public_subnet_ids" {
+variable "subnet_ids" {
   default     = []
   type        = list(string)
-  description = "Public subnet IDs to run on"
+  description = "Subnet IDs to run on"
 }
 
 variable "pipelines_bucket" {