diff --git a/src/Dockerfile b/src/Dockerfile index 8da1108a..e713cc80 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -15,7 +15,7 @@ RUN pip install . -t python # to change the hash of the file and get TF to realize it needs to be # redeployed. Ticket for a better solution: # https://gfw.atlassian.net/browse/GTC-1250 -# change 2 +# change 3 RUN yum install -y zip geos-devel diff --git a/src/datapump/globals.py b/src/datapump/globals.py index 113e18c2..3592b020 100644 --- a/src/datapump/globals.py +++ b/src/datapump/globals.py @@ -30,8 +30,8 @@ class Globals(EnvSettings): s3_bucket_data_lake: str = Field(env="S3_BUCKET_DATA_LAKE") s3_glad_path: Optional[str] = Field(env="S3_GLAD_PATH") ec2_key_name: Optional[str] = Field("", env="EC2_KEY_NAME") - public_subnet_ids: List[str] = Field( - json.loads(os.environ.get("PUBLIC_SUBNET_IDS", b"[]")) + subnet_ids: List[str] = Field( + json.loads(os.environ.get("SUBNET_IDS", b"[]")) ) emr_instance_profile: Optional[str] = Field("", env="EMR_INSTANCE_PROFILE") emr_service_role: Optional[str] = Field("", env="EMR_SERVICE_ROLE") diff --git a/src/datapump/jobs/geotrellis.py b/src/datapump/jobs/geotrellis.py index d0c89233..e33844da 100644 --- a/src/datapump/jobs/geotrellis.py +++ b/src/datapump/jobs/geotrellis.py @@ -908,8 +908,8 @@ def _instances(worker_count: int) -> Dict[str, Any]: if GLOBALS.ec2_key_name: instances["Ec2KeyName"] = GLOBALS.ec2_key_name - if GLOBALS.public_subnet_ids: - instances["Ec2SubnetIds"] = GLOBALS.public_subnet_ids + if GLOBALS.subnet_ids: + instances["Ec2SubnetIds"] = GLOBALS.subnet_ids return instances diff --git a/terraform/main.tf b/terraform/main.tf index 91b1f600..a21150b7 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -54,7 +54,7 @@ module "datapump" { glad_path = local.glad_path emr_instance_profile_name = data.terraform_remote_state.core.outputs.emr_instance_profile_name emr_service_role_name = data.terraform_remote_state.core.outputs.emr_service_role_name - public_subnet_ids = data.terraform_remote_state.core.outputs.public_subnet_ids + subnet_ids = data.terraform_remote_state.core.outputs.private_subnet_ids ec2_key_name = data.terraform_remote_state.core.outputs.key_pair_jterry_gfw gcs_secret_arn = data.terraform_remote_state.core.outputs.secrets_read-gfw-gee-export_arn read_gfw_api_secrets_policy = data.terraform_remote_state.core.outputs.secrets_read-gfw-api-token_policy_arn diff --git a/terraform/modules/datapump/cloudwatch.tf b/terraform/modules/datapump/cloudwatch.tf index 18a6e343..46b69e86 100644 --- a/terraform/modules/datapump/cloudwatch.tf +++ b/terraform/modules/datapump/cloudwatch.tf @@ -5,6 +5,13 @@ resource "aws_cloudwatch_event_rule" "everyday-11-pm-est" { tags = local.tags } +resource "aws_cloudwatch_event_rule" "everyday-10-pm-est" { + name = substr("everyday-10-pm-est${local.name_suffix}", 0, 64) + description = "Run everyday at 10 pm EST" + schedule_expression = "cron(0 6 ? * * *)" + tags = local.tags +} + resource "aws_cloudwatch_event_rule" "everyday-7-pm-est" { name = substr("everyday-7-pm-est${local.name_suffix}", 0, 64) description = "Run everyday at 7 pm EST" @@ -38,7 +45,7 @@ resource "aws_cloudwatch_event_target" "sync-deforestation-alerts" { } resource "aws_cloudwatch_event_target" "sync-glad" { - rule = aws_cloudwatch_event_rule.everyday-11-pm-est.name + rule = aws_cloudwatch_event_rule.everyday-10-pm-est.name target_id = substr("${local.project}-sync-glad${local.name_suffix}", 0, 64) arn = aws_sfn_state_machine.datapump.id input = "{\"command\": \"sync\", \"parameters\": {\"types\": [\"glad\"]}}" diff --git a/terraform/modules/datapump/lambdas.tf b/terraform/modules/datapump/lambdas.tf index 20592acc..ea880fd2 100644 --- a/terraform/modules/datapump/lambdas.tf +++ b/terraform/modules/datapump/lambdas.tf @@ -45,7 +45,7 @@ resource "aws_lambda_function" "executor" { S3_BUCKET_PIPELINE = var.pipelines_bucket S3_BUCKET_DATA_LAKE = var.data_lake_bucket GEOTRELLIS_JAR_PATH = var.geotrellis_jar_path - PUBLIC_SUBNET_IDS = jsonencode(var.public_subnet_ids) + SUBNET_IDS = jsonencode(var.subnet_ids) EC2_KEY_NAME = var.ec2_key_name EMR_SERVICE_ROLE = var.emr_service_role_name EMR_INSTANCE_PROFILE = var.emr_instance_profile_name diff --git a/terraform/modules/datapump/variables.tf b/terraform/modules/datapump/variables.tf index ad188ec6..62c9364e 100644 --- a/terraform/modules/datapump/variables.tf +++ b/terraform/modules/datapump/variables.tf @@ -65,10 +65,10 @@ variable "ec2_key_name" { description = "Key pair to use for SSHing into EC2" } -variable "public_subnet_ids" { +variable "subnet_ids" { default = [] type = list(string) - description = "Public subnet IDs to run on" + description = "Subnet IDs to run on" } variable "pipelines_bucket" {