Merge pull request #141 from wri/emr_use_private_subnets

GTC-2786 : Use private subnets for EMR
wri · Apr 25, 2024 · 4888c72 · 4888c72
2 parents 8faa931 + c4c48c5
commit 4888c72
Show file tree

Hide file tree

Showing 6 changed files with 9 additions and 9 deletions.
diff --git a/src/Dockerfile b/src/Dockerfile
@@ -15,7 +15,7 @@ RUN pip install . -t python
 # to change the hash of the file and get TF to realize it needs to be
 # redeployed. Ticket for a better solution:
 # https://gfw.atlassian.net/browse/GTC-1250
-# change 1
+# change 3
 
 RUN yum install -y zip geos-devel
 

diff --git a/src/datapump/globals.py b/src/datapump/globals.py
@@ -30,8 +30,8 @@ class Globals(EnvSettings):
     s3_bucket_data_lake: str = Field(env="S3_BUCKET_DATA_LAKE")
     s3_glad_path: Optional[str] = Field(env="S3_GLAD_PATH")
     ec2_key_name: Optional[str] = Field("", env="EC2_KEY_NAME")
-    public_subnet_ids: List[str] = Field(
-        json.loads(os.environ.get("PUBLIC_SUBNET_IDS", b"[]"))
+    subnet_ids: List[str] = Field(
+        json.loads(os.environ.get("SUBNET_IDS", b"[]"))
     )
     emr_instance_profile: Optional[str] = Field("", env="EMR_INSTANCE_PROFILE")
     emr_service_role: Optional[str] = Field("", env="EMR_SERVICE_ROLE")

diff --git a/src/datapump/jobs/geotrellis.py b/src/datapump/jobs/geotrellis.py
@@ -909,8 +909,8 @@ def _instances(worker_count: int) -> Dict[str, Any]:
         if GLOBALS.ec2_key_name:
             instances["Ec2KeyName"] = GLOBALS.ec2_key_name
 
-        if GLOBALS.public_subnet_ids:
-            instances["Ec2SubnetIds"] = GLOBALS.public_subnet_ids
+        if GLOBALS.subnet_ids:
+            instances["Ec2SubnetIds"] = GLOBALS.subnet_ids
 
         return instances
 

diff --git a/terraform/main.tf b/terraform/main.tf
@@ -54,7 +54,7 @@ module "datapump" {
   glad_path = local.glad_path
   emr_instance_profile_name = data.terraform_remote_state.core.outputs.emr_instance_profile_name
   emr_service_role_name = data.terraform_remote_state.core.outputs.emr_service_role_name
-  public_subnet_ids = data.terraform_remote_state.core.outputs.public_subnet_ids
+  subnet_ids = data.terraform_remote_state.core.outputs.private_subnet_ids
   ec2_key_name = data.terraform_remote_state.core.outputs.key_pair_jterry_gfw
   gcs_secret_arn = data.terraform_remote_state.core.outputs.secrets_read-gfw-gee-export_arn
   read_gfw_api_secrets_policy = data.terraform_remote_state.core.outputs.secrets_read-gfw-api-token_policy_arn

diff --git a/terraform/modules/datapump/lambdas.tf b/terraform/modules/datapump/lambdas.tf
@@ -45,7 +45,7 @@ resource "aws_lambda_function" "executor" {
       S3_BUCKET_PIPELINE             = var.pipelines_bucket
       S3_BUCKET_DATA_LAKE            = var.data_lake_bucket
       GEOTRELLIS_JAR_PATH            = var.geotrellis_jar_path
-      PUBLIC_SUBNET_IDS              = jsonencode(var.public_subnet_ids)
+      SUBNET_IDS                     = jsonencode(var.subnet_ids)
       EC2_KEY_NAME                   = var.ec2_key_name
       EMR_SERVICE_ROLE               = var.emr_service_role_name
       EMR_INSTANCE_PROFILE           = var.emr_instance_profile_name

diff --git a/terraform/modules/datapump/variables.tf b/terraform/modules/datapump/variables.tf
@@ -65,10 +65,10 @@ variable "ec2_key_name" {
   description = "Key pair to use for SSHing into EC2"
 }
 
-variable "public_subnet_ids" {
+variable "subnet_ids" {
   default     = []
   type        = list(string)
-  description = "Public subnet IDs to run on"
+  description = "Subnet IDs to run on"
 }
 
 variable "pipelines_bucket" {