From ee2b7d31aebdabe51c9b8570c7aff8c593b8ad45 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.almeida@liveramp.com>
Date: Thu, 23 May 2024 21:40:59 +0100
Subject: [PATCH 1/6] experiment - modularize network

---
 cloud_router.tf                             | 31 ------------------
 iam.tf                                      |  8 -----
 firewalls.tf => modules/network/firewall.tf | 23 ++++++-------
 modules/network/iam.tf                      |  6 ++++
 modules/network/router.tf                   | 30 +++++++++++++++++
 modules/network/subnetwork.tf               |  8 +++++
 modules/network/variables.tf                | 36 +++++++++++++++++++++
 network.tf                                  | 26 +++++++++------
 outputs.tf                                  |  5 +++
 variables.tf                                |  6 ++--
 10 files changed, 114 insertions(+), 65 deletions(-)
 rename firewalls.tf => modules/network/firewall.tf (81%)
 create mode 100644 modules/network/iam.tf
 create mode 100644 modules/network/router.tf
 create mode 100644 modules/network/subnetwork.tf
 create mode 100644 modules/network/variables.tf

diff --git a/cloud_router.tf b/cloud_router.tf
index 2ad605e..e69de29 100644
--- a/cloud_router.tf
+++ b/cloud_router.tf
@@ -1,31 +0,0 @@
-resource "google_compute_address" "cloud_nat_static_ip_address" {
-  project = var.data_plane_project
-  region  = var.gcp_region
-  count   = var.enable_dataproc_network ? 2 : 0
-  name    = lower("${var.installation_name}-${var.gcp_region}-nat-ip-${count.index}")
-}
-
-module "cloud_router" {
-  count   = var.enable_dataproc_network ? 1 : 0
-  source  = "terraform-google-modules/cloud-router/google"
-  version = "~> 6.0"
-  name    = lower("${var.installation_name}-${var.gcp_region}-router")
-  project = var.data_plane_project
-  network = google_compute_network.vpc_network[0].name
-  region  = var.gcp_region
-
-  nats = [{
-    name                               = lower("${var.installation_name}-${var.gcp_region}-nat")
-    nat_ip_allocate_option             = "MANUAL_ONLY"
-    source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
-    nat_ips                            = google_compute_address.cloud_nat_static_ip_address.*.self_link
-    min_ports_per_vm                   = 256 # Increased from default of 64
-    tcp_established_idle_timeout_sec   = 300 # Decreased from default of 1200s
-    tcp_transitory_idle_timeout_sec    = 180 # Increased from default of 30s
-
-    log_config = {
-      enable = true
-      filter = "ERRORS_ONLY"
-    }
-  }]
-}
diff --git a/iam.tf b/iam.tf
index 3014c9e..b88a1bc 100644
--- a/iam.tf
+++ b/iam.tf
@@ -16,14 +16,6 @@ resource "google_project_iam_member" "dataproc_editor" {
   member  = "serviceAccount:${google_service_account.tenant_data_access.email}"
 }
 
-resource "google_compute_subnetwork_iam_member" "vpc_subnetwork_user" {
-  count      = var.enable_dataproc_network ? 1 : 0
-  project    = var.data_plane_project
-  subnetwork = google_compute_subnetwork.dataproc_subnet[0].name
-  role       = "roles/compute.networkUser"
-  member     = "serviceAccount:${google_service_account.tenant_data_access.email}"
-}
-
 resource "google_project_iam_member" "bigquery_job_creator" {
   project = var.data_plane_project
   role    = "roles/bigquery.jobUser"
diff --git a/firewalls.tf b/modules/network/firewall.tf
similarity index 81%
rename from firewalls.tf
rename to modules/network/firewall.tf
index 7408852..e0d458c 100644
--- a/firewalls.tf
+++ b/modules/network/firewall.tf
@@ -1,8 +1,7 @@
 resource "google_compute_firewall" "allow_metastore_egress" {
-  count       = var.enable_dataproc_network ? 1 : 0
-  project     = google_compute_network.vpc_network[0].project
+  project     = var.project_id
   name        = "allow-${var.installation_name}-metastore-egress"
-  network     = google_compute_network.vpc_network[0].name
+  network     = var.network_name
   direction   = "EGRESS"
   priority    = "1000"
   description = "Allow EGRESS to Identity Engine Metastore CloudSQL instance"
@@ -14,17 +13,14 @@ resource "google_compute_firewall" "allow_metastore_egress" {
       "3306"
     ]
   }
-
-  destination_ranges = [
-    var.metastore_cidr_ip_address
-  ]
+  destination_ranges = [var.metastore_cidr_ip_address]
 }
 
+
 resource "google_compute_firewall" "allow_idapi_egress" {
-  count       = var.enable_dataproc_network ? 1 : 0
-  project     = google_compute_network.vpc_network[0].project
+  project     = var.project_id
   name        = "allow-${var.installation_name}-idapi-egress"
-  network     = google_compute_network.vpc_network[0].name
+  network     = var.network_name
   direction   = "EGRESS"
   priority    = "1000"
   description = "Allow EGRESS to LiveRamp ID-API instance"
@@ -36,16 +32,15 @@ resource "google_compute_firewall" "allow_idapi_egress" {
       "443"
     ]
   }
-
   destination_ranges = var.idapi_cidr_ip_addresses
 }
 
+
 module "dataproc-firewall-rules" {
-  count        = var.enable_dataproc_network ? 1 : 0
   source       = "terraform-google-modules/network/google//modules/firewall-rules"
   version      = "6.0.1"
-  project_id   = var.data_plane_project
-  network_name = google_compute_network.vpc_network[0].name
+  project_id   = var.project_id
+  network_name = var.network_name
 
   rules = [
     {
diff --git a/modules/network/iam.tf b/modules/network/iam.tf
new file mode 100644
index 0000000..0badc00
--- /dev/null
+++ b/modules/network/iam.tf
@@ -0,0 +1,6 @@
+resource "google_compute_subnetwork_iam_member" "subnet_user" {
+  project    = var.project_id
+  subnetwork = google_compute_subnetwork.dataproc_subnet[0].name
+  role       = "roles/compute.networkUser"
+  member     = var.subnet_user
+}
diff --git a/modules/network/router.tf b/modules/network/router.tf
new file mode 100644
index 0000000..bdadb43
--- /dev/null
+++ b/modules/network/router.tf
@@ -0,0 +1,30 @@
+resource "google_compute_address" "cloud_nat_static_ip_address" {
+  count   = 2
+  project = var.project_id
+  region  = var.region
+  name    = lower("${var.installation_name}-${var.region}-nat-ip-${count.index}")
+}
+
+module "cloud_router" {
+  source  = "terraform-google-modules/cloud-router/google"
+  version = "~> 6.0"
+  name    = lower("${var.installation_name}-${var.region}-router")
+  project = var.project_id
+  network = google_compute_network.vpc_network[0].name
+  region  = var.region
+
+  nats = [{
+    name                               = lower("${var.installation_name}-${var.region}-nat")
+    nat_ip_allocate_option             = "MANUAL_ONLY"
+    source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+    nat_ips                            = google_compute_address.cloud_nat_static_ip_address.*.self_link
+    min_ports_per_vm                   = 256 # Increased from default of 64
+    tcp_established_idle_timeout_sec   = 300 # Decreased from default of 1200s
+    tcp_transitory_idle_timeout_sec    = 180 # Increased from default of 30s
+
+    log_config = {
+      enable = true
+      filter = "ERRORS_ONLY"
+    }
+  }]
+}
diff --git a/modules/network/subnetwork.tf b/modules/network/subnetwork.tf
new file mode 100644
index 0000000..129f90d
--- /dev/null
+++ b/modules/network/subnetwork.tf
@@ -0,0 +1,8 @@
+resource "google_compute_subnetwork" "dataproc_subnet" {
+  project                  = var.project_id
+  region                   = var.region
+  name                     = lower("${var.installation_name}-${var.country_code}-dataproc")
+  ip_cidr_range            = var.subnet_ip4_cidr
+  network                  = var.network_name
+  private_ip_google_access = true
+}
\ No newline at end of file
diff --git a/modules/network/variables.tf b/modules/network/variables.tf
new file mode 100644
index 0000000..5590eb6
--- /dev/null
+++ b/modules/network/variables.tf
@@ -0,0 +1,36 @@
+variable "installation_name" {
+    type = string
+}
+
+variable "country_code" {
+    type = string
+}
+
+variable "project_id" {
+    type = string
+}
+
+variable "region" {
+    type = string
+}
+
+variable "network_name" {
+    type = string
+}
+
+variable "subnet_ip4_cidr" {
+    type = string
+}
+
+variable "subnet_user" {
+    type = string
+}
+
+variable "metastore_cidr_ip_address" {
+  type        = string
+}
+
+variable "idapi_cidr_ip_addresses" {
+  type        = list(string)
+  default     = []
+}
diff --git a/network.tf b/network.tf
index 67d29d4..a41ebb0 100644
--- a/network.tf
+++ b/network.tf
@@ -1,5 +1,5 @@
 resource "google_compute_network" "vpc_network" {
-  count                   = var.enable_dataproc_network ? 1 : 0
+  count                   = try(var.vpc_network_name, false)
   project                 = var.data_plane_project
   name                    = lower("${var.installation_name}-${var.country_code}-vpc")
   auto_create_subnetworks = false
@@ -7,12 +7,20 @@ resource "google_compute_network" "vpc_network" {
   description             = "The shared network for the identity graph"
 }
 
-resource "google_compute_subnetwork" "dataproc_subnet" {
-  count                    = var.enable_dataproc_network ? 1 : 0
-  project                  = var.data_plane_project
-  region                   = var.gcp_region
-  name                     = lower("${var.installation_name}-${var.country_code}-dataproc")
-  ip_cidr_range            = var.dataproc_subnet_ip4_cidr
-  network                  = google_compute_network.vpc_network[0].id
-  private_ip_google_access = true
+data "compute" "vpc_network" {
+  project = data_plane_project
+  name    = try(var.vpc_network_name, google_compute_network.vpc_network[0].name)
+}
+
+module "network" {
+  source                    = "./modules/network"
+  installation_name         = var.installation_name
+  country_code              = var.country_code
+  project_id                = var.data_plane_project
+  region                    = var.gcp_region
+  network_name              = data.compute.vpc_network
+  subnet_ip4_cidr           = var.dataproc_subnet_ip4_cidr
+  subnet_user               = "serviceAccount:${google_service_account.tenant_data_access.email}"
+  idapi_cidr_ip_addresses   = var.idapi_cidr_ip_addresses
+  metastore_cidr_ip_address = var.metastore_cidr_ip_address
 }
diff --git a/outputs.tf b/outputs.tf
index 1a6d8f4..56ff765 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -22,3 +22,8 @@ output "output_bucket_name" {
   value       = google_storage_bucket.tenant_output_bucket.name
   description = "The name of the GCS bucket that will be used to store the output files"
 }
+
+output "vpc_network_name" {
+  value       = google_compute_network.vpc_network.name
+  description = "The name of the VPC network"
+}
diff --git a/variables.tf b/variables.tf
index 3e57b44..aae764d 100644
--- a/variables.tf
+++ b/variables.tf
@@ -94,10 +94,10 @@ variable "idapi_cidr_ip_addresses" {
   description = "Portrait Engine ID-API instance CIDR IP addresses"
 }
 
-variable "enable_dataproc_network" {
+variable "vpc_network_name" {
   type        = bool
-  description = "Configure network bits for Dataproc - VPC, firewall rules etc"
-  default     = true
+  description = "The network to connect the data-plane to, if not specified, module will provision a dedicated one"
+  default     = null
 }
 
 variable "enable_kms" {

From 0f17e6240bbc91ca23859ba303e504ee21030a68 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.almeida@liveramp.com>
Date: Thu, 23 May 2024 21:49:50 +0100
Subject: [PATCH 2/6] experiment - modularize network

---
 cloud_router.tf | 0
 network.tf      | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 cloud_router.tf

diff --git a/cloud_router.tf b/cloud_router.tf
deleted file mode 100644
index e69de29..0000000
diff --git a/network.tf b/network.tf
index a41ebb0..51d83d0 100644
--- a/network.tf
+++ b/network.tf
@@ -8,7 +8,7 @@ resource "google_compute_network" "vpc_network" {
 }
 
 data "compute" "vpc_network" {
-  project = data_plane_project
+  project = var.data_plane_project
   name    = try(var.vpc_network_name, google_compute_network.vpc_network[0].name)
 }
 

From be9311dfb1fdc52143a2fcc7e1f8e8292297c638 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.almeida@liveramp.com>
Date: Thu, 23 May 2024 21:53:39 +0100
Subject: [PATCH 3/6] experiment - modularize network

---
 README.md | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index b84c9ee..5b3c06f 100644
--- a/README.md
+++ b/README.md
@@ -6,6 +6,7 @@ No requirements.
 
 | Name | Version |
 |------|---------|
+| <a name="provider_compute"></a> [compute](#provider\_compute) | n/a |
 | <a name="provider_google"></a> [google](#provider\_google) | n/a |
 | <a name="provider_google-beta"></a> [google-beta](#provider\_google-beta) | n/a |
 | <a name="provider_random"></a> [random](#provider\_random) | n/a |
@@ -14,9 +15,8 @@ No requirements.
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_cloud_router"></a> [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 6.0 |
-| <a name="module_dataproc-firewall-rules"></a> [dataproc-firewall-rules](#module\_dataproc-firewall-rules) | terraform-google-modules/network/google//modules/firewall-rules | 6.0.1 |
 | <a name="module_kms_crypto_key-iam-bindings"></a> [kms\_crypto\_key-iam-bindings](#module\_kms\_crypto\_key-iam-bindings) | terraform-google-modules/iam/google//modules/kms_crypto_keys_iam | n/a |
+| <a name="module_network"></a> [network](#module\_network) | ./modules/network | n/a |
 
 ## Resources
 
@@ -26,12 +26,7 @@ No requirements.
 | [google-beta_google_storage_bucket.tenant_input_bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource |
 | [google-beta_google_storage_bucket.tenant_output_bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource |
 | [google_bigquery_dataset.tenant_dataset](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource |
-| [google_compute_address.cloud_nat_static_ip_address](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource |
-| [google_compute_firewall.allow_idapi_egress](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource |
-| [google_compute_firewall.allow_metastore_egress](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource |
 | [google_compute_network.vpc_network](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network) | resource |
-| [google_compute_subnetwork.dataproc_subnet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork) | resource |
-| [google_compute_subnetwork_iam_member.vpc_subnetwork_user](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork_iam_member) | resource |
 | [google_kms_crypto_key.tenant_crypto_key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/kms_crypto_key) | resource |
 | [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/kms_key_ring) | resource |
 | [google_project_iam_member.allow_bq_connector_push_down](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
@@ -47,6 +42,7 @@ No requirements.
 | [google_storage_bucket_iam_policy.tenant_input_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_policy) | resource |
 | [google_storage_bucket_iam_policy.tenant_output_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_policy) | resource |
 | [random_id.generator](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource |
+| [compute_compute.vpc_network](https://registry.terraform.io/providers/hashicorp/compute/latest/docs/data-sources/compute) | data source |
 | [google_iam_policy.tenant_build_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/iam_policy) | data source |
 | [google_iam_policy.tenant_input_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/iam_policy) | data source |
 | [google_iam_policy.tenant_output_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/iam_policy) | data source |
@@ -65,7 +61,6 @@ No requirements.
 | <a name="input_data_retention_period_days"></a> [data\_retention\_period\_days](#input\_data\_retention\_period\_days) | The number of days this customers data will be stored before its automatically deleted | `number` | `0` | no |
 | <a name="input_data_viewers"></a> [data\_viewers](#input\_data\_viewers) | The users, groups & service accounts that should have read only access to this customers data | <pre>object({<br>    service_accounts = list(string)<br>    groups           = list(string)<br>    users            = list(string)<br>  })</pre> | n/a | yes |
 | <a name="input_dataproc_subnet_ip4_cidr"></a> [dataproc\_subnet\_ip4\_cidr](#input\_dataproc\_subnet\_ip4\_cidr) | Subnet used for Dataproc clusters | `string` | n/a | yes |
-| <a name="input_enable_dataproc_network"></a> [enable\_dataproc\_network](#input\_enable\_dataproc\_network) | Configure network bits for Dataproc - VPC, firewall rules etc | `bool` | `true` | no |
 | <a name="input_enable_kms"></a> [enable\_kms](#input\_enable\_kms) | Configure KMS to encrypt build, input and output buckets | `bool` | `true` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The environment this infrastructure is supported (eg.: dev, staging or prod) | `string` | n/a | yes |
 | <a name="input_gcp_region"></a> [gcp\_region](#input\_gcp\_region) | The GCP region to be used | `string` | n/a | yes |
@@ -81,6 +76,7 @@ No requirements.
 | <a name="input_storage_location"></a> [storage\_location](#input\_storage\_location) | The storage location for BigQuery and GCS. | `string` | n/a | yes |
 | <a name="input_tenant_orchestration_sa"></a> [tenant\_orchestration\_sa](#input\_tenant\_orchestration\_sa) | Tenant Orchestration ServiceAccount for remote execution | `string` | n/a | yes |
 | <a name="input_tenant_service_account_name"></a> [tenant\_service\_account\_name](#input\_tenant\_service\_account\_name) | Service Account name | `string` | `""` | no |
+| <a name="input_vpc_network_name"></a> [vpc\_network\_name](#input\_vpc\_network\_name) | The network to connect the data-plane to, if not specified, module will provision a dedicated one | `bool` | `null` | no |
 
 ## Outputs
 
@@ -91,3 +87,4 @@ No requirements.
 | <a name="output_output_bucket_name"></a> [output\_bucket\_name](#output\_output\_bucket\_name) | The name of the GCS bucket that will be used to store the output files |
 | <a name="output_tenant_bigquery_dataset_name"></a> [tenant\_bigquery\_dataset\_name](#output\_tenant\_bigquery\_dataset\_name) | The name of the BigQuery dataset that will be used to store the tenant data |
 | <a name="output_tenant_data_access_svc_account"></a> [tenant\_data\_access\_svc\_account](#output\_tenant\_data\_access\_svc\_account) | The service account object that will be used to access the tenant data |
+| <a name="output_vpc_network_name"></a> [vpc\_network\_name](#output\_vpc\_network\_name) | The name of the VPC network |

From b4e16068c1ec7575c15bfb98006b2bd211167ba4 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.almeida@liveramp.com>
Date: Thu, 23 May 2024 22:05:08 +0100
Subject: [PATCH 4/6] experiment - modularize network

---
 network.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/network.tf b/network.tf
index 51d83d0..ac1eab9 100644
--- a/network.tf
+++ b/network.tf
@@ -7,7 +7,7 @@ resource "google_compute_network" "vpc_network" {
   description             = "The shared network for the identity graph"
 }
 
-data "compute" "vpc_network" {
+data "google_compute_network" "vpc_network" {
   project = var.data_plane_project
   name    = try(var.vpc_network_name, google_compute_network.vpc_network[0].name)
 }
@@ -18,7 +18,7 @@ module "network" {
   country_code              = var.country_code
   project_id                = var.data_plane_project
   region                    = var.gcp_region
-  network_name              = data.compute.vpc_network
+  network_name              = data.google_compute_network.vpc_network
   subnet_ip4_cidr           = var.dataproc_subnet_ip4_cidr
   subnet_user               = "serviceAccount:${google_service_account.tenant_data_access.email}"
   idapi_cidr_ip_addresses   = var.idapi_cidr_ip_addresses

From 8197305e1119b24572fa3a5f52b0e7f04adabe0c Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.almeida@liveramp.com>
Date: Thu, 23 May 2024 22:31:35 +0100
Subject: [PATCH 5/6] experiment - modularize network

---
 modules/network/firewall.tf | 4 ++--
 modules/network/iam.tf      | 2 +-
 modules/network/router.tf   | 2 +-
 network.tf                  | 6 +++---
 outputs.tf                  | 2 +-
 variables.tf                | 2 +-
 6 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/modules/network/firewall.tf b/modules/network/firewall.tf
index e0d458c..083b544 100644
--- a/modules/network/firewall.tf
+++ b/modules/network/firewall.tf
@@ -48,7 +48,7 @@ module "dataproc-firewall-rules" {
       description             = "Allow Dataproc clusters to communicate over private IP to google APIs and other nodes"
       direction               = "INGRESS"
       priority                = 1000
-      ranges                  = [var.dataproc_subnet_ip4_cidr]
+      ranges                  = [var.subnet_ip4_cidr]
       source_tags             = null
       source_service_accounts = null
       target_tags             = null
@@ -75,7 +75,7 @@ module "dataproc-firewall-rules" {
       description             = "Allow Dataproc clusters to communicate over private IP to google APIs and other nodes"
       direction               = "EGRESS"
       priority                = 1000
-      ranges                  = [var.dataproc_subnet_ip4_cidr]
+      ranges                  = [var.subnet_ip4_cidr]
       source_tags             = null
       source_service_accounts = null
       target_tags             = null
diff --git a/modules/network/iam.tf b/modules/network/iam.tf
index 0badc00..caeeb6c 100644
--- a/modules/network/iam.tf
+++ b/modules/network/iam.tf
@@ -1,6 +1,6 @@
 resource "google_compute_subnetwork_iam_member" "subnet_user" {
   project    = var.project_id
-  subnetwork = google_compute_subnetwork.dataproc_subnet[0].name
+  subnetwork = google_compute_subnetwork.dataproc_subnet.name
   role       = "roles/compute.networkUser"
   member     = var.subnet_user
 }
diff --git a/modules/network/router.tf b/modules/network/router.tf
index bdadb43..74040e6 100644
--- a/modules/network/router.tf
+++ b/modules/network/router.tf
@@ -10,7 +10,7 @@ module "cloud_router" {
   version = "~> 6.0"
   name    = lower("${var.installation_name}-${var.region}-router")
   project = var.project_id
-  network = google_compute_network.vpc_network[0].name
+  network = var.network_name
   region  = var.region
 
   nats = [{
diff --git a/network.tf b/network.tf
index ac1eab9..ed9285e 100644
--- a/network.tf
+++ b/network.tf
@@ -1,5 +1,5 @@
 resource "google_compute_network" "vpc_network" {
-  count                   = try(var.vpc_network_name, false)
+  count                   = (var.vpc_network_name == null || var.vpc_network_name == "") ? 1 : 0
   project                 = var.data_plane_project
   name                    = lower("${var.installation_name}-${var.country_code}-vpc")
   auto_create_subnetworks = false
@@ -9,7 +9,7 @@ resource "google_compute_network" "vpc_network" {
 
 data "google_compute_network" "vpc_network" {
   project = var.data_plane_project
-  name    = try(var.vpc_network_name, google_compute_network.vpc_network[0].name)
+  name    = (var.vpc_network_name == null || var.vpc_network_name == "") ? google_compute_network.vpc_network[0].name : var.vpc_network_name
 }
 
 module "network" {
@@ -18,7 +18,7 @@ module "network" {
   country_code              = var.country_code
   project_id                = var.data_plane_project
   region                    = var.gcp_region
-  network_name              = data.google_compute_network.vpc_network
+  network_name              = data.google_compute_network.vpc_network.name
   subnet_ip4_cidr           = var.dataproc_subnet_ip4_cidr
   subnet_user               = "serviceAccount:${google_service_account.tenant_data_access.email}"
   idapi_cidr_ip_addresses   = var.idapi_cidr_ip_addresses
diff --git a/outputs.tf b/outputs.tf
index 56ff765..171a640 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -24,6 +24,6 @@ output "output_bucket_name" {
 }
 
 output "vpc_network_name" {
-  value       = google_compute_network.vpc_network.name
+  value       = data.google_compute_network.vpc_network.name
   description = "The name of the VPC network"
 }
diff --git a/variables.tf b/variables.tf
index aae764d..09b27c6 100644
--- a/variables.tf
+++ b/variables.tf
@@ -95,7 +95,7 @@ variable "idapi_cidr_ip_addresses" {
 }
 
 variable "vpc_network_name" {
-  type        = bool
+  type        = string
   description = "The network to connect the data-plane to, if not specified, module will provision a dedicated one"
   default     = null
 }

From 4c7e70d81928ba4fa0e7fee0da38c31dbaebdd91 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.almeida@liveramp.com>
Date: Fri, 24 May 2024 00:11:20 +0100
Subject: [PATCH 6/6] experiment - modularize

---
 main.tf                                       | 60 +++++++++++++----
 modules/network/iam.tf                        |  3 +-
 modules/network/subnetwork.tf                 |  2 +-
 modules/network/variables.tf                  |  8 +--
 bigquery.tf => modules/root/bigquery.tf       |  0
 buckets.tf => modules/root/buckets.tf         |  0
 crypto-key.tf => modules/root/crypto-key.tf   |  0
 iam.tf => modules/root/iam.tf                 |  0
 locals.tf => modules/root/locals.tf           |  0
 modules/root/main.tf                          | 20 ++++++
 outputs.tf => modules/root/outputs.tf         |  5 --
 .../root/service-accounts.tf                  |  0
 variables.tf => modules/root/variables.tf     |  6 --
 network.tf                                    | 26 --------
 output.tf                                     |  4 ++
 variable.tf                                   | 64 +++++++++++++++++++
 16 files changed, 139 insertions(+), 59 deletions(-)
 rename bigquery.tf => modules/root/bigquery.tf (100%)
 rename buckets.tf => modules/root/buckets.tf (100%)
 rename crypto-key.tf => modules/root/crypto-key.tf (100%)
 rename iam.tf => modules/root/iam.tf (100%)
 rename locals.tf => modules/root/locals.tf (100%)
 create mode 100644 modules/root/main.tf
 rename outputs.tf => modules/root/outputs.tf (87%)
 rename service-accounts.tf => modules/root/service-accounts.tf (100%)
 rename variables.tf => modules/root/variables.tf (94%)
 delete mode 100644 network.tf
 create mode 100644 output.tf
 create mode 100644 variable.tf

diff --git a/main.tf b/main.tf
index 4c50c18..85ccb10 100644
--- a/main.tf
+++ b/main.tf
@@ -1,20 +1,52 @@
-data "google_project" "data_plane_project" {
-  project_id = var.data_plane_project
+resource "google_compute_network" "vpc_network" {
+  count                   = (var.vpc_network_name == null || var.vpc_network_name == "") ? 1 : 0
+  project                 = var.project_id
+  name                    = lower("${var.installation_name}-${var.region}-vpc")
+  auto_create_subnetworks = false
+  routing_mode            = "GLOBAL"
+  description             = "The shared network for the identity graph"
 }
 
-data "google_storage_project_service_account" "data_plane_gcs_account" {
-  project = var.data_plane_project
+data "google_compute_network" "vpc_network" {
+  project = var.project_id
+  name    = (var.vpc_network_name == null || var.vpc_network_name == "") ? google_compute_network.vpc_network[0].name : var.vpc_network_name
 }
 
-resource "google_project_service" "enable_api" {
-  for_each = toset([
-    "accesscontextmanager.googleapis.com",
-    "cloudkms.googleapis.com",
-    "dataproc.googleapis.com",
-    "pubsub.googleapis.com"
-  ])
-  project = var.data_plane_project
-  service = each.value
+module "root" {
+  for_each = var.data_planes
+  source   = "./modules/root"
 
-  disable_on_destroy = false
+  environment              = var.environment
+  data_plane_project       = var.project_id
+  gcp_region               = var.region
+  installation_name        = var.installation_name
+  storage_location         = each.value.storage_region
+  key_management_location  = var.key_management_location
+  dataproc_subnet_ip4_cidr = var.subnet_ip4_cidr
+
+  organisation_id = each.value.organisation_id
+  name            = each.value.tenant_name
+  country_code    = each.value.country_code
+  data_viewers    = each.value.data_viewers
+  data_editors    = each.value.data_editors
+
+  tenant_orchestration_sa    = each.value.tenant_orchestration_sa
+  data_retention_period_days = each.value.data_retention_period_days
+  key_rotation_period_days   = each.value.key_rotation_period_days
+
+  metastore_cidr_ip_address = var.metastore_cidr_ip_address
+  idapi_cidr_ip_addresses   = var.idapi_cidr_ip_addresses
+}
+
+module "network" {
+    depends_on = [ module.root ]
+    source                    = "./modules/network"
+    installation_name         = var.installation_name
+    project_id                = var.project_id
+    region                    = var.region
+    network_name              = data.google_compute_network.vpc_network.name
+    subnet_ip4_cidr           = var.subnet_ip4_cidr
+    subnet_users              = []
+    idapi_cidr_ip_addresses   = var.idapi_cidr_ip_addresses
+    metastore_cidr_ip_address = var.metastore_cidr_ip_address
 }
diff --git a/modules/network/iam.tf b/modules/network/iam.tf
index caeeb6c..72b1121 100644
--- a/modules/network/iam.tf
+++ b/modules/network/iam.tf
@@ -1,6 +1,7 @@
 resource "google_compute_subnetwork_iam_member" "subnet_user" {
+  for_each   = toset(var.subnet_users)
   project    = var.project_id
   subnetwork = google_compute_subnetwork.dataproc_subnet.name
   role       = "roles/compute.networkUser"
-  member     = var.subnet_user
+  member     = "serviceAccount:${each.value}"
 }
diff --git a/modules/network/subnetwork.tf b/modules/network/subnetwork.tf
index 129f90d..9743103 100644
--- a/modules/network/subnetwork.tf
+++ b/modules/network/subnetwork.tf
@@ -1,7 +1,7 @@
 resource "google_compute_subnetwork" "dataproc_subnet" {
   project                  = var.project_id
   region                   = var.region
-  name                     = lower("${var.installation_name}-${var.country_code}-dataproc")
+  name                     = lower("${var.installation_name}-${var.region}-dataproc")
   ip_cidr_range            = var.subnet_ip4_cidr
   network                  = var.network_name
   private_ip_google_access = true
diff --git a/modules/network/variables.tf b/modules/network/variables.tf
index 5590eb6..d801cbf 100644
--- a/modules/network/variables.tf
+++ b/modules/network/variables.tf
@@ -2,10 +2,6 @@ variable "installation_name" {
     type = string
 }
 
-variable "country_code" {
-    type = string
-}
-
 variable "project_id" {
     type = string
 }
@@ -22,8 +18,8 @@ variable "subnet_ip4_cidr" {
     type = string
 }
 
-variable "subnet_user" {
-    type = string
+variable "subnet_users" {
+    type = list(string)
 }
 
 variable "metastore_cidr_ip_address" {
diff --git a/bigquery.tf b/modules/root/bigquery.tf
similarity index 100%
rename from bigquery.tf
rename to modules/root/bigquery.tf
diff --git a/buckets.tf b/modules/root/buckets.tf
similarity index 100%
rename from buckets.tf
rename to modules/root/buckets.tf
diff --git a/crypto-key.tf b/modules/root/crypto-key.tf
similarity index 100%
rename from crypto-key.tf
rename to modules/root/crypto-key.tf
diff --git a/iam.tf b/modules/root/iam.tf
similarity index 100%
rename from iam.tf
rename to modules/root/iam.tf
diff --git a/locals.tf b/modules/root/locals.tf
similarity index 100%
rename from locals.tf
rename to modules/root/locals.tf
diff --git a/modules/root/main.tf b/modules/root/main.tf
new file mode 100644
index 0000000..4c50c18
--- /dev/null
+++ b/modules/root/main.tf
@@ -0,0 +1,20 @@
+data "google_project" "data_plane_project" {
+  project_id = var.data_plane_project
+}
+
+data "google_storage_project_service_account" "data_plane_gcs_account" {
+  project = var.data_plane_project
+}
+
+resource "google_project_service" "enable_api" {
+  for_each = toset([
+    "accesscontextmanager.googleapis.com",
+    "cloudkms.googleapis.com",
+    "dataproc.googleapis.com",
+    "pubsub.googleapis.com"
+  ])
+  project = var.data_plane_project
+  service = each.value
+
+  disable_on_destroy = false
+}
diff --git a/outputs.tf b/modules/root/outputs.tf
similarity index 87%
rename from outputs.tf
rename to modules/root/outputs.tf
index 171a640..1a6d8f4 100644
--- a/outputs.tf
+++ b/modules/root/outputs.tf
@@ -22,8 +22,3 @@ output "output_bucket_name" {
   value       = google_storage_bucket.tenant_output_bucket.name
   description = "The name of the GCS bucket that will be used to store the output files"
 }
-
-output "vpc_network_name" {
-  value       = data.google_compute_network.vpc_network.name
-  description = "The name of the VPC network"
-}
diff --git a/service-accounts.tf b/modules/root/service-accounts.tf
similarity index 100%
rename from service-accounts.tf
rename to modules/root/service-accounts.tf
diff --git a/variables.tf b/modules/root/variables.tf
similarity index 94%
rename from variables.tf
rename to modules/root/variables.tf
index 09b27c6..c688d55 100644
--- a/variables.tf
+++ b/modules/root/variables.tf
@@ -94,12 +94,6 @@ variable "idapi_cidr_ip_addresses" {
   description = "Portrait Engine ID-API instance CIDR IP addresses"
 }
 
-variable "vpc_network_name" {
-  type        = string
-  description = "The network to connect the data-plane to, if not specified, module will provision a dedicated one"
-  default     = null
-}
-
 variable "enable_kms" {
   type        = bool
   description = "Configure KMS to encrypt build, input and output buckets"
diff --git a/network.tf b/network.tf
deleted file mode 100644
index ed9285e..0000000
--- a/network.tf
+++ /dev/null
@@ -1,26 +0,0 @@
-resource "google_compute_network" "vpc_network" {
-  count                   = (var.vpc_network_name == null || var.vpc_network_name == "") ? 1 : 0
-  project                 = var.data_plane_project
-  name                    = lower("${var.installation_name}-${var.country_code}-vpc")
-  auto_create_subnetworks = false
-  routing_mode            = "GLOBAL"
-  description             = "The shared network for the identity graph"
-}
-
-data "google_compute_network" "vpc_network" {
-  project = var.data_plane_project
-  name    = (var.vpc_network_name == null || var.vpc_network_name == "") ? google_compute_network.vpc_network[0].name : var.vpc_network_name
-}
-
-module "network" {
-  source                    = "./modules/network"
-  installation_name         = var.installation_name
-  country_code              = var.country_code
-  project_id                = var.data_plane_project
-  region                    = var.gcp_region
-  network_name              = data.google_compute_network.vpc_network.name
-  subnet_ip4_cidr           = var.dataproc_subnet_ip4_cidr
-  subnet_user               = "serviceAccount:${google_service_account.tenant_data_access.email}"
-  idapi_cidr_ip_addresses   = var.idapi_cidr_ip_addresses
-  metastore_cidr_ip_address = var.metastore_cidr_ip_address
-}
diff --git a/output.tf b/output.tf
new file mode 100644
index 0000000..64d8783
--- /dev/null
+++ b/output.tf
@@ -0,0 +1,4 @@
+output "vpc_network_name" {
+  value       = data.google_compute_network.vpc_network.name
+  description = "The name of the VPC network"
+}
diff --git a/variable.tf b/variable.tf
new file mode 100644
index 0000000..7295a91
--- /dev/null
+++ b/variable.tf
@@ -0,0 +1,64 @@
+variable "installation_name" {
+    type = string
+}
+
+variable "project_id" {
+  type        = string
+}
+
+variable "region" {
+  type        = string
+}
+
+variable "environment" {
+  type        = string
+  description = "The environment this infrastructure is supported (dev, staging or prod)"
+}
+
+variable "vpc_network_name" {
+  type        = string
+  description = "The network to connect the data-plane to, if not specified, module will provision a dedicated one"
+  default     = null
+}
+
+variable "subnet_ip4_cidr" {
+    type = string
+}
+
+variable "key_management_location" {
+    type = string
+}
+
+variable "metastore_cidr_ip_address" {
+  type        = string
+  description = "Identity Engine Metastore CloudSQL instance CIDR IP address"
+}
+
+variable "idapi_cidr_ip_addresses" {
+  type        = list(string)
+  default     = []
+  description = "Identity Engine ID-API instance CIDR IP addresses"
+}
+
+variable "data_planes" {
+  type = map(object({
+    tenant_name     = string
+    organisation_id = string
+    country_code    = string
+    storage_region  = optional(string)
+    data_editors = object({
+      service_accounts = list(string)
+      groups           = list(string)
+      users            = list(string)
+    })
+    data_viewers = object({
+      service_accounts = list(string)
+      groups           = list(string)
+      users            = list(string)
+    })
+    tenant_orchestration_sa    = string
+    data_retention_period_days = number
+    key_rotation_period_days   = number
+  }))
+  description = "A map of portrait engine data-planes with their customer specific configuration. Set storage_region to enable regional storage"
+}