Add private worker pools to resources

docs/tfengine/schemas/resources.md

@@ -118,6 +118,11 @@
 | gke_clusters.master_ipv4_cidr_block | IP range in CIDR notation to use for the hosted master network. | string | false | - | - |
 | gke_clusters.name | Name of GKE cluster. | string | true | - | - |
 | gke_clusters.network | Name of the GKE cluster's network. | string | false | - | - |
+| gke_clusters.network_peering_routes_config | GKE clustes network peering's route settings. See <https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering_routes_config>. | object | false | - | - |
+| gke_clusters.network_peering_routes_config.export_custom_routes | Whether to export the custom routes to the peer network. | boolean | false | - | - |
+| gke_clusters.network_peering_routes_config.import_custom_routes | Whether to import the custom routes to the peer network. | boolean | false | - | - |
+| gke_clusters.network_peering_routes_config.network | The name of the primary network for the peering. | string | false | - | - |
+| gke_clusters.network_peering_routes_config.project | The ID of the project in which the resource belongs. | string | false | - | - |
 | gke_clusters.network_project_id | Name of network project. If unset, the current project will be used. | string | false | - | ^[a-z][a-z0-9\-]{4,28}[a-z0-9]$ |
 | gke_clusters.node_pools | List of maps containing node pools. For supported fields, see the [module example](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/examples/node_pool_update_variant). | array(object) | false | - | - |
 | gke_clusters.resource_name | Override for Terraform resource name. If unset, defaults to normalized name. Normalization will make all characters alphanumeric with underscores. | string | false | - | - |
@@ -198,6 +203,15 @@
 | kubernetes_service_accounts.name | Name of the KSA. | string | true | - | - |
 | kubernetes_service_accounts.namespace | Namespace to where the KSA will be created. | string | true | - | - |
 | kubernetes_service_accounts.provider | The alias of the kubernetes provider. This field allows the resource to authenticate with the intended cluster. See <https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs> | string | false | - | - |
+| private_worker_pools | Cloudbuild private worker pool config.<br><br>This creates a worker pool with a dedicated computer network subnet and creates a peering connection with the subnet.  Must be created in the networks project used in the GKE cluster. | array(object) | false | - | - |
+| private_worker_pools.compute_region | Region to create worker pool in. Can be defined in global data block. | string | false | - | - |
+| private_worker_pools.gke_vpn_connection | HA VPN connection between the private worker pool vpc and the gke network vpc. See <https://registry.terraform.io/modules/terraform-google-modules/vpn/google/latest/submodules/vpn_ha>. | object | false | - | - |
+| private_worker_pools.gke_vpn_connection.gke_control_plane_range | The CIDR of the gke cluster control plane range. E.g. 192.168.0.0/28. | string | true | - | - |
+| private_worker_pools.gke_vpn_connection.gke_name | Name of the GKE cluster the worker pool can access. | string | true | - | - |
+| private_worker_pools.gke_vpn_connection.gke_network | Name of the bastion host's subnet. | string | true | - | - |
+| private_worker_pools.name | Name of private worker pool. | string | true | - | - |
+| private_worker_pools.pool_address | IP address of the worker pool IP range. | string | true | - | - |
+| private_worker_pools.pool_prefix_length | The prefix length of the worker pool IP range. | integer | true | - | - |
 | pubsub_topics | [Module](https://github.com/terraform-google-modules/terraform-google-pubsub) | array() | false | - | - |
 | pubsub_topics.labels | Labels to set on the topic. | object | false | - | - |
 | pubsub_topics.labels.*pattern* | - | string | false | - | .+ |

examples/tfengine/generated/team/cicd/triggers.tf

@@ -35,7 +35,7 @@ resource "google_cloudbuild_trigger" "validate_prod" {
 
   substitutions = {
     _TERRAFORM_ROOT = "terraform"
-    _MANAGED_DIRS   = "project_secrets project_networks project_apps project_data additional_iam_members"
+    _MANAGED_DIRS   = "project_secrets project_networks project_apps project_data additional_iam_members kubernetes"
     _WORKER_POOL    = "projects/example-prod-devops/locations/us-east1/workerPools/cicd-pool"
   }
 
@@ -65,7 +65,7 @@ resource "google_cloudbuild_trigger" "plan_prod" {
 
   substitutions = {
     _TERRAFORM_ROOT = "terraform"
-    _MANAGED_DIRS   = "project_secrets project_networks project_apps project_data additional_iam_members"
+    _MANAGED_DIRS   = "project_secrets project_networks project_apps project_data additional_iam_members kubernetes"
     _WORKER_POOL    = "projects/example-prod-devops/locations/us-east1/workerPools/cicd-pool"
   }
 
@@ -96,7 +96,7 @@ resource "google_cloudbuild_trigger" "apply_prod" {
 
   substitutions = {
     _TERRAFORM_ROOT = "terraform"
-    _MANAGED_DIRS   = "project_secrets project_networks project_apps project_data additional_iam_members"
+    _MANAGED_DIRS   = "project_secrets project_networks project_apps project_data additional_iam_members kubernetes"
     _WORKER_POOL    = "projects/example-prod-devops/locations/us-east1/workerPools/cicd-pool"
   }
 

examples/tfengine/generated/team/project_apps/main.tf

@@ -217,11 +217,17 @@ module "gke_cluster" {
   regional           = true
   network_project_id = "example-prod-networks"
 
-  network                 = "network"
-  subnetwork              = "gke-subnet"
-  ip_range_pods           = "pods-range"
-  ip_range_services       = "services-range"
-  master_ipv4_cidr_block  = "192.168.0.0/28"
+  network                = "network"
+  subnetwork             = "gke-subnet"
+  ip_range_pods          = "pods-range"
+  ip_range_services      = "services-range"
+  master_ipv4_cidr_block = "172.16.0.0/28"
+  master_authorized_networks = [
+    {
+      cidr_block   = "192.168.0.0/16"
+      display_name = "cloudbuild"
+    },
+  ]
   skip_provisioners       = true
   enable_private_endpoint = false
   release_channel         = "STABLE"
@@ -240,6 +246,13 @@ module "gke_cluster" {
     module.project
   ]
 }
+resource "google_compute_network_peering_routes_config" "peering_gke_cluster" {
+  network              = "network"
+  project              = "example-prod-networks"
+  import_custom_routes = false
+  export_custom_routes = true
+  peering              = module.gke_cluster.peering_name
+}
 
 module "project_iam_members" {
   source  = "terraform-google-modules/iam/google//modules/projects_iam"

examples/tfengine/generated/team/project_networks/main.tf

@@ -52,6 +52,7 @@ module "project" {
     "iap.googleapis.com",
     "servicenetworking.googleapis.com",
     "sqladmin.googleapis.com",
+    "cloudbuild.googleapis.com",
   ]
 }
 
@@ -172,3 +173,162 @@ resource "google_service_account" "bastion_accessor" {
 
   project = module.project.project_id
 }
+resource "google_project_service" "servicenetworking" {
+  service            = "servicenetworking.googleapis.com"
+  project            = module.project.project_id
+  disable_on_destroy = false
+}
+
+module "worker_pool_network" {
+  source  = "terraform-google-modules/network/google"
+  version = "~> 3.3.0"
+
+  network_name = "worker-pool-network"
+  project_id   = module.project.project_id
+
+  subnets = []
+}
+
+resource "google_compute_global_address" "worker_pool_address" {
+  provider      = google-beta
+  name          = "worker-pool-address"
+  purpose       = "VPC_PEERING"
+  network       = module.worker_pool_network.network_self_link
+  address_type  = "INTERNAL"
+  address       = "192.168.0.0"
+  prefix_length = 16
+  project       = module.project.project_id
+}
+
+resource "google_service_networking_connection" "worker_pool_connection" {
+  network                 = module.worker_pool_network.network_self_link
+  service                 = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [google_compute_global_address.worker_pool_address.name]
+  depends_on              = [google_project_service.servicenetworking]
+}
+
+resource "google_compute_network_peering_routes_config" "worker_pool_peering" {
+  network              = module.worker_pool_network.network_name
+  peering              = "servicenetworking-googleapis-com"
+  import_custom_routes = false
+  export_custom_routes = true
+  project              = module.project.project_id
+  depends_on = [
+    google_service_networking_connection.worker_pool_connection,
+    module.worker_pool_network,
+  ]
+}
+
+module "private_pool_gcloud" {
+  source                 = "terraform-google-modules/gcloud/google"
+  version                = "~> 3.0.1"
+  additional_components  = []
+  create_cmd_entrypoint  = "gcloud"
+  create_cmd_body        = "builds worker-pools create private-pool --region=us-central1 --peered-network=projects/$${module.project.project_id}/global/networks/$${module.worker_pool_network.network_name} --project=$${module.project.project_id} --quiet"
+  destroy_cmd_entrypoint = "gcloud"
+  destroy_cmd_body       = "builds worker-pools delete private-pool --region=us-central1  --project=$${module.project.project_id} --quiet"
+  module_depends_on = [
+    google_compute_network_peering_routes_config.worker_pool_peering,
+  ]
+}
+module "worker_pool_vpn_ha_1" {
+  source           = "terraform-google-modules/vpn/google//modules/vpn_ha"
+  version          = "~> 1.5.0"
+  project_id       = module.project.project_id
+  region           = "us-central1"
+  network          = module.worker_pool_network.network_self_link
+  name             = "worker-pool-net-to-gke-cluster-net"
+  peer_gcp_gateway = module.worker_pool_vpn_ha_2.self_link
+  router_asn       = 64514
+  tunnels = {
+    remote-0 = {
+      bgp_peer = {
+        address = "169.254.1.1"
+        asn     = 64513
+      }
+      bgp_peer_options = {
+        advertise_mode = "CUSTOM"
+        advertise_ip_ranges = {
+          "192.168.0.0/16" : ""
+        }
+        route_priority   = 1000
+        advertise_groups = null
+      }
+      bgp_session_range               = "169.254.1.2/30"
+      ike_version                     = 2
+      vpn_gateway_interface           = 0
+      peer_external_gateway_interface = null
+      shared_secret                   = ""
+    }
+    remote-1 = {
+      bgp_peer = {
+        address = "169.254.2.1"
+        asn     = 64513
+      }
+      bgp_peer_options = {
+        advertise_mode = "CUSTOM"
+        advertise_ip_ranges = {
+          "192.168.0.0/16" : ""
+        }
+        route_priority   = 1000
+        advertise_groups = null
+      }
+      bgp_session_range               = "169.254.2.2/30"
+      ike_version                     = 2
+      vpn_gateway_interface           = 1
+      peer_external_gateway_interface = null
+      shared_secret                   = ""
+    }
+  }
+}
+
+module "worker_pool_vpn_ha_2" {
+  source           = "terraform-google-modules/vpn/google//modules/vpn_ha"
+  version          = "~> 1.5.0"
+  project_id       = module.project.project_id
+  region           = "us-central1"
+  network          = module.network.network.network.self_link
+  name             = "gke-cluster-net-to-worker-pool-net"
+  router_asn       = 64513
+  peer_gcp_gateway = module.worker_pool_vpn_ha_1.self_link
+  tunnels = {
+    remote-0 = {
+      bgp_peer = {
+        address = "169.254.1.2"
+        asn     = 64514
+      }
+      bgp_peer_options = {
+        advertise_mode = "CUSTOM"
+        advertise_ip_ranges = {
+          "172.16.0.0/28" : ""
+        }
+        route_priority   = 1000
+        advertise_groups = null
+      }
+      bgp_session_range               = "169.254.1.1/30"
+      ike_version                     = 2
+      vpn_gateway_interface           = 0
+      peer_external_gateway_interface = null
+      shared_secret                   = module.worker_pool_vpn_ha_1.random_secret
+    }
+    remote-1 = {
+      bgp_peer = {
+        address = "169.254.2.2"
+        asn     = 64514
+      }
+      bgp_peer_options = {
+        advertise_mode = "CUSTOM"
+        advertise_ip_ranges = {
+          "172.16.0.0/28" : ""
+        }
+        route_priority   = 1000
+        advertise_groups = null
+      }
+      bgp_session_range               = "169.254.2.1/30"
+      ike_version                     = 2
+      vpn_gateway_interface           = 1
+      peer_external_gateway_interface = null
+      shared_secret                   = module.worker_pool_vpn_ha_1.random_secret
+    }
+  }
+}

examples/tfengine/modules/foundation.hcl

@@ -141,6 +141,7 @@ template "cicd" {
           "project_apps",
           "project_data",
           "additional_iam_members",
+          "kubernetes"
         ]
         worker_pool = {
           project  = "{{.prefix}}-{{.env}}-devops"

examples/tfengine/modules/team.hcl

@@ -86,6 +86,7 @@ template "project_networks" {
         "iap.googleapis.com",
         "servicenetworking.googleapis.com",
         "sqladmin.googleapis.com",
+        "cloudbuild.googleapis.com",
       ]
     }
     resources = {
@@ -102,6 +103,7 @@ template "project_networks" {
             secondary_ranges = [
               {
                 name     = "pods-range"
+                // TODO double check the ip_ranges.
                 ip_range = "172.16.0.0/14"
               },
               {
@@ -150,6 +152,16 @@ EOF
 
         }]
       }]
+      private_worker_pools = [{
+        name                      = "worker-pool"
+        pool_address              = "192.168.0.0"
+        pool_prefix_length        = 16
+        gke_vpn_connection = {
+          gke_name                = "gke-cluster"
+          gke_control_plane_range = "172.16.0.0/28"
+          gke_network             = "$${module.network.network.network.self_link}"
+        }
+      }]
     }
   }
 }
@@ -182,7 +194,12 @@ template "project_apps" {
         subnet                 = "gke-subnet"
         ip_range_pods_name     = "pods-range"
         ip_range_services_name = "services-range"
-        master_ipv4_cidr_block = "192.168.0.0/28"
+        master_ipv4_cidr_block = "172.16.0.0/28"
+
+        master_authorized_networks = [{
+          display_name: "cloudbuild"
+          cidr_block: "192.168.0.0/16"
+        }]
 
         # Set custom node pool to control machine type.
         node_pools = [{
@@ -192,6 +209,12 @@ template "project_apps" {
         labels = {
           type = "no-phi"
         }
+        network_peering_routes_config = {
+          network      = "network"
+          import_custom_routes = false
+          export_custom_routes = true
+          project = "{{.prefix}}-{{.env}}-networks"
+        }
       }]
       binary_authorization = {
         admission_whitelist_patterns = [{

internal/template/funcmap.go

@@ -27,6 +27,7 @@ var funcMap = map[string]interface{}{
 	"get":               get,
 	"has":               has,
 	"hcl":               hcl,
+	"sub":               sub,
 	"hclField":          hclField,
 	"merge":             merge,
 	"replace":           replace,
@@ -83,6 +84,12 @@ func hcl(v interface{}) (string, error) {
 	return strings.TrimSpace(string(b)), nil
 }
 
+// sub computes the subtraction between two integers.
+// (e.g. sub(5, 3) returns 2.)
+func sub(a int, b int) int {
+	return a - b
+}
+
 // hclField returns a hcl marshaled field e.g. `name = "foo"`, if present.
 // For required fields, use the hcl func.
 func hclField(m map[string]interface{}, key string) (string, error) {

templates/tfengine/components/resources/gke_clusters/main.tf

@@ -75,4 +75,14 @@ module "{{resourceName . "name"}}" {
     module.project
   ]
 }
+
+{{- if has . "network_peering_routes_config"}}
+resource "google_compute_network_peering_routes_config" "peering_{{resourceName . "name"}}" {
+  network = {{hcl .network_peering_routes_config.network}}
+  project = {{hcl .network_peering_routes_config.project}}
+  import_custom_routes = {{hcl .network_peering_routes_config.import_custom_routes}}
+  export_custom_routes = {{hcl .network_peering_routes_config.export_custom_routes}}
+  peering = module.{{resourceName . "name"}}.peering_name
+}
+{{- end}}
 {{end -}}