Leverage common service module (closes #6)

Author: jbeemster

README.md

@@ -29,14 +29,14 @@ To start loading "enriched" data into Postgres:
 ```hcl
 module "enriched_topic" {
   source  = "snowplow-devops/pubsub-topic/google"
-  version = "0.1.0"
+  version = "0.3.0"
 
   name = "enriched-topic"
 }
 
 module "pipeline_db" {
   source  = "snowplow-devops/cloud-sql/google"
-  version = "0.1.0"
+  version = "0.3.0"
 
   name = "pipeline-db"
 
@@ -95,7 +95,7 @@ To load the "bad" data instead:
 ```hcl
 module "bad_1_topic" {
   source  = "snowplow-devops/pubsub-topic/google"
-  version = "0.1.0"
+  version = "0.3.0"
 
   name = "bad-1-topic"
 }
@@ -146,7 +146,7 @@ module "postgres_loader_bad" {
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.15 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
 | <a name="requirement_google"></a> [google](#requirement\_google) | >= 3.44.0 |
 
 ## Providers
@@ -159,24 +159,22 @@ module "postgres_loader_bad" {
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_telemetry"></a> [telemetry](#module\_telemetry) | snowplow-devops/telemetry/snowplow | 0.2.0 |
+| <a name="module_service"></a> [service](#module\_service) | snowplow-devops/service-ce/google | 0.1.0 |
+| <a name="module_telemetry"></a> [telemetry](#module\_telemetry) | snowplow-devops/telemetry/snowplow | 0.5.0 |
 
 ## Resources
 
 | Name | Type |
 |------|------|
 | [google_compute_firewall.egress](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource |
 | [google_compute_firewall.ingress_ssh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource |
-| [google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template) | resource |
-| [google_compute_region_instance_group_manager.grp](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_instance_group_manager) | resource |
 | [google_project_iam_member.sa_cloud_sql_client](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.sa_logging_log_writer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.sa_pubsub_publisher](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.sa_pubsub_subscriber](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.sa_pubsub_viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_pubsub_subscription.in](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource |
 | [google_service_account.sa](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
-| [google_compute_image.ubuntu_20_04](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source |
 
 ## Inputs
 
@@ -189,10 +187,11 @@ module "postgres_loader_bad" {
 | <a name="input_in_topic_name"></a> [in\_topic\_name](#input\_in\_topic\_name) | The name of the input pubsub topic that the loader will pull data from | `string` | n/a | yes |
 | <a name="input_name"></a> [name](#input\_name) | A name which will be pre-pended to the resources created | `string` | n/a | yes |
 | <a name="input_network"></a> [network](#input\_network) | The name of the network to deploy within | `string` | n/a | yes |
-| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The id of the project in which this resource is created | `string` | n/a | yes |
+| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The project ID in which the stack is being deployed | `string` | n/a | yes |
 | <a name="input_purpose"></a> [purpose](#input\_purpose) | The type of data the loader will be pulling which can be one of ENRICHED\_EVENTS or JSON (Note: JSON can be used for loading bad rows) | `string` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | The name of the region to deploy within | `string` | n/a | yes |
 | <a name="input_schema_name"></a> [schema\_name](#input\_schema\_name) | The database schema to load data into (e.g atomic \| atomic\_bad) | `string` | n/a | yes |
+| <a name="input_app_version"></a> [app\_version](#input\_app\_version) | App version to use. This variable facilitates dev flow, the modules may not work with anything other than the default value. | `string` | `"0.3.1"` | no |
 | <a name="input_associate_public_ip_address"></a> [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | Whether to assign a public ip address to this instance; if false this instance must be behind a Cloud NAT to connect to the internet | `bool` | `true` | no |
 | <a name="input_custom_iglu_resolvers"></a> [custom\_iglu\_resolvers](#input\_custom\_iglu\_resolvers) | The custom Iglu Resolvers that will be used by the loader to resolve and validate events | <pre>list(object({<br>    name            = string<br>    priority        = number<br>    uri             = string<br>    api_key         = string<br>    vendor_prefixes = list(string)<br>  }))</pre> | `[]` | no |
 | <a name="input_db_host"></a> [db\_host](#input\_db\_host) | The hostname of the database to connect to (Note: if db\_instance\_name is non-empty this setting is ignored) | `string` | `""` | no |
@@ -201,6 +200,7 @@ module "postgres_loader_bad" {
 | <a name="input_default_iglu_resolvers"></a> [default\_iglu\_resolvers](#input\_default\_iglu\_resolvers) | The default Iglu Resolvers that will be used by the loader to resolve and validate events | <pre>list(object({<br>    name            = string<br>    priority        = number<br>    uri             = string<br>    api_key         = string<br>    vendor_prefixes = list(string)<br>  }))</pre> | <pre>[<br>  {<br>    "api_key": "",<br>    "name": "Iglu Central",<br>    "priority": 10,<br>    "uri": "http://iglucentral.com",<br>    "vendor_prefixes": []<br>  },<br>  {<br>    "api_key": "",<br>    "name": "Iglu Central - Mirror 01",<br>    "priority": 20,<br>    "uri": "http://mirror01.iglucentral.com",<br>    "vendor_prefixes": []<br>  }<br>]</pre> | no |
 | <a name="input_gcp_logs_enabled"></a> [gcp\_logs\_enabled](#input\_gcp\_logs\_enabled) | Whether application logs should be reported to GCP Logging | `bool` | `true` | no |
 | <a name="input_in_max_concurrent_checkpoints"></a> [in\_max\_concurrent\_checkpoints](#input\_in\_max\_concurrent\_checkpoints) | The maximum number of concurrent effects for the topic checkpointing system - essentially how many concurrent acks we will make to PubSub | `number` | `100` | no |
+| <a name="input_java_opts"></a> [java\_opts](#input\_java\_opts) | Custom JAVA Options | `string` | `"-XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75"` | no |
 | <a name="input_labels"></a> [labels](#input\_labels) | The labels to append to this resource | `map(string)` | `{}` | no |
 | <a name="input_machine_type"></a> [machine\_type](#input\_machine\_type) | The machine type to use | `string` | `"e2-small"` | no |
 | <a name="input_ssh_block_project_keys"></a> [ssh\_block\_project\_keys](#input\_ssh\_block\_project\_keys) | Whether to block project wide SSH keys | `bool` | `true` | no |

main.tf

@@ -3,7 +3,7 @@ locals {
   module_version = "0.2.1"
 
   app_name    = "snowplow-postgres-loader"
-  app_version = "0.3.1"
+  app_version = var.app_version
 
   local_labels = {
     name           = var.name
@@ -21,7 +21,7 @@ locals {
 
 module "telemetry" {
   source  = "snowplow-devops/telemetry/snowplow"
-  version = "0.2.0"
+  version = "0.5.0"
 
   count = var.telemetry_enabled ? 1 : 0
 
@@ -34,11 +34,6 @@ module "telemetry" {
   module_version   = local.module_version
 }
 
-data "google_compute_image" "ubuntu_20_04" {
-  family  = "ubuntu-2004-lts"
-  project = "ubuntu-os-cloud"
-}
-
 # --- IAM: Service Account setup
 
 resource "google_service_account" "sa" {
@@ -47,28 +42,33 @@ resource "google_service_account" "sa" {
 }
 
 resource "google_project_iam_member" "sa_pubsub_viewer" {
-  role   = "roles/pubsub.viewer"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/pubsub.viewer"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 resource "google_project_iam_member" "sa_pubsub_subscriber" {
-  role   = "roles/pubsub.subscriber"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/pubsub.subscriber"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 resource "google_project_iam_member" "sa_pubsub_publisher" {
-  role   = "roles/pubsub.publisher"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/pubsub.publisher"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 resource "google_project_iam_member" "sa_logging_log_writer" {
-  role   = "roles/logging.logWriter"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/logging.logWriter"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 resource "google_project_iam_member" "sa_cloud_sql_client" {
-  role   = "roles/cloudsql.client"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/cloudsql.client"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 # --- CE: Firewall rules
@@ -181,112 +181,45 @@ locals {
   })
 
   startup_script = templatefile("${path.module}/templates/startup-script.sh.tmpl", {
-    config        = local.config
-    iglu_resolver = local.iglu_resolver
-    version       = local.app_version
-    db_host       = local.db_host
-    db_port       = var.db_port
-    db_name       = var.db_name
-    db_username   = var.db_username
-    db_password   = var.db_password
-    schema_name   = var.schema_name
+    config_b64        = base64encode(local.config)
+    iglu_resolver_b64 = base64encode(local.iglu_resolver)
+    version           = local.app_version
+    db_host           = local.db_host
+    db_port           = var.db_port
+    db_name           = var.db_name
+    db_username       = var.db_username
+    db_password       = var.db_password
+    schema_name       = var.schema_name
 
     db_instance_name        = var.db_instance_name
     cloud_sql_proxy_enabled = var.db_instance_name != ""
 
     telemetry_script = join("", module.telemetry.*.gcp_ubuntu_20_04_user_data)
 
     gcp_logs_enabled = var.gcp_logs_enabled
-  })
-
-  ssh_keys_metadata = <<EOF
-%{for v in var.ssh_key_pairs~}
-    ${v.user_name}:${v.public_key}
-%{endfor~}
-EOF
-}
-
-resource "google_compute_instance_template" "tpl" {
-  name_prefix = "${var.name}-"
-  description = "This template is used to create Postgres Loader instances"
-
-  instance_description = var.name
-  machine_type         = var.machine_type
-
-  scheduling {
-    automatic_restart   = true
-    on_host_maintenance = "MIGRATE"
-  }
-
-  disk {
-    source_image = var.ubuntu_20_04_source_image == "" ? data.google_compute_image.ubuntu_20_04.self_link : var.ubuntu_20_04_source_image
-    auto_delete  = true
-    boot         = true
-    disk_type    = "pd-standard"
-    disk_size_gb = 10
-  }
-
-  # Note: Only one of either network or subnetwork can be supplied
-  #       https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#network_interface
-  network_interface {
-    network    = var.subnetwork == "" ? var.network : ""
-    subnetwork = var.subnetwork
-
-    dynamic "access_config" {
-      for_each = var.associate_public_ip_address ? [1] : []
-
-      content {
-        network_tier = "PREMIUM"
-      }
-    }
-  }
-
-  service_account {
-    email  = google_service_account.sa.email
-    scopes = ["cloud-platform"]
-  }
 
-  metadata_startup_script = local.startup_script
-
-  metadata = {
-    block-project-ssh-keys = var.ssh_block_project_keys
-
-    ssh-keys = local.ssh_keys_metadata
-  }
-
-  tags = [var.name]
-
-  labels = local.labels
-
-  lifecycle {
-    create_before_destroy = true
-  }
+    java_opts = var.java_opts
+  })
 }
 
-resource "google_compute_region_instance_group_manager" "grp" {
-  name = "${var.name}-grp"
-
-  base_instance_name = var.name
-  region             = var.region
-
-  target_size = var.target_size
-
-  version {
-    name              = "${local.app_name}-${local.app_version}"
-    instance_template = google_compute_instance_template.tpl.self_link
-  }
-
-  update_policy {
-    type                  = "PROACTIVE"
-    minimal_action        = "REPLACE"
-    max_unavailable_fixed = 3
-  }
-
-  wait_for_instances = true
-
-  timeouts {
-    create = "20m"
-    update = "20m"
-    delete = "30m"
-  }
+module "service" {
+  source  = "snowplow-devops/service-ce/google"
+  version = "0.1.0"
+
+  user_supplied_script        = local.startup_script
+  name                        = var.name
+  instance_group_version_name = "${local.app_name}-${local.app_version}"
+  labels                      = local.labels
+
+  region     = var.region
+  network    = var.network
+  subnetwork = var.subnetwork
+
+  ubuntu_20_04_source_image   = var.ubuntu_20_04_source_image
+  machine_type                = var.machine_type
+  target_size                 = var.target_size
+  ssh_block_project_keys      = var.ssh_block_project_keys
+  ssh_key_pairs               = var.ssh_key_pairs
+  service_account_email       = google_service_account.sa.email
+  associate_public_ip_address = var.associate_public_ip_address
 }

outputs.tf

@@ -1,14 +1,14 @@
 output "manager_id" {
-  value       = google_compute_region_instance_group_manager.grp.id
+  value       = module.service.manager_id
   description = "Identifier for the instance group manager"
 }
 
 output "manager_self_link" {
-  value       = google_compute_region_instance_group_manager.grp.self_link
+  value       = module.service.manager_self_link
   description = "The URL for the instance group manager"
 }
 
 output "instance_group_url" {
-  value       = google_compute_region_instance_group_manager.grp.instance_group
+  value       = module.service.instance_group_url
   description = "The full URL of the instance group created by the manager"
 }

templates/startup-script.sh.tmpl

@@ -1,33 +1,13 @@
-#!/bin/bash
-set -e -x
-
-# -----------------------------------------------------------------------------
-#  BASE INSTALL
-# -----------------------------------------------------------------------------
-
 readonly CONFIG_DIR=/opt/snowplow/config
 
-function install_base_packages() {
-  sudo apt install wget curl unzip -y
-}
-
-function install_docker_ce() {
-  sudo apt install docker.io -y
-  sudo systemctl enable --now docker
-}
-
-sudo apt update -y
-
-install_base_packages
-install_docker_ce
-
 sudo mkdir -p $${CONFIG_DIR}
-sudo cat << EOF > $${CONFIG_DIR}/postgres_loader.json
-${config}
+
+sudo base64 --decode << EOF > $${CONFIG_DIR}/postgres_loader.json
+${config_b64}
 EOF
 
-sudo cat << EOF > $${CONFIG_DIR}/iglu_resolver.json
-${iglu_resolver}
+sudo base64 --decode << EOF > $${CONFIG_DIR}/iglu_resolver.json
+${iglu_resolver_b64}
 EOF
 
 # Create the schema to load data into
@@ -57,7 +37,7 @@ sudo docker run \
 %{ if gcp_logs_enabled ~}
   --log-driver gcplogs \
 %{ endif ~}
-  -v $${CONFIG_DIR}:/snowplow/config \
+  --mount type=bind,source=$${CONFIG_DIR},target=/snowplow/config \
   -e 'PGUSER=${db_username}' \
   -e 'PGPASSWORD=${db_password}' \
   postgres:13 \
@@ -69,14 +49,15 @@ sudo docker run \
   --name postgres_loader \
   --restart always \
   --network host \
+  --memory=$(get_application_memory_mb)m \
 %{ if gcp_logs_enabled ~}
   --log-driver gcplogs \
 %{ else ~}
   --log-opt max-size=10m \
   --log-opt max-file=5 \
 %{ endif ~}
-  -v $${CONFIG_DIR}:/snowplow/config \
-  -e 'JAVA_OPTS=-Dorg.slf4j.simpleLogger.defaultLogLevel=info' \
+  --mount type=bind,source=$${CONFIG_DIR},target=/snowplow/config \
+  --env JDK_JAVA_OPTIONS='${java_opts}' \
   snowplow/snowplow-postgres-loader:${version} \
   --config /snowplow/config/postgres_loader.json \
   --resolver /snowplow/config/iglu_resolver.json