add: test and lint

.devcontainer/Dockerfile

@@ -6,21 +6,29 @@ RUN apt-get update && \
     apt-transport-https \
     ca-certificates \
     curl \
-    gnupg
+    gnupg \
+    lsb-release
 
 SHELL [ "/bin/bash", "-o", "pipefail", "-c" ]
 
 # Install gcloud
 # ref: https://cloud.google.com/sdk/docs/install#deb
 RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \
-    | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
-RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
+    | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \
+    echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
     | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
 
+# Install Trivy
+# ref: https://trivy.dev/dev/getting-started/installation/
+RUN curl https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add - && \
+    echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" \
+    | tee -a /etc/apt/sources.list.d/trivy.list
+
 # hadolint ignore=DL3008
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    google-cloud-cli
+    google-cloud-cli \
+    trivy
 
 
 FROM mcr.microsoft.com/vscode/devcontainers/base:bookworm
@@ -29,4 +37,5 @@ LABEL maintainer="a5chin <[email protected]>"
 
 COPY --from=builder --chown=vscode: /usr/bin/python* /usr/bin/python*
 COPY --from=builder --chown=vscode: /usr/bin/gcloud /usr/bin/gcloud
+COPY --from=builder --chown=vscode: /usr/bin/trivy /usr/bin/trivy
 COPY --from=builder --chown=vscode: /usr/lib /usr/lib

.devcontainer/devcontainer.json

@@ -15,6 +15,7 @@
     "customizations": {
         "vscode": {
             "extensions": [
+                "aquasecurityofficial.trivy-vulnerability-scanner",
                 "codezombiech.gitignore",
                 "eamodio.gitlens",
                 "exiasr.hadolint",

.github/workflows/terraform.yml

@@ -0,0 +1,104 @@
+name: Terraform
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+
+    outputs:
+      matrix: ${{ steps.extract_modules.outputs.matrix }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Extract terraform modules
+        id: extract_modules
+        run: |
+          echo "matrix=$(ls modules | jq -R -s -c '{ "modules": split("\n")[:-1] }')" > $GITHUB_OUTPUT
+
+  format:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.9.8
+
+      - name: Terraform fmt
+        run: terraform fmt -check -recursive
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/cache@v4
+        name: Cache plugin dir
+        with:
+          path: ~/.tflint.d/plugins
+          key: ${{ matrix.os }}-tflint-${{ hashFiles('.tflint.hcl') }}
+
+      - uses: terraform-linters/setup-tflint@v4
+        name: Setup TFLint
+        with:
+          tflint_version: latest
+
+      - name: Init TFLint
+        run: tflint --init
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: tflint
+        run: tflint
+
+  tests:
+    runs-on: ubuntu-latest
+
+    needs: [setup]
+
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
+
+    defaults:
+      run:
+        working-directory: modules/${{ matrix.modules }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.9.8
+
+      - name: Run terraform test
+        run: |
+          terraform init
+          terraform test
+
+  validate:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.9.8
+
+      - name: Terraform validate
+        run: terraform validate

.gitignore

@@ -10,8 +10,8 @@ crash.log
 crash.*.log
 
 # Exclude all .tfvars files, which are likely to contain sensitive data, such as
-# password, private keys, and other secrets. These should not be part of version 
-# control as they are data points which are potentially sensitive and subject 
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
 # to change depending on the environment.
 *.tfvars
 *.tfvars.json
@@ -35,3 +35,6 @@ override.tf.json
 # Ignore CLI configuration files
 .terraformrc
 terraform.rc
+
+# Ignore .terraform.lock.hcl files under modules directory
+modules/*/.terraform.lock.hcl

.pre-commit-config.yaml

@@ -2,65 +2,17 @@ default_stages: [commit]
 
 repos:
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.96.1
+    rev: "v1.96.1"
     hooks:
+      - id: terraform_docs
+        name: terraform-docs
+        description: "Run terraform-docs"
+
       - id: terraform_fmt
         name: terraform-fmt
         description: "Run 'terraform fmt' for format"
         args: [recursive]
 
-  - repo: https://github.com/terraform-docs/terraform-docs
-    rev: "v0.18.0"
-    hooks:
-      - id: terraform-docs-go
-        name: terraform-docs-gcs2spanner
-        args:
-          [
-            "markdown",
-            "table",
-            "--output-file",
-            "./README.md",
-            "./modules/gcs2spanner/",
-          ]
-
-  - repo: https://github.com/terraform-docs/terraform-docs
-    rev: "v0.18.0"
-    hooks:
-      - id: terraform-docs-go
-        name: terraform-docs-log2bq
-        args:
-          [
-            "markdown",
-            "table",
-            "--output-file",
-            "./README.md",
-            "./modules/log2bq/",
-          ]
-
-  - repo: https://github.com/terraform-docs/terraform-docs
-    rev: "v0.18.0"
-    hooks:
-      - id: terraform-docs-go
-        name: terraform-docs-monitoring-tools
-        args:
-          [
-            "markdown",
-            "table",
-            "--output-file",
-            "./README.md",
-            "./modules/monitoring-tools/",
-          ]
-
-  - repo: https://github.com/terraform-docs/terraform-docs
-    rev: "v0.18.0"
-    hooks:
-      - id: terraform-docs-go
-        name: terraform-docs-microservices
-        args:
-          [
-            "markdown",
-            "table",
-            "--output-file",
-            "./README.md",
-            "./modules/microservices/",
-          ]
+      - id: terraform_tflint
+        name: terraform-lint
+        description: "Run tflint"

.tflint.hcl

@@ -0,0 +1,13 @@
+tflint {
+  required_version = ">= 0.50"
+}
+
+config {
+  call_module_type = "all"
+}
+
+plugin "google" {
+  enabled = true
+  version = "0.30.0"
+  source  = "github.com/terraform-linters/tflint-ruleset-google"
+}

.vscode/extensions.json

@@ -1,5 +1,6 @@
 {
     "recommendations": [
+        "aquasecurityofficial.trivy-vulnerability-scanner",
         "codezombiech.gitignore",
         "eamodio.gitlens",
         "exiasr.hadolint",

modules/gcs2spanner/README.md

@@ -12,9 +12,9 @@
 
 | Name | Version |
 |------|---------|
-| <a name="provider_archive"></a> [archive](#provider\_archive) | >=2.4.0 |
-| <a name="provider_google"></a> [google](#provider\_google) | >= 5.22.0 |
-| <a name="provider_google-beta"></a> [google-beta](#provider\_google-beta) | >= 5.22.0 |
+| <a name="provider_archive"></a> [archive](#provider\_archive) | 2.6.0 |
+| <a name="provider_google"></a> [google](#provider\_google) | 6.12.0 |
+| <a name="provider_google-beta"></a> [google-beta](#provider\_google-beta) | 6.12.0 |
 
 ## Modules
 
@@ -24,7 +24,7 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [google-beta_google_project_service_identity.storage](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_service_identity) | resource |
+| [google-beta_google_project_service_identity.storage](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_project_service_identity) | resource |
 | [google_cloud_run_v2_service_iam_member.event](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_v2_service_iam_member) | resource |
 | [google_cloudfunctions2_function.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudfunctions2_function) | resource |
 | [google_compute_network.dataflow](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network) | resource |
@@ -41,19 +41,19 @@ No modules.
 | [google_storage_bucket.functions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource |
 | [google_storage_bucket_iam_member.data](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource |
 | [google_storage_bucket_object.functions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource |
-| [archive_file.functions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/file) | data source |
+| [archive_file.functions](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source |
 | [google_project.main](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_dataflow"></a> [dataflow](#input\_dataflow) | The Dataflow parameters | <pre>object({<br>    dataflow = object({<br>      name              = string<br>      gcsPath           = optional(string, "gs://dataflow-templates/2024-01-30-01_RC00/GCS_Avro_to_Cloud_Spanner")<br>      temp_gcs_location = string<br>      parameters = object({<br>        instanceId = string<br>        databaseId = string<br>        subnetwork = string<br>      })<br>      sa = object({<br>        id = string<br>      })<br>    })<br>  })</pre> | n/a | yes |
-| <a name="input_functions"></a> [functions](#input\_functions) | The Cloud Functions parameter for creating Dataflow jobs,<br>    using gcsPath as the Google-provided template available in the public GCS bucket | <pre>object({<br>    name                             = string<br>    bucket                           = string<br>    max_instance_count               = optional(number, 1)<br>    min_instance_count               = optional(number, 0)<br>    available_memory                 = optional(string, "512Mi")<br>    timeout_seconds                  = optional(number, 60)<br>    max_instance_request_concurrency = optional(number, 80)<br>    available_cpu                    = optional(number, 1)<br>    sa = object({<br>      id = string<br>    })<br>    event = object({<br>      sa = object({<br>        id = string<br>      })<br>    })<br>  })</pre> | n/a | yes |
-| <a name="input_gcs"></a> [gcs](#input\_gcs) | The GCS parameters to receive aggregate data | <pre>object({<br>    name = string<br>    lifecycle_rule = optional(<br>      object({<br>        age    = number<br>        action = string<br>        }), {<br>        age    = 90<br>        action = "Delete"<br>      }<br>    )<br>    allows = set(string)<br>  })</pre> | n/a | yes |
+| <a name="input_dataflow"></a> [dataflow](#input\_dataflow) | The Dataflow parameters | <pre>object({<br/>    name              = string<br/>    gcsPath           = optional(string, "gs://dataflow-templates/2024-01-30-01_RC00/GCS_Avro_to_Cloud_Spanner")<br/>    temp_gcs_location = string<br/>    parameters = object({<br/>      instanceId = string<br/>      databaseId = string<br/>      subnetwork = string<br/>    })<br/>    sa = object({<br/>      id = string<br/>    })<br/>  })</pre> | n/a | yes |
+| <a name="input_functions"></a> [functions](#input\_functions) | The Cloud Functions parameter for creating Dataflow jobs,<br/>    using gcsPath as the Google-provided template available in the public GCS bucket | <pre>object({<br/>    name                             = string<br/>    bucket                           = string<br/>    max_instance_count               = optional(number, 1)<br/>    min_instance_count               = optional(number, 0)<br/>    available_memory                 = optional(string, "512Mi")<br/>    timeout_seconds                  = optional(number, 60)<br/>    max_instance_request_concurrency = optional(number, 80)<br/>    available_cpu                    = optional(number, 1)<br/>    sa = object({<br/>      id = string<br/>    })<br/>    event = object({<br/>      sa = object({<br/>        id = string<br/>      })<br/>    })<br/>  })</pre> | n/a | yes |
+| <a name="input_gcs"></a> [gcs](#input\_gcs) | The GCS parameters to receive aggregate data | <pre>object({<br/>    name = string<br/>    lifecycle_rule = optional(<br/>      object({<br/>        age    = number<br/>        action = string<br/>        }), {<br/>        age    = 90<br/>        action = "Delete"<br/>      }<br/>    )<br/>    allows = set(string)<br/>  })</pre> | n/a | yes |
 | <a name="input_location"></a> [location](#input\_location) | The location of the Dataflow | `string` | n/a | yes |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The ID of the Project | `string` | n/a | yes |
-| <a name="input_vpc"></a> [vpc](#input\_vpc) | Settings for VPC | <pre>object({<br>    network = object({<br>      name = string<br>    })<br>    subnetwork = object({<br>      name          = string<br>      ip_cidr_range = string<br>    })<br>  })</pre> | n/a | yes |
+| <a name="input_vpc"></a> [vpc](#input\_vpc) | Settings for VPC | <pre>object({<br/>    network = object({<br/>      name = string<br/>    })<br/>    subnetwork = object({<br/>      name          = string<br/>      ip_cidr_range = string<br/>    })<br/>  })</pre> | n/a | yes |
 
 ## Outputs
 

modules/gcs2spanner/functoins.tf

@@ -7,9 +7,11 @@ locals {
 }
 
 resource "google_cloudfunctions2_function" "main" {
-  name     = var.functions.name
+  project  = var.project_id
   location = var.location
 
+  name = var.functions.name
+
   build_config {
     runtime     = "python310"
     entry_point = "create"
@@ -59,8 +61,10 @@ resource "google_cloudfunctions2_function" "main" {
 }
 
 resource "google_storage_bucket" "functions" {
+  project  = var.project_id
+  location = var.location
+
   name                        = var.functions.bucket
-  location                    = var.location
   force_destroy               = false
   public_access_prevention    = "enforced"
   uniform_bucket_level_access = true

modules/gcs2spanner/gcs.tf

@@ -6,8 +6,10 @@ locals {
 }
 
 resource "google_storage_bucket" "data" {
+  project  = var.project_id
+  location = var.location
+
   name                        = var.gcs.name
-  location                    = var.location
   force_destroy               = false
   public_access_prevention    = "enforced"
   uniform_bucket_level_access = true

modules/gcs2spanner/provider.tf

@@ -3,15 +3,15 @@ terraform {
 
   required_providers {
     archive = {
-      source  = "hashicorp/google"
+      source  = "hashicorp/archive"
       version = ">=2.4.0"
     }
     google = {
       source  = "hashicorp/google"
       version = ">= 5.22.0"
     }
     google-beta = {
-      source  = "hashicorp/google"
+      source  = "hashicorp/google-beta"
       version = ">= 5.22.0"
     }
   }