Merge pull request #39 from gruntwork-io/better-tests

Better (Parallel) Tests

.circleci/config.yml

@@ -3,7 +3,7 @@ defaults: &defaults
   environment:
     GRUNTWORK_INSTALLER_VERSION: v0.0.21
     TERRATEST_LOG_PARSER_VERSION: v0.13.13
-    KUBERGRUNT_VERSION: v0.3.8
+    KUBERGRUNT_VERSION: v0.3.9
     HELM_VERSION: v2.11.0
     MODULE_CI_VERSION: v0.13.3
     TERRAFORM_VERSION: 0.11.8
@@ -94,6 +94,11 @@ jobs:
           sudo apt-get remove -y google-cloud-sdk
           sudo /opt/google-cloud-sdk/bin/gcloud --quiet components update
           sudo /opt/google-cloud-sdk/bin/gcloud --quiet components update beta kubectl
+    - run:
+        name: configure kubectl
+        command: |
+          mkdir -p ${HOME}/.kube
+          touch ${HOME}/.kube/config
     - run:
         name: run tests
         command: |

examples/gke-basic-tiller/main.tf

@@ -57,8 +57,9 @@ data "google_client_config" "client" {}
 data "google_client_openid_userinfo" "terraform_user" {}
 
 provider "kubernetes" {
-  load_config_file = false
+  version = "~> 1.5.2"
 
+  load_config_file       = false
   host                   = "${data.template_file.gke_host_endpoint.rendered}"
   token                  = "${data.template_file.access_token.rendered}"
   cluster_ca_certificate = "${data.template_file.cluster_ca_certificate.rendered}"
@@ -207,6 +208,11 @@ module "vpc_network" {
 resource "null_resource" "configure_kubectl" {
   provisioner "local-exec" {
     command = "gcloud beta container clusters get-credentials ${module.gke_cluster.name} --region ${var.region} --project ${var.project}"
+
+    # Use environment variables to allow custom kubectl config paths
+    environment = {
+      KUBECONFIG = "${var.kubectl_config_path != "" ? "${var.kubectl_config_path}" : ""}"
+    }
   }
 
   depends_on = ["google_container_node_pool.node_pool"]
@@ -330,6 +336,13 @@ resource "null_resource" "grant_and_configure_helm" {
 
     kubergrunt helm configure --helm-home ${pathexpand("~/.helm")} --tiller-namespace ${local.tiller_namespace} --resource-namespace ${local.resource_namespace} --rbac-user ${data.google_client_openid_userinfo.terraform_user.email} ${local.kubectl_auth_config}
     EOF
+
+    # Use environment variables for Kubernetes credentials to avoid leaking into the logs
+    environment = {
+      KUBECTL_SERVER_ENDPOINT = "${data.template_file.gke_host_endpoint.rendered}"
+      KUBECTL_CA_DATA         = "${base64encode(data.template_file.cluster_ca_certificate.rendered)}"
+      KUBECTL_TOKEN           = "${data.template_file.access_token.rendered}"
+    }
   }
 
   depends_on = ["null_resource.wait_for_tiller"]

examples/gke-basic-tiller/variables.tf

@@ -35,6 +35,13 @@ variable "cluster_service_account_description" {
   default     = "Example GKE Cluster Service Account managed by Terraform"
 }
 
+# Kubectl options
+
+variable "kubectl_config_path" {
+  description = "Path to the kubectl config file. Defaults to $HOME/.kube/config"
+  default     = ""
+}
+
 # Tiller TLS  settings
 
 variable "tls_subject" {

main.tf

@@ -56,8 +56,9 @@ data "google_client_config" "client" {}
 data "google_client_openid_userinfo" "terraform_user" {}
 
 provider "kubernetes" {
-  load_config_file = false
+  version = "~> 1.5.2"
 
+  load_config_file       = false
   host                   = "${data.template_file.gke_host_endpoint.rendered}"
   token                  = "${data.template_file.access_token.rendered}"
   cluster_ca_certificate = "${data.template_file.cluster_ca_certificate.rendered}"
@@ -224,6 +225,11 @@ module "vpc_network" {
 resource "null_resource" "configure_kubectl" {
   provisioner "local-exec" {
     command = "gcloud beta container clusters get-credentials ${module.gke_cluster.name} --region ${var.region} --project ${var.project}"
+
+    # Use environment variables to allow custom kubectl config paths
+    environment = {
+      KUBECONFIG = "${var.kubectl_config_path != "" ? "${var.kubectl_config_path}" : ""}"
+    }
   }
 
   depends_on = ["google_container_node_pool.node_pool"]
@@ -347,6 +353,13 @@ resource "null_resource" "grant_and_configure_helm" {
 
     kubergrunt helm configure --helm-home ${pathexpand("~/.helm")} --tiller-namespace ${local.tiller_namespace} --resource-namespace ${local.resource_namespace} --rbac-user ${data.google_client_openid_userinfo.terraform_user.email} ${local.kubectl_auth_config}
     EOF
+
+    # Use environment variables for Kubernetes credentials to avoid leaking into the logs
+    environment = {
+      KUBECTL_SERVER_ENDPOINT = "${data.template_file.gke_host_endpoint.rendered}"
+      KUBECTL_CA_DATA         = "${base64encode(data.template_file.cluster_ca_certificate.rendered)}"
+      KUBECTL_TOKEN           = "${data.template_file.access_token.rendered}"
+    }
   }
 
   depends_on = ["null_resource.wait_for_tiller"]

test/Gopkg.toml

@@ -23,7 +23,7 @@
 
 [[constraint]]
   name = "github.com/gruntwork-io/terratest"
-  version = "0.13.28"
+  version = "0.14.6"
 
 [prune]
   go-tests = true

test/gke_basic_tiller_test.go

@@ -2,26 +2,25 @@ package test
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/gcp"
 	"github.com/gruntwork-io/terratest/modules/helm"
-	"github.com/gruntwork-io/terratest/modules/http-helper"
+	http_helper "github.com/gruntwork-io/terratest/modules/http-helper"
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/gruntwork-io/terratest/modules/terraform"
-	"github.com/gruntwork-io/terratest/modules/test-structure"
+	test_structure "github.com/gruntwork-io/terratest/modules/test-structure"
+	"github.com/stretchr/testify/require"
 )
 
 func TestGKEBasicTiller(t *testing.T) {
-	// We are temporarily stopping the tests from running in parallel due to conflicting
-	// kubectl configs. This is a limitation in the current Terratest functions and will
-	// be fixed in a later release.
-	//t.Parallel()
+	t.Parallel()
 
 	// Uncomment any of the following to skip that section during the test
 	// os.Setenv("SKIP_create_test_copy_of_examples", "true")
@@ -43,19 +42,27 @@ func TestGKEBasicTiller(t *testing.T) {
 
 	test_structure.RunTestStage(t, "create_terratest_options", func() {
 		gkeBasicTillerTerraformModulePath := test_structure.LoadString(t, workingDir, "gkeBasicTillerTerraformModulePath")
+		tmpKubeConfigPath := k8s.CopyHomeKubeConfigToTemp(t)
+		kubectlOptions := k8s.NewKubectlOptions("", tmpKubeConfigPath)
 		uniqueID := random.UniqueId()
 		project := gcp.GetGoogleProjectIDFromEnvVar(t)
 		region := gcp.GetRandomRegion(t, project, nil, nil)
-		gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, gkeBasicTillerTerraformModulePath)
+		gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region,
+			gkeBasicTillerTerraformModulePath, tmpKubeConfigPath)
 		test_structure.SaveString(t, workingDir, "uniqueID", uniqueID)
 		test_structure.SaveString(t, workingDir, "project", project)
 		test_structure.SaveString(t, workingDir, "region", region)
 		test_structure.SaveTerraformOptions(t, workingDir, gkeClusterTerratestOptions)
+		test_structure.SaveKubectlOptions(t, workingDir, kubectlOptions)
 	})
 
 	defer test_structure.RunTestStage(t, "cleanup", func() {
 		gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir)
 		terraform.Destroy(t, gkeClusterTerratestOptions)
+
+		kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir)
+		err := os.Remove(kubectlOptions.ConfigPath)
+		require.NoError(t, err)
 	})
 
 	test_structure.RunTestStage(t, "terraform_apply", func() {
@@ -64,18 +71,17 @@ func TestGKEBasicTiller(t *testing.T) {
 	})
 
 	test_structure.RunTestStage(t, "wait_for_workers", func() {
-		verifyGkeNodesAreReady(t)
+		kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir)
+		verifyGkeNodesAreReady(t, kubectlOptions)
 	})
 
 	test_structure.RunTestStage(t, "helm_install", func() {
 		// Path to the helm chart we will test
 		helmChartPath := "charts/minimal-pod"
 
-		// Setup the kubectl config and context. Here we choose to use the defaults, which is:
-		// - HOME/.kube/config for the kubectl config file
-		// - Current context of the kubectl config file
+		// Load the temporary kubectl config file and use its current context
 		// We also specify that we are working in the default namespace (required to get the Pod)
-		kubectlOptions := k8s.NewKubectlOptions("", "")
+		kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir)
 		kubectlOptions.Namespace = "default"
 
 		// We generate a unique release name so that we can refer to after deployment.
@@ -97,6 +103,7 @@ func TestGKEBasicTiller(t *testing.T) {
 				"HELM_TLS_VERIFY": "true",
 				"HELM_TLS_ENABLE": "true",
 			},
+			KubectlOptions: kubectlOptions,
 		}
 
 		// Deploy the chart using `helm install`. Note that we use the version without `E`, since we want to assert the

test/gke_cluster_test.go

@@ -1,22 +1,22 @@
 package test
 
 import (
+	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/gruntwork-io/terratest/modules/gcp"
+	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/gruntwork-io/terratest/modules/shell"
 	"github.com/gruntwork-io/terratest/modules/terraform"
-	"github.com/gruntwork-io/terratest/modules/test-structure"
+	test_structure "github.com/gruntwork-io/terratest/modules/test-structure"
+	"github.com/stretchr/testify/require"
 )
 
 func TestGKECluster(t *testing.T) {
-	// We are temporarily stopping the tests from running in parallel due to conflicting
-	// kubectl configs. This is a limitation in the current Terratest functions and will
-	// be fixed in a later release.
-	//t.Parallel()
+	t.Parallel()
 
 	var testcases = []struct {
 		testName      string
@@ -38,10 +38,7 @@ func TestGKECluster(t *testing.T) {
 		testCase := testCase
 
 		t.Run(testCase.testName, func(t *testing.T) {
-			// We are temporarily stopping the tests from running in parallel due to conflicting
-			// kubectl configs. This is a limitation in the current Terratest functions and will
-			// be fixed in a later release.
-			//t.Parallel()
+			t.Parallel()
 
 			// Uncomment any of the following to skip that section during the test
 			//os.Setenv("SKIP_create_test_copy_of_examples", "true")
@@ -63,19 +60,27 @@ func TestGKECluster(t *testing.T) {
 
 			test_structure.RunTestStage(t, "create_terratest_options", func() {
 				gkeClusterTerraformModulePath := test_structure.LoadString(t, workingDir, "gkeClusterTerraformModulePath")
+				tmpKubeConfigPath := k8s.CopyHomeKubeConfigToTemp(t)
+				kubectlOptions := k8s.NewKubectlOptions("", tmpKubeConfigPath)
 				uniqueID := random.UniqueId()
 				project := gcp.GetGoogleProjectIDFromEnvVar(t)
 				region := gcp.GetRandomRegion(t, project, nil, nil)
-				gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, gkeClusterTerraformModulePath)
+				gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region,
+					gkeClusterTerraformModulePath, tmpKubeConfigPath)
 				test_structure.SaveString(t, workingDir, "uniqueID", uniqueID)
 				test_structure.SaveString(t, workingDir, "project", project)
 				test_structure.SaveString(t, workingDir, "region", region)
 				test_structure.SaveTerraformOptions(t, workingDir, gkeClusterTerratestOptions)
+				test_structure.SaveKubectlOptions(t, workingDir, kubectlOptions)
 			})
 
 			defer test_structure.RunTestStage(t, "cleanup", func() {
 				gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir)
 				terraform.Destroy(t, gkeClusterTerratestOptions)
+
+				kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir)
+				err := os.Remove(kubectlOptions.ConfigPath)
+				require.NoError(t, err)
 			})
 
 			test_structure.RunTestStage(t, "terraform_apply", func() {
@@ -85,6 +90,7 @@ func TestGKECluster(t *testing.T) {
 
 			test_structure.RunTestStage(t, "configure_kubectl", func() {
 				gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir)
+				kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir)
 				project := test_structure.LoadString(t, workingDir, "project")
 				region := test_structure.LoadString(t, workingDir, "region")
 				clusterName := gkeClusterTerratestOptions.Vars["cluster_name"].(string)
@@ -93,13 +99,17 @@ func TestGKECluster(t *testing.T) {
 				cmd := shell.Command{
 					Command: "gcloud",
 					Args:    []string{"beta", "container", "clusters", "get-credentials", clusterName, "--region", region, "--project", project},
+					Env: map[string]string{
+						"KUBECONFIG": kubectlOptions.ConfigPath,
+					},
 				}
 
 				shell.RunCommand(t, cmd)
 			})
 
 			test_structure.RunTestStage(t, "wait_for_workers", func() {
-				verifyGkeNodesAreReady(t)
+				kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir)
+				verifyGkeNodesAreReady(t, kubectlOptions)
 			})
 		})
 	}