diff --git a/.circleci/config.yml b/.circleci/config.yml index 350740c..94fce18 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ defaults: &defaults environment: GRUNTWORK_INSTALLER_VERSION: v0.0.21 TERRATEST_LOG_PARSER_VERSION: v0.13.13 - KUBERGRUNT_VERSION: v0.3.8 + KUBERGRUNT_VERSION: v0.3.9 HELM_VERSION: v2.11.0 MODULE_CI_VERSION: v0.13.3 TERRAFORM_VERSION: 0.11.8 @@ -94,6 +94,11 @@ jobs: sudo apt-get remove -y google-cloud-sdk sudo /opt/google-cloud-sdk/bin/gcloud --quiet components update sudo /opt/google-cloud-sdk/bin/gcloud --quiet components update beta kubectl + - run: + name: configure kubectl + command: | + mkdir -p ${HOME}/.kube + touch ${HOME}/.kube/config - run: name: run tests command: | diff --git a/examples/gke-basic-tiller/main.tf b/examples/gke-basic-tiller/main.tf index c3cf63d..910cd7e 100644 --- a/examples/gke-basic-tiller/main.tf +++ b/examples/gke-basic-tiller/main.tf @@ -57,8 +57,9 @@ data "google_client_config" "client" {} data "google_client_openid_userinfo" "terraform_user" {} provider "kubernetes" { - load_config_file = false + version = "~> 1.5.2" + load_config_file = false host = "${data.template_file.gke_host_endpoint.rendered}" token = "${data.template_file.access_token.rendered}" cluster_ca_certificate = "${data.template_file.cluster_ca_certificate.rendered}" @@ -207,6 +208,11 @@ module "vpc_network" { resource "null_resource" "configure_kubectl" { provisioner "local-exec" { command = "gcloud beta container clusters get-credentials ${module.gke_cluster.name} --region ${var.region} --project ${var.project}" + + # Use environment variables to allow custom kubectl config paths + environment = { + KUBECONFIG = "${var.kubectl_config_path != "" ? "${var.kubectl_config_path}" : ""}" + } } depends_on = ["google_container_node_pool.node_pool"] @@ -330,6 +336,13 @@ resource "null_resource" "grant_and_configure_helm" { kubergrunt helm configure --helm-home ${pathexpand("~/.helm")} --tiller-namespace ${local.tiller_namespace} --resource-namespace ${local.resource_namespace} --rbac-user ${data.google_client_openid_userinfo.terraform_user.email} ${local.kubectl_auth_config} EOF + + # Use environment variables for Kubernetes credentials to avoid leaking into the logs + environment = { + KUBECTL_SERVER_ENDPOINT = "${data.template_file.gke_host_endpoint.rendered}" + KUBECTL_CA_DATA = "${base64encode(data.template_file.cluster_ca_certificate.rendered)}" + KUBECTL_TOKEN = "${data.template_file.access_token.rendered}" + } } depends_on = ["null_resource.wait_for_tiller"] diff --git a/examples/gke-basic-tiller/variables.tf b/examples/gke-basic-tiller/variables.tf index baf4a12..1cb4b44 100644 --- a/examples/gke-basic-tiller/variables.tf +++ b/examples/gke-basic-tiller/variables.tf @@ -35,6 +35,13 @@ variable "cluster_service_account_description" { default = "Example GKE Cluster Service Account managed by Terraform" } +# Kubectl options + +variable "kubectl_config_path" { + description = "Path to the kubectl config file. Defaults to $HOME/.kube/config" + default = "" +} + # Tiller TLS settings variable "tls_subject" { diff --git a/main.tf b/main.tf index 69ea747..da1e6ef 100644 --- a/main.tf +++ b/main.tf @@ -56,8 +56,9 @@ data "google_client_config" "client" {} data "google_client_openid_userinfo" "terraform_user" {} provider "kubernetes" { - load_config_file = false + version = "~> 1.5.2" + load_config_file = false host = "${data.template_file.gke_host_endpoint.rendered}" token = "${data.template_file.access_token.rendered}" cluster_ca_certificate = "${data.template_file.cluster_ca_certificate.rendered}" @@ -224,6 +225,11 @@ module "vpc_network" { resource "null_resource" "configure_kubectl" { provisioner "local-exec" { command = "gcloud beta container clusters get-credentials ${module.gke_cluster.name} --region ${var.region} --project ${var.project}" + + # Use environment variables to allow custom kubectl config paths + environment = { + KUBECONFIG = "${var.kubectl_config_path != "" ? "${var.kubectl_config_path}" : ""}" + } } depends_on = ["google_container_node_pool.node_pool"] @@ -347,6 +353,13 @@ resource "null_resource" "grant_and_configure_helm" { kubergrunt helm configure --helm-home ${pathexpand("~/.helm")} --tiller-namespace ${local.tiller_namespace} --resource-namespace ${local.resource_namespace} --rbac-user ${data.google_client_openid_userinfo.terraform_user.email} ${local.kubectl_auth_config} EOF + + # Use environment variables for Kubernetes credentials to avoid leaking into the logs + environment = { + KUBECTL_SERVER_ENDPOINT = "${data.template_file.gke_host_endpoint.rendered}" + KUBECTL_CA_DATA = "${base64encode(data.template_file.cluster_ca_certificate.rendered)}" + KUBECTL_TOKEN = "${data.template_file.access_token.rendered}" + } } depends_on = ["null_resource.wait_for_tiller"] diff --git a/test/Gopkg.lock b/test/Gopkg.lock index 8dbd30e..fe7d890 100644 --- a/test/Gopkg.lock +++ b/test/Gopkg.lock @@ -18,7 +18,7 @@ version = "v0.35.1" [[projects]] - digest = "1:f0ca0b3256cfe939f5b5d7fb9980d3805f0a9c0d7e2ce5c990112627e77018d7" + digest = "1:6fee3887aa799d9c2c2017d2738c5532b9cbee3fa94589232f11db00ff1e6ec3" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -60,6 +60,7 @@ "service/autoscaling", "service/cloudwatchlogs", "service/ec2", + "service/ecs", "service/iam", "service/kms", "service/rds", @@ -230,7 +231,7 @@ version = "v0.4.2" [[projects]] - digest = "1:12cf87df04f335d94d1346edd77536ca505a4dda0afd5137150cd440e5c2d32c" + digest = "1:9caaf503201ab0dfd178194b54306c45e72b6ea1dc7de5e432d933418149f9aa" name = "github.com/gruntwork-io/terratest" packages = [ "modules/aws", @@ -252,8 +253,8 @@ "modules/test-structure", ] pruneopts = "UT" - revision = "b6077c8c1ecfd290ee1f37e1e51472a34d30cb38" - version = "v0.13.29" + revision = "54393d85e00e2614efa6bd22e963fe174cc128d5" + version = "v0.14.6" [[projects]] digest = "1:a0cefd27d12712af4b5018dc7046f245e1e3b5760e2e848c30b171b570708f9b" @@ -496,7 +497,7 @@ [[projects]] branch = "master" - digest = "1:d586ebbd814cf4a680afbac35457f94a4ec9ae8305d9831b7a290c32baa0bb31" + digest = "1:a4df62df5ca682765882afdd14d4c904f2841a2fa455b044f4b04545f66cbd74" name = "google.golang.org/api" packages = [ "compute/v1", @@ -507,6 +508,7 @@ "internal", "iterator", "option", + "oslogin/v1", "storage/v1", "transport/http", "transport/http/internal/propagation", @@ -777,10 +779,10 @@ "github.com/gruntwork-io/terratest/modules/logger", "github.com/gruntwork-io/terratest/modules/random", "github.com/gruntwork-io/terratest/modules/retry", - "github.com/gruntwork-io/terratest/modules/shell", "github.com/gruntwork-io/terratest/modules/terraform", "github.com/gruntwork-io/terratest/modules/test-structure", "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/require", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/test/Gopkg.toml b/test/Gopkg.toml index 2436a79..d96c92f 100644 --- a/test/Gopkg.toml +++ b/test/Gopkg.toml @@ -23,7 +23,7 @@ [[constraint]] name = "github.com/gruntwork-io/terratest" - version = "0.13.28" + version = "0.14.6" [prune] go-tests = true diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go index 3b47e20..258955f 100644 --- a/test/gke_basic_tiller_test.go +++ b/test/gke_basic_tiller_test.go @@ -2,6 +2,7 @@ package test import ( "fmt" + "os" "path/filepath" "strings" "testing" @@ -9,19 +10,17 @@ import ( "github.com/gruntwork-io/terratest/modules/gcp" "github.com/gruntwork-io/terratest/modules/helm" - "github.com/gruntwork-io/terratest/modules/http-helper" + http_helper "github.com/gruntwork-io/terratest/modules/http-helper" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/gruntwork-io/terratest/modules/test-structure" + test_structure "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/require" ) func TestGKEBasicTiller(t *testing.T) { - // We are temporarily stopping the tests from running in parallel due to conflicting - // kubectl configs. This is a limitation in the current Terratest functions and will - // be fixed in a later release. - //t.Parallel() + t.Parallel() // Uncomment any of the following to skip that section during the test // os.Setenv("SKIP_create_test_copy_of_examples", "true") @@ -43,19 +42,27 @@ func TestGKEBasicTiller(t *testing.T) { test_structure.RunTestStage(t, "create_terratest_options", func() { gkeBasicTillerTerraformModulePath := test_structure.LoadString(t, workingDir, "gkeBasicTillerTerraformModulePath") + tmpKubeConfigPath := k8s.CopyHomeKubeConfigToTemp(t) + kubectlOptions := k8s.NewKubectlOptions("", tmpKubeConfigPath) uniqueID := random.UniqueId() project := gcp.GetGoogleProjectIDFromEnvVar(t) region := gcp.GetRandomRegion(t, project, nil, nil) - gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, gkeBasicTillerTerraformModulePath) + gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, + gkeBasicTillerTerraformModulePath, tmpKubeConfigPath) test_structure.SaveString(t, workingDir, "uniqueID", uniqueID) test_structure.SaveString(t, workingDir, "project", project) test_structure.SaveString(t, workingDir, "region", region) test_structure.SaveTerraformOptions(t, workingDir, gkeClusterTerratestOptions) + test_structure.SaveKubectlOptions(t, workingDir, kubectlOptions) }) defer test_structure.RunTestStage(t, "cleanup", func() { gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) terraform.Destroy(t, gkeClusterTerratestOptions) + + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) + err := os.Remove(kubectlOptions.ConfigPath) + require.NoError(t, err) }) test_structure.RunTestStage(t, "terraform_apply", func() { @@ -64,18 +71,17 @@ func TestGKEBasicTiller(t *testing.T) { }) test_structure.RunTestStage(t, "wait_for_workers", func() { - verifyGkeNodesAreReady(t) + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) + verifyGkeNodesAreReady(t, kubectlOptions) }) test_structure.RunTestStage(t, "helm_install", func() { // Path to the helm chart we will test helmChartPath := "charts/minimal-pod" - // Setup the kubectl config and context. Here we choose to use the defaults, which is: - // - HOME/.kube/config for the kubectl config file - // - Current context of the kubectl config file + // Load the temporary kubectl config file and use its current context // We also specify that we are working in the default namespace (required to get the Pod) - kubectlOptions := k8s.NewKubectlOptions("", "") + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) kubectlOptions.Namespace = "default" // We generate a unique release name so that we can refer to after deployment. @@ -97,6 +103,7 @@ func TestGKEBasicTiller(t *testing.T) { "HELM_TLS_VERIFY": "true", "HELM_TLS_ENABLE": "true", }, + KubectlOptions: kubectlOptions, } // Deploy the chart using `helm install`. Note that we use the version without `E`, since we want to assert the diff --git a/test/gke_cluster_test.go b/test/gke_cluster_test.go index f72ccb1..b75cd2d 100644 --- a/test/gke_cluster_test.go +++ b/test/gke_cluster_test.go @@ -1,22 +1,22 @@ package test import ( + "os" "path/filepath" "testing" "github.com/gruntwork-io/terratest/modules/gcp" + "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/gruntwork-io/terratest/modules/test-structure" + test_structure "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/require" ) func TestGKECluster(t *testing.T) { - // We are temporarily stopping the tests from running in parallel due to conflicting - // kubectl configs. This is a limitation in the current Terratest functions and will - // be fixed in a later release. - //t.Parallel() + t.Parallel() var testcases = []struct { testName string @@ -38,10 +38,7 @@ func TestGKECluster(t *testing.T) { testCase := testCase t.Run(testCase.testName, func(t *testing.T) { - // We are temporarily stopping the tests from running in parallel due to conflicting - // kubectl configs. This is a limitation in the current Terratest functions and will - // be fixed in a later release. - //t.Parallel() + t.Parallel() // Uncomment any of the following to skip that section during the test //os.Setenv("SKIP_create_test_copy_of_examples", "true") @@ -63,19 +60,27 @@ func TestGKECluster(t *testing.T) { test_structure.RunTestStage(t, "create_terratest_options", func() { gkeClusterTerraformModulePath := test_structure.LoadString(t, workingDir, "gkeClusterTerraformModulePath") + tmpKubeConfigPath := k8s.CopyHomeKubeConfigToTemp(t) + kubectlOptions := k8s.NewKubectlOptions("", tmpKubeConfigPath) uniqueID := random.UniqueId() project := gcp.GetGoogleProjectIDFromEnvVar(t) region := gcp.GetRandomRegion(t, project, nil, nil) - gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, gkeClusterTerraformModulePath) + gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, + gkeClusterTerraformModulePath, tmpKubeConfigPath) test_structure.SaveString(t, workingDir, "uniqueID", uniqueID) test_structure.SaveString(t, workingDir, "project", project) test_structure.SaveString(t, workingDir, "region", region) test_structure.SaveTerraformOptions(t, workingDir, gkeClusterTerratestOptions) + test_structure.SaveKubectlOptions(t, workingDir, kubectlOptions) }) defer test_structure.RunTestStage(t, "cleanup", func() { gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) terraform.Destroy(t, gkeClusterTerratestOptions) + + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) + err := os.Remove(kubectlOptions.ConfigPath) + require.NoError(t, err) }) test_structure.RunTestStage(t, "terraform_apply", func() { @@ -85,6 +90,7 @@ func TestGKECluster(t *testing.T) { test_structure.RunTestStage(t, "configure_kubectl", func() { gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) project := test_structure.LoadString(t, workingDir, "project") region := test_structure.LoadString(t, workingDir, "region") clusterName := gkeClusterTerratestOptions.Vars["cluster_name"].(string) @@ -93,13 +99,17 @@ func TestGKECluster(t *testing.T) { cmd := shell.Command{ Command: "gcloud", Args: []string{"beta", "container", "clusters", "get-credentials", clusterName, "--region", region, "--project", project}, + Env: map[string]string{ + "KUBECONFIG": kubectlOptions.ConfigPath, + }, } shell.RunCommand(t, cmd) }) test_structure.RunTestStage(t, "wait_for_workers", func() { - verifyGkeNodesAreReady(t) + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) + verifyGkeNodesAreReady(t, kubectlOptions) }) }) } diff --git a/test/terratest_options.go b/test/terratest_options.go index 497ab8c..17d8d8c 100644 --- a/test/terratest_options.go +++ b/test/terratest_options.go @@ -14,14 +14,17 @@ func createGKEClusterTerraformOptions( project string, region string, templatePath string, + kubeConfigPath string, ) *terraform.Options { gkeClusterName := strings.ToLower(fmt.Sprintf("gke-cluster-%s", uniqueID)) + gkeServiceAccountName := strings.ToLower(fmt.Sprintf("gke-cluster-sa-%s", uniqueID)) terraformVars := map[string]interface{}{ - "region": region, - "location": region, - "project": project, - "cluster_name": gkeClusterName, + "region": region, + "location": region, + "project": project, + "cluster_name": gkeClusterName, + "cluster_service_account_name": gkeServiceAccountName, "tls_subject": map[string]string{ "common_name": "tiller", "org": "Gruntwork", @@ -30,8 +33,9 @@ func createGKEClusterTerraformOptions( "common_name": "helm", "org": "Gruntwork", }, - "force_undeploy": true, - "undeploy_release": true, + "force_undeploy": true, + "undeploy_release": true, + "kubectl_config_path": kubeConfigPath, } terratestOptions := terraform.Options{ diff --git a/test/test_helpers.go b/test/test_helpers.go index f2d51ed..33ef675 100644 --- a/test/test_helpers.go +++ b/test/test_helpers.go @@ -14,7 +14,7 @@ import ( // kubeWaitUntilNumNodes continuously polls the Kubernetes cluster until there are the expected number of nodes // registered (regardless of readiness). -func kubeWaitUntilNumNodes(t *testing.T, numNodes int, retries int, sleepBetweenRetries time.Duration) { +func kubeWaitUntilNumNodes(t *testing.T, kubectlOptions *k8s.KubectlOptions, numNodes int, retries int, sleepBetweenRetries time.Duration) { statusMsg := fmt.Sprintf("Wait for %d Kube Nodes to be registered.", numNodes) message, err := retry.DoWithRetryE( t, @@ -22,7 +22,7 @@ func kubeWaitUntilNumNodes(t *testing.T, numNodes int, retries int, sleepBetween retries, sleepBetweenRetries, func() (string, error) { - nodes, err := k8s.GetNodesE(t) + nodes, err := k8s.GetNodesE(t, kubectlOptions) if err != nil { return "", err } @@ -40,9 +40,9 @@ func kubeWaitUntilNumNodes(t *testing.T, numNodes int, retries int, sleepBetween } // Verify that all the nodes in the cluster reach the Ready state. -func verifyGkeNodesAreReady(t *testing.T) { - kubeWaitUntilNumNodes(t, 3, 30, 10*time.Second) - k8s.WaitUntilAllNodesReady(t, 30, 10*time.Second) - readyNodes := k8s.GetReadyNodes(t) +func verifyGkeNodesAreReady(t *testing.T, kubectlOptions *k8s.KubectlOptions) { + kubeWaitUntilNumNodes(t, kubectlOptions, 3, 30, 10*time.Second) + k8s.WaitUntilAllNodesReady(t, kubectlOptions, 30, 10*time.Second) + readyNodes := k8s.GetReadyNodes(t, kubectlOptions) assert.Equal(t, len(readyNodes), 3) } diff --git a/variables.tf b/variables.tf index 9ffcb30..5b6b79e 100644 --- a/variables.tf +++ b/variables.tf @@ -35,6 +35,13 @@ variable "cluster_service_account_description" { default = "Example GKE Cluster Service Account managed by Terraform" } +# Kubectl options + +variable "kubectl_config_path" { + description = "Path to the kubectl config file. Defaults to $HOME/.kube/config" + default = "" +} + # Tiller TLS settings variable "tls_subject" {