Implement JMX E2E Tests in Terraform Framework

.gitignore

@@ -10,4 +10,5 @@ terraform.*
 **/.terraform/*
 /test/**/final_*.yml
 coverage.txt
-generator/resources/*complete*.json
+generator/resources/*complete*.json
+terraform/eks/e2e/helm-charts

environment/metadata.go

@@ -47,6 +47,7 @@ type MetaData struct {
 	AmpWorkspaceId                              string
 	Region                                      string
 	K8sVersion                                  string
+	Destroy                                     bool
 	HelmChartsBranch                            string
 	CloudwatchAgentRepository                   string
 	CloudwatchAgentTag                          string
@@ -87,6 +88,7 @@ type MetaDataStrings struct {
 	AmpWorkspaceId                              string
 	Region                                      string
 	K8sVersion                                  string
+	Destroy                                     string
 	HelmChartsBranch                            string
 	CloudwatchAgentRepository                   string
 	CloudwatchAgentTag                          string
@@ -136,6 +138,7 @@ func registerEKSData(d *MetaDataStrings) {
 func registerEKSE2ETestData(dataString *MetaDataStrings) {
 	flag.StringVar(&(dataString.Region), "region", "", "AWS region")
 	flag.StringVar(&(dataString.K8sVersion), "k8s_version", "", "Kubernetes version")
+	flag.StringVar(&(dataString.Destroy), "destroy", "false", "Whether to run in destroy mode (true/false)")
 	flag.StringVar(&(dataString.HelmChartsBranch), "helm_charts_branch", "", "Helm charts branch")
 	flag.StringVar(&(dataString.CloudwatchAgentRepository), "cloudwatch_agent_repository", "", "CloudWatch Agent repository")
 	flag.StringVar(&(dataString.CloudwatchAgentTag), "cloudwatch_agent_tag", "", "CloudWatch Agent tag")
@@ -318,6 +321,7 @@ func GetEnvironmentMetaData() *MetaData {
 	metaDataStorage.AmpWorkspaceId = registeredMetaDataStrings.AmpWorkspaceId
 	metaDataStorage.Region = registeredMetaDataStrings.Region
 	metaDataStorage.K8sVersion = registeredMetaDataStrings.K8sVersion
+	metaDataStorage.Destroy = registeredMetaDataStrings.Destroy == "true"
 	metaDataStorage.HelmChartsBranch = registeredMetaDataStrings.HelmChartsBranch
 	metaDataStorage.CloudwatchAgentRepository = registeredMetaDataStrings.CloudwatchAgentRepository
 	metaDataStorage.CloudwatchAgentTag = registeredMetaDataStrings.CloudwatchAgentTag

generator/resources/eks_e2e_jmx_test_matrix.json

@@ -0,0 +1,5 @@
+[
+  {
+    "nodes": 1
+  }
+]

generator/test_case_generator.go

@@ -30,6 +30,7 @@ type matrixRow struct {
 	CaCertPath          string `json:"caCertPath"`
 	ValuesPerMinute     int    `json:"values_per_minute"` // Number of metrics to be sent or number of log lines to write
 	K8sVersion          string `json:"k8sVersion"`
+	Nodes               int    `json:"nodes"`
 	TerraformDir        string `json:"terraform_dir"`
 	UseSSM              bool   `json:"useSSM"`
 	ExcludedTests       string `json:"excludedTests"`
@@ -247,7 +248,7 @@ var testTypeToTestConfig = map[string][]testConfig{
 }
 
 var testTypeToTestConfigE2E = map[string][]testConfig{
-	"eks_e2e": {
+	"eks_e2e_jmx": {
 		{testDir: "../../../test/e2e/jmx"},
 	},
 }

terraform/eks/e2e/main.tf

@@ -39,10 +39,10 @@ resource "aws_eks_node_group" "this" {
     min_size     = var.nodes
   }
 
-  ami_type       = "AL2_x86_64"
+  ami_type       = var.ami_type
   capacity_type  = "ON_DEMAND"
   disk_size      = 20
-  instance_types = ["t3a.medium"]
+  instance_types = [var.instance_type]
 
   depends_on = [
     aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy,
@@ -52,6 +52,15 @@ resource "aws_eks_node_group" "this" {
   ]
 }
 
+resource "aws_security_group_rule" "nodeport_inbound" {
+  type              = "ingress"
+  from_port         = 30080
+  to_port           = 30080
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id
+}
+
 resource "aws_iam_role" "node_role" {
   name = "${local.cluster_name}-Worker-Role-${module.common.testing_id}"
 
@@ -109,6 +118,12 @@ resource "null_resource" "helm_charts" {
 resource "null_resource" "validator" {
   depends_on = [aws_eks_cluster.this, aws_eks_node_group.this, null_resource.helm_charts]
 
+  triggers = {
+    cluster_name = aws_eks_cluster.this.name
+    region       = var.region
+    test_dir     = var.test_dir
+  }
+
   provisioner "local-exec" {
     command = <<-EOT
       echo "Validating K8s resources and metrics"
@@ -134,4 +149,17 @@ resource "null_resource" "validator" {
       -sample_app="${var.test_dir}/${var.sample_app}"
     EOT
   }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = <<-EOT
+      echo "Running cleanup for K8s resources"
+      go test -timeout 30m -v ${self.triggers.test_dir} \
+      -destroy=true \
+      -region=${self.triggers.region} \
+      -eksClusterName=${self.triggers.cluster_name} \
+      -computeType=EKS \
+      -eksDeploymentStrategy=DAEMON
+    EOT
+  }
 }

terraform/eks/e2e/variables.tf

@@ -21,6 +21,16 @@ variable "nodes" {
   default = 1
 }
 
+variable "ami_type" {
+  type    = string
+  default = "AL2_x86_64"
+}
+
+variable "instance_type" {
+  type    = string
+  default = "t3a.medium"
+}
+
 variable "helm_charts_branch" {
   type    = string
   default = "main"