Programmatical Subnets for EKS

funcbench/Makefile

@@ -15,14 +15,12 @@ cluster_create:
 	$(INFRA_CMD) $(PROVIDER) cluster create -a ${AUTH_FILE} \
 		-v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v ZONE:${ZONE} -v CLUSTER_NAME:funcbench-${PR_NUMBER} -v PR_NUMBER:${PR_NUMBER} \
 		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
-		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \
 		-f manifests/cluster_$(PROVIDER).yaml
 
 cluster_delete:
 	$(INFRA_CMD) $(PROVIDER) cluster delete -a ${AUTH_FILE} \
 		-v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v ZONE:${ZONE} -v CLUSTER_NAME:funcbench-${PR_NUMBER} -v PR_NUMBER:${PR_NUMBER} \
 		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
-		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \
 		-f manifests/cluster_$(PROVIDER).yaml
 
 resource_apply:

infra/README.md

@@ -83,33 +83,33 @@ Commands:
     eks info -v hashStable:COMMIT1 -v hashTesting:COMMIT2
 
   eks cluster create
-    eks cluster create -a credentials -f FileOrFolder
+    eks cluster create -a authFile -f FileOrFolder
 
   eks cluster delete
-    eks cluster delete -a credentials -f FileOrFolder
+    eks cluster delete -a authFile -f FileOrFolder
 
   eks nodes create
     eks nodes create -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v
-    CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3
+    CLUSTER_NAME:test
 
   eks nodes delete
     eks nodes delete -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v
-    CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3
+    CLUSTER_NAME:test
 
   eks nodes check-running
     eks nodes check-running -a credentails -f FileOrFolder -v ZONE:eu-west-1 -v
-    CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3
+    CLUSTER_NAME:test
 
   eks nodes check-deleted
     eks nodes check-deleted -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v
-    CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3
+    CLUSTER_NAME:test
 
   eks resource apply
     eks resource apply -a credentials -f manifestsFileOrFolder -v
     hashStable:COMMIT1 -v hashTesting:COMMIT2
 
   eks resource delete
-    eks resource delete -a credentials -f manifestsFileOrFolder -v
+    eks resource delete -a authFile -f manifestsFileOrFolder -v
     hashStable:COMMIT1 -v hashTesting:COMMIT2
 
 

infra/infra.go

@@ -113,7 +113,7 @@ func main() {
 	k8sEKS := app.Command("eks", "Amazon Elastic Kubernetes Service - https://aws.amazon.com/eks").
 		Action(e.SetupDeploymentResources)
 	k8sEKS.Flag("auth", "filename which consist eks credentials.").
-		PlaceHolder("credentials").
+		PlaceHolder("authFile").
 		Short('a').
 		StringVar(&e.Auth)
 
@@ -124,22 +124,22 @@ func main() {
 	k8sEKSCluster := k8sEKS.Command("cluster", "manage EKS clusters").
 		Action(e.NewEKSClient).
 		Action(e.EKSDeploymentParse)
-	k8sEKSCluster.Command("create", "eks cluster create -a credentials -f FileOrFolder").
+	k8sEKSCluster.Command("create", "eks cluster create -a authFile -f FileOrFolder").
 		Action(e.ClusterCreate)
-	k8sEKSCluster.Command("delete", "eks cluster delete -a credentials -f FileOrFolder").
+	k8sEKSCluster.Command("delete", "eks cluster delete -a authFile -f FileOrFolder").
 		Action(e.ClusterDelete)
 
 	// Cluster node-pool operations
 	k8sEKSNodeGroup := k8sEKS.Command("nodes", "manage EKS clusters nodegroups").
 		Action(e.NewEKSClient).
 		Action(e.EKSDeploymentParse)
-	k8sEKSNodeGroup.Command("create", "eks nodes create -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3").
+	k8sEKSNodeGroup.Command("create", "eks nodes create -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test").
 		Action(e.NodeGroupCreate)
-	k8sEKSNodeGroup.Command("delete", "eks nodes delete -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3").
+	k8sEKSNodeGroup.Command("delete", "eks nodes delete -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test").
 		Action(e.NodeGroupDelete)
-	k8sEKSNodeGroup.Command("check-running", "eks nodes check-running -a credentails -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3").
+	k8sEKSNodeGroup.Command("check-running", "eks nodes check-running -a credentails -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test").
 		Action(e.AllNodeGroupsRunning)
-	k8sEKSNodeGroup.Command("check-deleted", "eks nodes check-deleted -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test -v EKS_SUBNET_IDS: subnetId1,subnetId2,subnetId3").
+	k8sEKSNodeGroup.Command("check-deleted", "eks nodes check-deleted -a authFile -f FileOrFolder -v ZONE:eu-west-1 -v CLUSTER_NAME:test").
 		Action(e.AllNodeGroupsDeleted)
 
 	// K8s resource operations.
@@ -149,7 +149,7 @@ func main() {
 		Action(e.NewK8sProvider)
 	k8sEKSResource.Command("apply", "eks resource apply -a credentials -f manifestsFileOrFolder -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
 		Action(e.ResourceApply)
-	k8sEKSResource.Command("delete", "eks resource delete -a credentials -f manifestsFileOrFolder -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
+	k8sEKSResource.Command("delete", "eks resource delete -a authFile -f manifestsFileOrFolder -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
 		Action(e.ResourceDelete)
 
 	if _, err := app.Parse(os.Args[1:]); err != nil {

pkg/provider/eks/eks.go

@@ -27,6 +27,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	awsSession "github.com/aws/aws-sdk-go/aws/session"
+	cloudFormation "github.com/aws/aws-sdk-go/service/cloudformation"
 	eks "github.com/aws/aws-sdk-go/service/eks"
 	"github.com/pkg/errors"
 	k8sProvider "github.com/prometheus/test-infra/pkg/provider/k8s"
@@ -57,6 +58,8 @@ type EKS struct {
 	clientEKS *eks.EKS
 	// The aws session used in abstraction of aws credentials.
 	sessionAWS *awsSession.Session
+	// The cloudFormation client used when performing CloudFormation requests.
+	clientCF *cloudFormation.CloudFormation
 	// The k8s provider used when we work with the manifest files.
 	k8sProvider *k8sProvider.K8s
 	// Final DeploymentFiles files.
@@ -118,7 +121,10 @@ func (c *EKS) NewEKSClient(*kingpin.ParseContext) error {
 	}))
 
 	c.sessionAWS = awsSess
+
 	c.clientEKS = eks.New(awsSess)
+	c.clientCF = cloudFormation.New(awsSess)
+
 	c.ctx = context.Background()
 	return nil
 }
@@ -154,6 +160,19 @@ func (c *EKS) EKSDeploymentParse(*kingpin.ParseContext) error {
 		return err
 	}
 
+	subnetIds, err := c.listdownSubnetIds(c.DeploymentVars["CLUSTER_NAME"])
+
+	if err != nil {
+		log.Fatalf("Error in listing down subnets: %v", err)
+	}
+
+	c.DeploymentVars = provider.MergeDeploymentVars(
+		c.DeploymentVars,
+		map[string]string{
+			"EKS_SUBNET_IDS": strings.Join(subnetIds, c.DeploymentVars["SEPARATOR"]),
+		},
+	)
+
 	deploymentResource, err := provider.DeploymentsParse(c.DeploymentFiles, c.DeploymentVars)
 	if err != nil {
 		return fmt.Errorf("Couldn't parse deployment files: %v", err)
@@ -203,6 +222,157 @@ func (c *EKS) K8SDeploymentsParse(*kingpin.ParseContext) error {
 	return nil
 }
 
+// listdownSubnetIds lists all subnet ids for given cluster name, if vpc for given cluster does not exists it creates vpc with given configuration.
+func (c *EKS) listdownSubnetIds(clusterName string) ([]string, error) {
+	subnetIds, err := c.showSubnetIds(clusterName)
+
+	if err != nil {
+		return []string{}, err
+	}
+
+	if len(subnetIds) == 0 {
+		subnetIds, err = c.createStack(clusterName)
+		if err != nil {
+			return []string{}, err
+		}
+	}
+
+	return subnetIds, err
+}
+
+// createStack creates a new vpc and prints desired subnetIds.
+func (c *EKS) createStack(clusterName string) ([]string, error) {
+	stackName := fmt.Sprintf("%s-vpc", clusterName)
+	req := &cloudFormation.CreateStackInput{
+		TemplateURL: aws.String("https://amazon-eks.s3.us-west-2.amazonaws.com/cloudformation/2020-07-23/amazon-eks-vpc-sample.yaml"),
+		StackName:   aws.String(stackName),
+		Tags: []*cloudFormation.Tag{
+			{
+				Key:   aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", clusterName)),
+				Value: aws.String("shared"),
+			},
+		},
+		DisableRollback: aws.Bool(true),
+	}
+
+	log.Printf("Stack create request: name:'%s'", *req.StackName)
+
+	_, err := c.clientCF.CreateStack(req)
+	if err != nil {
+		return nil, fmt.Errorf("Couldn't create stack '%s' ,err: %s", *req.StackName, err)
+	}
+
+	err = provider.RetryUntilTrue(
+		fmt.Sprintf("creating stack:%v", *req.StackName),
+		provider.GlobalRetryCount,
+		func() (bool, error) { return c.stackCreated(*req.StackName) },
+	)
+
+	if err != nil {
+		return nil, fmt.Errorf("creating stack err:%v", err)
+	}
+
+	return c.showSubnetIds(clusterName)
+}
+
+// deleteStack deletes vpc stack.
+func (c *EKS) deleteStack(clusterName string) error {
+	stackName := fmt.Sprintf("%s-vpc", clusterName)
+	req := &cloudFormation.DeleteStackInput{
+		StackName: aws.String(stackName),
+	}
+
+	log.Printf("Delete stack request: name: '%s'", *req.StackName)
+
+	_, err := c.clientCF.DeleteStack(req)
+	if err != nil {
+		return fmt.Errorf("Couldn't delete stack '%s' ,err: %s", *req.StackName, err)
+	}
+
+	err = provider.RetryUntilTrue(
+		fmt.Sprintf("deleting stack: %s", *req.StackName),
+		provider.GlobalRetryCount,
+		func() (bool, error) { return c.stackDeleted(*req.StackName) },
+	)
+
+	if err != nil {
+		return fmt.Errorf("deleting stack err:%v", err)
+	}
+
+	return nil
+}
+
+// stackCreated checks whether stack has been created.
+func (c *EKS) stackCreated(name string) (bool, error) {
+	req := &cloudFormation.DescribeStacksInput{
+		StackName: aws.String(name),
+	}
+	stackRes, err := c.clientCF.DescribeStacks(req)
+	if err != nil {
+		return false, fmt.Errorf("Couldn't get stack status: %v", err)
+	}
+	if *stackRes.Stacks[0].StackStatus == cloudFormation.StackStatusCreateFailed {
+		return false, fmt.Errorf("Stack not in a status to become ready - %s", *stackRes.Stacks[0].StackStatus)
+	}
+	if *stackRes.Stacks[0].StackStatus == cloudFormation.StackStatusCreateComplete {
+		return true, nil
+	}
+	log.Printf("Stack '%s' status: %s", name, *stackRes.Stacks[0].StackStatus)
+	return false, nil
+}
+
+// stackDeleted checks whether stack has been deleted.
+func (c *EKS) stackDeleted(name string) (bool, error) {
+	req := &cloudFormation.DescribeStacksInput{
+		StackName: aws.String(name),
+	}
+	stackRes, err := c.clientCF.DescribeStacks(req)
+
+	if err != nil {
+
+		if err.(awserr.Error).Code() == "ValidationError" {
+			return true, nil
+		}
+
+		return false, fmt.Errorf("Couldn't get stack status: %v", err)
+	}
+
+	if *stackRes.Stacks[0].StackStatus == cloudFormation.StackStatusDeleteFailed {
+		return false, fmt.Errorf("Stack delete failed - %s", *stackRes.Stacks[0].StackStatus)
+	}
+
+	log.Printf("Stack '%s' status: %s", name, *stackRes.Stacks[0].StackStatus)
+	return false, nil
+}
+
+// showSubnetIds returns all subnet ids for given cluster configuration.
+func (c *EKS) showSubnetIds(clusterName string) ([]string, error) {
+	stackName := fmt.Sprintf("%s-vpc", clusterName)
+	req := &cloudFormation.DescribeStacksInput{
+		StackName: aws.String(stackName),
+	}
+	stackRes, err := c.clientCF.DescribeStacks(req)
+
+	if awsErr, ok := err.(awserr.Error); ok {
+		if awsErr.Code() != "ValidationError" {
+			return []string{}, awsErr
+		}
+	}
+
+	for _, stack := range stackRes.Stacks {
+		if *stack.StackStatus == cloudFormation.StackStatusCreateComplete {
+			stackOutputs := stack.Outputs
+			for _, output := range stackOutputs {
+				if *output.OutputKey == "SubnetIds" {
+					return strings.Split(*output.OutputValue, ","), nil
+				}
+			}
+		}
+	}
+
+	return []string{}, nil
+}
+
 // ClusterCreate create a new cluster or applies changes to an existing cluster.
 func (c *EKS) ClusterCreate(*kingpin.ParseContext) error {
 	req := &eksCluster{}
@@ -321,6 +491,13 @@ func (c *EKS) ClusterDelete(*kingpin.ParseContext) error {
 		if err != nil {
 			return fmt.Errorf("removing cluster err:%v", err)
 		}
+
+		log.Printf("Removing subnets related to cluster '%s'", *req.Cluster.Name)
+		err = c.deleteStack(*req.Cluster.Name)
+
+		if err != nil {
+			log.Fatalf("removing cluster's subnets err:%v", err)
+		}
 	}
 	return nil
 }

prombench/Makefile

@@ -1,4 +1,5 @@
 INFRA_CMD        ?= ../infra/infra
+PROVIDER 		 ?= gke
 
 PROVIDER 		 ?= gke
 
@@ -12,7 +13,6 @@ node_create:
 	${INFRA_CMD} ${PROVIDER} nodes create -a ${AUTH_FILE} \
 		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
 		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
-		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \
 		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
 		-f manifests/prombench/nodes_${PROVIDER}.yaml
 
@@ -36,22 +36,19 @@ node_delete:
 	$(INFRA_CMD) ${PROVIDER} nodes delete -a ${AUTH_FILE} \
 		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
 		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
-		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \
 		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
 		-f manifests/prombench/nodes_${PROVIDER}.yaml
 
 all_nodes_running:
 	$(INFRA_CMD) ${PROVIDER} nodes check-running -a ${AUTH_FILE} \
 		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
 		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
-		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} -v SEPARATOR:${SEPARATOR} \
 		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
 		-f manifests/prombench/nodes_${PROVIDER}.yaml	
 
 all_nodes_deleted:
 	$(INFRA_CMD) ${PROVIDER} nodes check-deleted -a ${AUTH_FILE} \
 		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
 		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
-		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} -v SEPARATOR:${SEPARATOR} \
 		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
 		-f manifests/prombench/nodes_${PROVIDER}.yaml

prombench/docs/eks.md

@@ -32,11 +32,9 @@ export EKS_CLUSTER_ROLE_ARN=<Amazon EKS cluster role ARN>
 # By default SEPARATOR DeploymentVar is set to `,` but you can override it by exporting and
 # then passing it with the -v flag. It is used to split DeploymentVar into a slice.
 export SEPARATOR=, 
-export EKS_SUBNET_IDS=SUBNETID1,SUBNETID2,SUBNETID3
 
 ../infra/infra eks cluster create -a $AUTH_FILE -v ZONE:$ZONE \
     -v EKS_WORKER_ROLE_ARN:$EKS_WORKER_ROLE_ARN -v EKS_CLUSTER_ROLE_ARN:$EKS_CLUSTER_ROLE_ARN \
-    -v EKS_SUBNET_IDS:$EKS_SUBNET_IDS \
     -v CLUSTER_NAME:$CLUSTER_NAME \
     -f manifests/cluster_eks.yaml
 ```
@@ -97,7 +95,6 @@ export PR_NUMBER=<PR to benchmark against the selected $RELEASE>
 ```shell
 ../infra/infra eks nodes create -a $AUTH_FILE \
     -v ZONE:$ZONE -v EKS_WORKER_ROLE_ARN:$EKS_WORKER_ROLE_ARN -v EKS_CLUSTER_ROLE_ARN:$EKS_CLUSTER_ROLE_ARN \
-    -v EKS_SUBNET_IDS:$EKS_SUBNET_IDS \
     -v CLUSTER_NAME:$CLUSTER_NAME \
     -v PR_NUMBER:$PR_NUMBER -f manifests/prombench/nodes_eks.yaml
 ```

prombench/manifests/cluster_gke.yaml

@@ -1,4 +1,8 @@
+<<<<<<< HEAD
 projectid: {{ .GKE_PROJECT_ID }}
+=======
+projectid: {{ .PROJECT_ID }}
+>>>>>>> Chnage as per review
 zone: {{ .ZONE }}
 cluster:
   name: {{ .CLUSTER_NAME }}