Re-arrange directory structure and docs for GKE to support multiple providers (#425)

* Add documentation for GKE and update file names in both prombench and funcbench
* Add and update variables in funcbench and prombench Makefiles

Signed-off-by: Drumil Patel <[email protected]>

Author: weastel

.github/workflows/grafana_dashboard.yaml

@@ -14,15 +14,15 @@ jobs:
       id: apply_configmap
       uses: docker://prominfra/prombench:master
       env:
-        AUTH_FILE: ${{ secrets.TEST_INFRA_GKE_AUTH }}
-        PROJECT_ID: macro-mile-203600
+        AUTH_FILE: ${{ secrets.TEST_INFRA_PROVIDER_AUTH }}
+        GKE_PROJECT_ID: macro-mile-203600
         CLUSTER_NAME: test-infra
         ZONE: europe-west3-a
         TEST_INFRA_REPO: https://github.com/prometheus/test-infra.git
       with:
         args: >-
           ./prombench gke resource apply -a $AUTH_FILE
-          -v PROJECT_ID:$PROJECT_ID -v ZONE:$ZONE
+          -v GKE_PROJECT_ID:$GKE_PROJECT_ID -v ZONE:$ZONE
           -v CLUSTER_NAME:$CLUSTER_NAME
           -f manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml
     - name: Comment Failure Message

funcbench/Makefile

@@ -1,5 +1,6 @@
 INFRA_CMD        ?= ../infra/infra
 
+PROVIDER 		 ?= gke
 ifeq ($(PACKAGE_PATH),)
 PACKAGE_PATH = ./...
 endif
@@ -11,18 +12,18 @@ deploy: cluster_create resource_apply
 clean: resource_delete cluster_delete
 
 cluster_create:
-	$(INFRA_CMD) gke cluster create -a ${AUTH_FILE} \
-		-v PROJECT_ID:${PROJECT_ID} -v ZONE:${ZONE} -v CLUSTER_NAME:funcbench-${PR_NUMBER} -v PR_NUMBER:${PR_NUMBER} \
-		-f manifests/cluster.yaml
+	$(INFRA_CMD) $(PROVIDER) cluster create -a ${AUTH_FILE} \
+		-v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v ZONE:${ZONE} -v CLUSTER_NAME:funcbench-${PR_NUMBER} -v PR_NUMBER:${PR_NUMBER} \
+		-f manifests/cluster_$(PROVIDER).yaml
 
 cluster_delete:
-	$(INFRA_CMD) gke cluster delete -a ${AUTH_FILE} \
-		-v PROJECT_ID:${PROJECT_ID} -v ZONE:${ZONE} -v CLUSTER_NAME:funcbench-${PR_NUMBER} -v PR_NUMBER:${PR_NUMBER} \
-		-f manifests/cluster.yaml
+	$(INFRA_CMD) $(PROVIDER) cluster delete -a ${AUTH_FILE} \
+		-v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v ZONE:${ZONE} -v CLUSTER_NAME:funcbench-${PR_NUMBER} -v PR_NUMBER:${PR_NUMBER} \
+		-f manifests/cluster_$(PROVIDER).yaml
 
 resource_apply:
-	$(INFRA_CMD) gke resource apply -a ${AUTH_FILE} \
-		-v ZONE:${ZONE} -v PROJECT_ID:${PROJECT_ID} -v CLUSTER_NAME:funcbench-${PR_NUMBER} \
+	$(INFRA_CMD) $(PROVIDER) resource apply -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:funcbench-${PR_NUMBER} \
 		-v PR_NUMBER:${PR_NUMBER} -v GITHUB_TOKEN:${GITHUB_TOKEN} \
 		-v GITHUB_ORG:${GITHUB_ORG} -v GITHUB_REPO:${GITHUB_REPO} \
 		-v BRANCH:${BRANCH} -v 'BENCH_FUNC_REGEX:${BENCH_FUNC_REGEX}' \
@@ -31,8 +32,8 @@ resource_apply:
 
 # Removal of namespace should be at the end, after all other resources get removed.
 resource_delete:
-	$(INFRA_CMD) gke resource delete -a ${AUTH_FILE} \
-		-v ZONE:${ZONE} -v PROJECT_ID:${PROJECT_ID} -v CLUSTER_NAME:funcbench-${PR_NUMBER} \
+	$(INFRA_CMD) $(PROVIDER) resource delete -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:funcbench-${PR_NUMBER} \
 		-v PR_NUMBER:${PR_NUMBER} -v GITHUB_TOKEN:${GITHUB_TOKEN} \
 		-v GITHUB_ORG:${GITHUB_ORG} -v GITHUB_REPO:${GITHUB_REPO} \
 		-v BRANCH:${BRANCH} -v 'BENCH_FUNC_REGEX:${BENCH_FUNC_REGEX}' \

funcbench/manifests/cluster.yaml renamed to funcbench/manifests/cluster_gke.yaml

@@ -1,4 +1,4 @@
-projectid: {{ .PROJECT_ID }}
+projectid: {{ .GKE_PROJECT_ID }}
 zone: {{ .ZONE }}
 cluster:
   name: {{ .CLUSTER_NAME }}

infra/README.md

@@ -38,26 +38,26 @@ Commands:
   gke cluster delete
     gke cluster delete -a service-account.json -f FileOrFolder
 
-  gke nodepool create
-    gke nodepool create -a service-account.json -f FileOrFolder
+  gke nodes create
+    gke nodes create -a service-account.json -f FileOrFolder
 
-  gke nodepool delete
-    gke nodepool delete -a service-account.json -f FileOrFolder
+  gke nodes delete
+    gke nodes delete -a service-account.json -f FileOrFolder
 
-  gke nodepool check-running
-    gke nodepool check-running -a service-account.json -f FileOrFolder
+  gke nodes check-running
+    gke nodes check-running -a service-account.json -f FileOrFolder
 
-  gke nodepool check-deleted
-    gke nodepool check-deleted -a service-account.json -f FileOrFolder
+  gke nodes check-deleted
+    gke nodes check-deleted -a service-account.json -f FileOrFolder
 
   gke resource apply
     gke resource apply -a service-account.json -f manifestsFileOrFolder -v
-    PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v
+    GKE_PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v
     hashStable:COMMIT1 -v hashTesting:COMMIT2
 
   gke resource delete
     gke resource delete -a service-account.json -f manifestsFileOrFolder -v
-    PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v
+    GKE_PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v
     hashStable:COMMIT1 -v hashTesting:COMMIT2
 
 

infra/infra.go

@@ -60,26 +60,26 @@ func main() {
 		Action(g.ClusterDelete)
 
 	// Cluster node-pool operations
-	k8sGKENodePool := k8sGKE.Command("nodepool", "manage GKE clusters nodepools").
+	k8sGKENodePool := k8sGKE.Command("nodes", "manage GKE clusters nodepools").
 		Action(g.NewGKEClient).
 		Action(g.GKEDeploymentsParse)
-	k8sGKENodePool.Command("create", "gke nodepool create -a service-account.json -f FileOrFolder").
+	k8sGKENodePool.Command("create", "gke nodes create -a service-account.json -f FileOrFolder").
 		Action(g.NodePoolCreate)
-	k8sGKENodePool.Command("delete", "gke nodepool delete -a service-account.json -f FileOrFolder").
+	k8sGKENodePool.Command("delete", "gke nodes delete -a service-account.json -f FileOrFolder").
 		Action(g.NodePoolDelete)
-	k8sGKENodePool.Command("check-running", "gke nodepool check-running -a service-account.json -f FileOrFolder").
+	k8sGKENodePool.Command("check-running", "gke nodes check-running -a service-account.json -f FileOrFolder").
 		Action(g.AllNodepoolsRunning)
-	k8sGKENodePool.Command("check-deleted", "gke nodepool check-deleted -a service-account.json -f FileOrFolder").
+	k8sGKENodePool.Command("check-deleted", "gke nodes check-deleted -a service-account.json -f FileOrFolder").
 		Action(g.AllNodepoolsDeleted)
 
 	// K8s resource operations.
-	k8sGKEResource := k8sGKE.Command("resource", `Apply and delete different k8s resources - deployments, services, config maps etc.Required variables -v PROJECT_ID, -v ZONE: -west1-b -v CLUSTER_NAME`).
+	k8sGKEResource := k8sGKE.Command("resource", `Apply and delete different k8s resources - deployments, services, config maps etc.Required variables -v GKE_PROJECT_ID, -v ZONE: -west1-b -v CLUSTER_NAME`).
 		Action(g.NewGKEClient).
 		Action(g.K8SDeploymentsParse).
 		Action(g.NewK8sProvider)
-	k8sGKEResource.Command("apply", "gke resource apply -a service-account.json -f manifestsFileOrFolder -v PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
+	k8sGKEResource.Command("apply", "gke resource apply -a service-account.json -f manifestsFileOrFolder -v GKE_PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
 		Action(g.ResourceApply)
-	k8sGKEResource.Command("delete", "gke resource delete -a service-account.json -f manifestsFileOrFolder -v PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
+	k8sGKEResource.Command("delete", "gke resource delete -a service-account.json -f manifestsFileOrFolder -v GKE_PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
 		Action(g.ResourceDelete)
 
 	if _, err := app.Parse(os.Args[1:]); err != nil {

pkg/provider/gke/gke.go

@@ -196,7 +196,7 @@ func (c *GKE) K8SDeploymentsParse(*kingpin.ParseContext) error {
 
 // checkDeploymentVarsAndFiles checks whether the requied deployment vars are passed.
 func (c *GKE) checkDeploymentVarsAndFiles() error {
-	reqDepVars := []string{"PROJECT_ID", "ZONE", "CLUSTER_NAME"}
+	reqDepVars := []string{"GKE_PROJECT_ID", "ZONE", "CLUSTER_NAME"}
 	for _, k := range reqDepVars {
 		if v, ok := c.DeploymentVars[k]; !ok || v == "" {
 			return fmt.Errorf("missing required %v variable", k)
@@ -518,7 +518,7 @@ func (c *GKE) AllNodepoolsDeleted(*kingpin.ParseContext) error {
 func (c *GKE) NewK8sProvider(*kingpin.ParseContext) error {
 	// Get the authentication certificate for the cluster using the GKE client.
 	req := &containerpb.GetClusterRequest{
-		ProjectId: c.DeploymentVars["PROJECT_ID"],
+		ProjectId: c.DeploymentVars["GKE_PROJECT_ID"],
 		Zone:      c.DeploymentVars["ZONE"],
 		ClusterId: c.DeploymentVars["CLUSTER_NAME"],
 	}

prombench/Makefile

@@ -1,43 +1,45 @@
 INFRA_CMD        ?= ../infra/infra
 
+PROVIDER 		 ?= gke
+
 .PHONY: deploy clean
 deploy: nodepool_create resource_apply
 # GCP sometimes takes longer than 30 tries when trying to delete nodepools
 # if k8s resources are not already cleared
 clean: resource_delete nodepool_delete
 
 nodepool_create:
-	$(INFRA_CMD) gke nodepool create -a ${AUTH_FILE} \
-		-v ZONE:${ZONE} -v PROJECT_ID:${PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
-		-f manifests/prombench/nodepools.yaml
+	$(INFRA_CMD) $(PROVIDER) nodes create -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+		-f manifests/prombench/nodes_$(PROVIDER).yaml
 
 resource_apply:
-	$(INFRA_CMD) gke resource apply -a ${AUTH_FILE} \
-		-v ZONE:${ZONE} -v PROJECT_ID:${PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} \
+	$(INFRA_CMD) $(PROVIDER) resource apply -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} \
 		-v PR_NUMBER:${PR_NUMBER} -v RELEASE:${RELEASE} -v DOMAIN_NAME:${DOMAIN_NAME} \
 		-v GITHUB_ORG:${GITHUB_ORG} -v GITHUB_REPO:${GITHUB_REPO} \
 		-f manifests/prombench/benchmark
 
 # Required because namespace and cluster-role are not part of the created nodepools
 resource_delete:
-	$(INFRA_CMD) gke resource delete -a ${AUTH_FILE} \
-		-v ZONE:${ZONE} -v PROJECT_ID:${PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+	$(INFRA_CMD) $(PROVIDER) resource delete -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
 		-f manifests/prombench/benchmark/1c_cluster-role-binding.yaml \
 		-f manifests/prombench/benchmark/1a_namespace.yaml
 
 nodepool_delete:
-	$(INFRA_CMD) gke nodepool delete -a ${AUTH_FILE} \
-		-v ZONE:${ZONE} -v PROJECT_ID:${PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
-		-f manifests/prombench/nodepools.yaml
+	$(INFRA_CMD) $(PROVIDER) nodes delete -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} -v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+		-f manifests/prombench/nodes_$(PROVIDER).yaml
 
 all_nodepools_running:
-	$(INFRA_CMD) gke nodepool check-running -a ${AUTH_FILE} \
-		-v ZONE:${ZONE} -v PROJECT_ID:${PROJECT_ID} \
+	$(INFRA_CMD) $(PROVIDER) nodes check-running -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
 		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
-		-f manifests/prombench/nodepools.yaml
+		-f manifests/prombench/nodes_$(PROVIDER).yaml
 
 all_nodepools_deleted:
-	$(INFRA_CMD) gke nodepool check-deleted -a ${AUTH_FILE} \
-		-v ZONE:${ZONE} -v PROJECT_ID:${PROJECT_ID} \
+	$(INFRA_CMD) $(PROVIDER) nodes check-deleted -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
 		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
-		-f manifests/prombench/nodepools.yaml
+		-f manifests/prombench/nodes_$(PROVIDER).yaml

prombench/README.md

@@ -9,90 +9,27 @@ It is designed to support adding more k8s providers.
 
 The `/manifest` directory contains all the kubernetes manifest files.
 
-- `cluster.yaml` : This is used to create the Main Node.
+- `cluster_gke.yaml` : This is used to create the Main Node in gke.
 - `cluster-infra/` : These are the persistent components of the Main Node.
 - `prombench/` : These resources are created and destroyed for each prombench test.
 
-## Setup prombench
+## Setup and run prombench
 
-1. [Create the main node](#create-the-main-node)
-2. [Deploy monitoring components](#deploy-monitoring-components)
-3. [Setup GitHub Actions](#setup-github-actions)
+Prombench can be run on various providers, following are the provider specific instructions:
+    
+- Instructions for [Google Kubernetes Engine](docs/gke.md)
 
-### Create the Main Node
-
----
-
-- Create a new project on Google Cloud.
-- Create a [Service Account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) on GKE with role `Kubernetes Engine Service Agent` & `Kubernetes Engine Admin`. If using gcloud cli add the [`roles/container.admin`](https://cloud.google.com/kubernetes-engine/docs/how-to/iam#kubernetes-engine-roles) and [`roles/iam.serviceAccountUser`](https://cloud.google.com/kubernetes-engine/docs/how-to/iam#service_account_user) roles to the GCP serviceAccount and download the json file.
-
-- Set the following environment variables and deploy the cluster.
-
-```
-export PROJECT_ID=<google-cloud project-id>
-export CLUSTER_NAME=prombench
-export ZONE=us-east1-b
-export AUTH_FILE=<path to service-account.json>
-
-../infra/infra gke cluster create -a $AUTH_FILE -v PROJECT_ID:$PROJECT_ID \
-    -v ZONE:$ZONE -v CLUSTER_NAME:$CLUSTER_NAME -f manifests/cluster.yaml
-```
-
-### Deploy monitoring components
-
-> Collecting, monitoring and displaying the test results and logs
-
----
-
-- [Optional] If used with the Github integration generate a GitHub auth token.
-  - Login with the [Prombot account](https://github.com/prombot) and generate a [new auth token](https://github.com/settings/tokens).
-  - With permissions: `public_repo`, `read:org`, `write:discussion`.
-
-```
-export SERVICEACCOUNT_CLIENT_EMAIL=<client-email present in service-account.json>
-export GRAFANA_ADMIN_PASSWORD=password
-export DOMAIN_NAME=prombench.prometheus.io // Can be set to any other custom domain or an empty string when not used with the Github integration.
-export OAUTH_TOKEN=<generated token from github or set to an empty string " ">
-export WH_SECRET=<github webhook secret>
-export GITHUB_ORG=prometheus
-export GITHUB_REPO=prometheus
-```
-
-- Deploy the [nginx-ingress-controller](https://github.com/kubernetes/ingress-nginx), Prometheus-Meta, Loki, Grafana, Alertmanager & Github Notifier.
-
-```
-../infra/infra gke resource apply -a $AUTH_FILE -v PROJECT_ID:$PROJECT_ID -v ZONE:$ZONE \
-    -v CLUSTER_NAME:$CLUSTER_NAME -v DOMAIN_NAME:$DOMAIN_NAME \
-    -v GRAFANA_ADMIN_PASSWORD:$GRAFANA_ADMIN_PASSWORD \
-    -v SERVICEACCOUNT_CLIENT_EMAIL:$SERVICEACCOUNT_CLIENT_EMAIL \
-    -v OAUTH_TOKEN="$(printf $OAUTH_TOKEN | base64 -w 0)" \
-    -v WH_SECRET="$(printf $WH_SECRET | base64 -w 0)" \
-    -v GITHUB_ORG:$GITHUB_ORG -v GITHUB_REPO:$GITHUB_REPO \
-    -f manifests/cluster-infra
-```
-
-> Note: Use `-v GKE_AUTH="$(echo $AUTH_FILE | base64 -w 0)"` if you're passing the data directly into `$AUTH_FILE`
-
-- The output will show the ingress IP which will be used to point the domain name to. Alternatively you can see it from the GKE/Services tab.
-- Set the `A record` for `<DOMAIN_NAME>` to point to `nginx-ingress-controller` IP address.
-- The services will be accessible at:
-  - Grafana :: `http://<DOMAIN_NAME>/grafana`
-  - Prometheus :: `http://<DOMAIN_NAME>/prometheus-meta`
-  - Logs :: `http://<DOMAIN_NAME>/grafana/explore`
-
-### Setup GitHub Actions
+## Setup GitHub Actions
 
 Place a workflow file in the `.github` directory of the repository.
 See the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository for an example.
 
-Create a github action `TEST_INFRA_GKE_AUTH` secret with the base64 encoded content of the `service-account.json` file.
+Create a github action `TEST_INFRA_PROVIDER_AUTH` secret with the base64 encoded content of the `AUTH_FILE`.
 
 ```
 cat $AUTH_FILE | base64 -w 0
 ```
 
-## Usage
-
 ### Trigger tests via a Github comment.
 <!-- If you change the heading, also change the anchor in the comment monitor config map. -->
 
@@ -113,35 +50,6 @@ cat $AUTH_FILE | base64 -w 0
 
 - `/prombench cancel`
 
-### Start a benchmarking test manually
-
----
-
-- Set the following environment variables.
-
-```
-export RELEASE=<master or any prometheus release(ex: v2.3.0) >
-export PR_NUMBER=<PR to benchmark against the selected $RELEASE>
-```
-
-- Create the nodepools for the k8s objects
-
-```
-../infra/infra gke nodepool create -a $AUTH_FILE \
-    -v ZONE:$ZONE -v PROJECT_ID:$PROJECT_ID -v CLUSTER_NAME:$CLUSTER_NAME \
-    -v PR_NUMBER:$PR_NUMBER -f manifests/prombench/nodepools.yaml
-```
-
-- Deploy the k8s objects
-
-```
-../infra/infra gke resource apply -a $AUTH_FILE \
-    -v ZONE:$ZONE -v PROJECT_ID:$PROJECT_ID -v CLUSTER_NAME:$CLUSTER_NAME \
-    -v PR_NUMBER:$PR_NUMBER -v RELEASE:$RELEASE -v DOMAIN_NAME:$DOMAIN_NAME \
-    -v GITHUB_ORG:${GITHUB_ORG} -v GITHUB_REPO:${GITHUB_REPO} \
-    -f manifests/prombench/benchmark
-```
-
 ### Building Docker Image
 
 ```