fix: adjust scale test execution code

Signed-off-by: Alex Castilio dos Santos <[email protected]>

Author: alexcastilio

.github/workflows/scale-test.yaml

@@ -11,10 +11,9 @@ on:
         description: "AKS Cluster Name"
         required: true
         type: string
-      location:
-        description: "Azure Location"
+      image_tag:
+        description: "Image Tag (if not set, default for this commit will be used)"
         type: string
-        default: ${{ vars.AZURE_LOCATION }}
       num_deployments:
         description: "Number of Traffic Deployments"
         default: 1000
@@ -27,20 +26,8 @@ on:
         description: "Number of Network Policies"
         default: 1000
         type: number
-      num_unique_labels_per_pod: 
-        description: "Number of Unique Labels per Pod"
-        default: 2
-        type: number
-      num_unique_labels_per_deployment:
-        description: "Number of Unique Labels per Deployment"
-        default: 2
-        type: number
-      num_shared_labels_per_pod:
-        description: "Number of Shared Labels per Pod"
-        default: 3
-        type: number
-      delete_labels: 
-        description: "Delete Labels"
+      cleanup: 
+        description: "Clean up environment after test"
         default: true
         type: boolean
 
@@ -54,10 +41,6 @@ on:
         description: "AKS Cluster Name"
         required: true
         type: string
-      location:
-        description: "Azure Location"
-        type: string
-        default: ${{ vars.AZURE_LOCATION }}
       num_deployments:
         description: "Number of Traffic Deployments"
         default: 1000
@@ -70,20 +53,8 @@ on:
         description: "Number of Network Policies"
         default: 1000
         type: number
-      num_unique_labels_per_pod: 
-        description: "Number of Unique Labels per Pod"
-        default: 2
-        type: number
-      num_unique_labels_per_deployment:
-        description: "Number of Unique Labels per Deployment"
-        default: 2
-        type: number
-      num_shared_labels_per_pod:
-        description: "Number of Shared Labels per Pod"
-        default: 3
-        type: number
-      delete_labels: 
-        description: "Delete Labels"
+      cleanup: 
+        description: "Clean up environment after test"
         default: true
         type: boolean
 
@@ -116,17 +87,16 @@ jobs:
       - name: Run Scale Test
         env:
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION }}
-          AZURE_LOCATION: ${{ input.location }}
           AZURE_RESOURCE_GROUP: ${{ inputs.resource_group }}
           CLUSTER_NAME: ${{ inputs.cluster_name }}
           NUM_DEPLOYMENTS: ${{ inputs.num_deployments }}
           NUM_REPLICAS: ${{ inputs.num_replicas }}
           NUM_NETPOLS: ${{ inputs.num_netpol }}
-          NUM_UNIQUE_LABELS_PER_POD: ${{ inputs.num_unique_labels_per_pod }}
-          NUM_SHARED_LABELS_PER_POD: ${{ inputs.num_shared_labels_per_pod }}
-          NUM_UNIQUE_LABELS_PER_DEPLOYMENT: ${{ inputs.num_unique_labels_per_deployment }}
-          DELETE_LABELS: ${{ inputs.delete_labels }}
+          CLEANUP: ${{ inputs.cleanup }}
+          IMAGE_REGISTRY: ${{ vars.ACR_NAME }}
+          IMAGE_NAMESPACE: ${{ github.repository }}
+          TAG: ${{ inputs.image_tag }}
         shell: bash
         run: |
           set -euo pipefail
-          go test -v ./test/e2e/. -timeout 300m -tags=scale -count=1  -args -image-tag=$(make version) -image-registry=${{vars.ACR_NAME}} -image-namespace=${{github.repository}}
+          go test -v ./test/e2e/. -timeout 300m -tags=scale -count=1  -args -image-tag=$( [[ $TAG == "" ]] && make version || echo $TAG ) -create-infra=false -delete-infra=false 

test/e2e/framework/kubernetes/install-retina-helm.go

@@ -19,7 +19,7 @@ import (
 
 const (
 	createTimeout = 20 * time.Minute // windows is slow
-	deleteTimeout = 60 * time.Second
+	deleteTimeout = 5 * time.Minute
 )
 
 var (

test/e2e/framework/kubernetes/uninstall-helm.go

@@ -0,0 +1,45 @@
+package kubernetes
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/cli"
+)
+
+type UninstallHelmChart struct {
+	Namespace          string
+	ReleaseName        string
+	KubeConfigFilePath string
+}
+
+func (i *UninstallHelmChart) Run() error {
+	settings := cli.New()
+	settings.KubeConfig = i.KubeConfigFilePath
+	actionConfig := new(action.Configuration)
+
+	err := actionConfig.Init(settings.RESTClientGetter(), i.Namespace, os.Getenv("HELM_DRIVER"), log.Printf)
+	if err != nil {
+		return fmt.Errorf("failed to initialize helm action config: %w", err)
+	}
+
+	delclient := action.NewUninstall(actionConfig)
+	delclient.Wait = true
+	delclient.Timeout = deleteTimeout
+	_, err = delclient.Run(i.ReleaseName)
+	if err != nil {
+		return fmt.Errorf("failed to delete existing release %s: %w", i.ReleaseName, err)
+	}
+
+	return nil
+}
+
+func (i *UninstallHelmChart) Prevalidate() error {
+	return nil
+}
+
+func (i *UninstallHelmChart) Stop() error {
+	return nil
+}

test/e2e/framework/scaletest/create-resources.go

@@ -85,7 +85,9 @@ func (c *CreateResources) getResources() []runtime.Object {
 
 	kapingerClusterRoleBinding := kapinger.GetKapingerClusterRoleBinding()
 
-	objs = append(objs, kapingerClusterRole, kapingerClusterRoleBinding)
+	kapingerSA := kapinger.GetKapingerServiceAccount()
+
+	objs = append(objs, kapingerClusterRole, kapingerClusterRoleBinding, kapingerSA)
 	// c.generateKwokNodes()
 	log.Println("Finished generating YAMLs")
 	return objs
@@ -101,6 +103,9 @@ func (c *CreateResources) generateDeployments() []runtime.Object {
 	}
 	template := kapinger.GetKapingerDeployment()
 
+	if template.Labels == nil {
+		template.Labels = make(map[string]string)
+	}
 	template.Labels["is-real"] = "true"
 	template.Spec.Template.Labels["is-real"] = "true"
 	template.Spec.Template.Spec.NodeSelector["scale-test"] = "true"

test/e2e/framework/scaletest/templates/networkpolicy.go

@@ -19,6 +19,9 @@ var (
 				"Ingress",
 				"Egress",
 			},
+			PodSelector: metav1.LabelSelector{
+				MatchLabels: map[string]string{},
+			},
 		},
 	}
 )

test/e2e/jobs/jobs.go

@@ -89,6 +89,18 @@ func InstallRetina(kubeConfigFilePath, chartPath string) *types.Job {
 	return job
 }
 
+func UninstallRetina(kubeConfigFilePath, chartPath string) *types.Job {
+	job := types.NewJob("Uninstall Retina")
+
+	job.AddStep(&kubernetes.UninstallHelmChart{
+		Namespace:          common.KubeSystemNamespace,
+		ReleaseName:        "retina",
+		KubeConfigFilePath: kubeConfigFilePath,
+	}, nil)
+
+	return job
+}
+
 func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string, testPodNamespace string) *types.Job {
 	job := types.NewJob("Install and test Retina with basic metrics")
 

test/e2e/jobs/scale.go

@@ -15,14 +15,14 @@ func DefaultScaleTestOptions() scaletest.Options {
 		NumKwokDeployments:            0,
 		NumKwokReplicas:               0,
 		MaxRealPodsPerNode:            100,
-		NumRealDeployments:            3,
-		RealPodType:                   "agnhost",
-		NumRealReplicas:               2,
-		NumRealServices:               1,
+		NumRealDeployments:            1000,
+		RealPodType:                   "kapinger",
+		NumRealReplicas:               40,
+		NumRealServices:               1000,
 		NumNetworkPolicies:            10,
 		NumUnapliedNetworkPolicies:    10,
-		NumUniqueLabelsPerPod:         2,
-		NumUniqueLabelsPerDeployment:  2,
+		NumUniqueLabelsPerPod:         0,
+		NumUniqueLabelsPerDeployment:  1,
 		NumSharedLabelsPerPod:         3,
 		KubeconfigPath:                "",
 		RestartNpmPods:                false,
@@ -95,6 +95,8 @@ func ScaleTest(opt *scaletest.Options) *types.Job {
 		NumSharedLabelsPerPod: opt.NumSharedLabelsPerPod,
 	}, nil)
 
+	// job.AddStep(&kubernetes.DeleteNamespace{}, nil)
+
 	// TODO: Add steps to get the state of the cluster
 
 	// job.AddStep(&kubernetes.GetDeployment{})

test/e2e/retina_e2e_test.go

@@ -1,3 +1,5 @@
+//go:build e2e
+
 package retina
 
 import (
@@ -18,12 +20,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var (
-	locations   = []string{"eastus2", "centralus", "southcentralus", "uksouth", "centralindia", "westus2"}
-	createInfra = flag.Bool("create-infra", true, "create a Resource group, vNET and AKS cluster for testing")
-	deleteInfra = flag.Bool("delete-infra", true, "delete a Resource group, vNET and AKS cluster for testing")
-)
-
 // TestE2ERetina tests all e2e scenarios for retina
 func TestE2ERetina(t *testing.T) {
 	ctx, cancel := helpers.Context(t)

test/e2e/scale_test.go

@@ -4,6 +4,7 @@ package retina
 
 import (
 	"crypto/rand"
+	"flag"
 	"math/big"
 	"os"
 	"os/user"
@@ -26,7 +27,19 @@ func TestE2ERetina_Scale(t *testing.T) {
 	curuser, err := user.Current()
 	require.NoError(t, err)
 
-	clusterName := curuser.Username + common.NetObsRGtag + strconv.FormatInt(time.Now().Unix(), 10)
+	flag.Parse()
+
+	clusterName := os.Getenv("CLUSTER_NAME")
+	if clusterName == "" {
+		username := curuser.Username
+		// Truncate the username to 8 characters
+		if len(username) > 8 {
+			username = username[:8]
+			t.Logf("Username is too long, truncating to 8 characters: %s", username)
+		}
+		clusterName = username + common.NetObsRGtag + strconv.FormatInt(time.Now().Unix(), 10)
+		t.Logf("CLUSTER_NAME is not set, generating a random cluster name: %s", clusterName)
+	}
 
 	subID := os.Getenv("AZURE_SUBSCRIPTION_ID")
 	require.NotEmpty(t, subID)
@@ -70,11 +83,36 @@ func TestE2ERetina_Scale(t *testing.T) {
 	installRetina := types.NewRunner(t, jobs.InstallRetina(kubeConfigFilePath, chartPath))
 	installRetina.Run(ctx)
 
+	t.Cleanup(func() {
+		_ = jobs.UninstallRetina(kubeConfigFilePath, chartPath).Run()
+	})
+
 	// Scale test
 	opt := jobs.DefaultScaleTestOptions()
 	opt.KubeconfigPath = kubeConfigFilePath
-	opt.RealPodType = "kapinger"
-	opt.DeleteLabels = true
+
+	NumDeployments := os.Getenv("NUM_DEPLOYMENTS")
+	NumReplicas := os.Getenv("NUM_REPLICAS")
+	NumNetworkPolicies := os.Getenv("NUM_NET_POL")
+	CleanUp := os.Getenv("CLEANUP")
+
+	if NumDeployments != "" {
+		opt.NumRealDeployments, err = strconv.Atoi(NumDeployments)
+		opt.NumRealServices = opt.NumRealDeployments
+		require.NoError(t, err)
+	}
+	if NumReplicas != "" {
+		opt.NumRealReplicas, err = strconv.Atoi(NumReplicas)
+		require.NoError(t, err)
+	}
+	if NumNetworkPolicies != "" {
+		opt.NumNetworkPolicies, err = strconv.Atoi(NumNetworkPolicies)
+		require.NoError(t, err)
+	}
+	if CleanUp != "" {
+		opt.DeleteLabels, err = strconv.ParseBool(CleanUp)
+		require.NoError(t, err)
+	}
 
 	scale := types.NewRunner(t, jobs.ScaleTest(&opt))
 	scale.Run(ctx)

test/e2e/shared_vars.go

@@ -0,0 +1,11 @@
+//go:build scale || e2e
+
+package retina
+
+import "flag"
+
+var (
+	locations   = []string{"eastus2", "centralus", "southcentralus", "uksouth", "centralindia", "westus2"}
+	createInfra = flag.Bool("create-infra", true, "create a Resource group, vNET and AKS cluster for testing")
+	deleteInfra = flag.Bool("delete-infra", true, "delete a Resource group, vNET and AKS cluster for testing")
+)