Merge pull request #1553 from Nordix/mquhuy/add-e2e-upgrade-cleanup

🌱 Add ironic and bmo cleanup for upgrade BMO E2E tests
metal3-io · Feb 20, 2024 · 3cd8e38 · 3cd8e38
2 parents 23786ed + 0309d9d
commit 3cd8e38
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 21 deletions.
diff --git a/test/e2e/common.go b/test/e2e/common.go
@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"os"
@@ -21,6 +22,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
+	testexec "sigs.k8s.io/cluster-api/test/framework/exec"
 
 	capm3_e2e "github.com/metal3-io/cluster-api-provider-metal3/test/e2e"
 
@@ -409,10 +411,10 @@ func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.Au
 	Expect(isExpectedBootMode).To(BeTrue(), fmt.Sprintf("Expected booting from %s, but found different mode", expectedBootMode))
 }
 
-// BuildAndApplyKustomizeInput provides input for BuildAndApplyKustomize().
+// BuildAndApplyKustomizationInput provides input for BuildAndApplyKustomize().
 // If WaitForDeployment and/or WatchDeploymentLogs is set to true, then DeploymentName
 // and DeploymentNamespace are expected.
-type BuildAndApplyKustomizeInput struct {
+type BuildAndApplyKustomizationInput struct {
 	// Path to the kustomization to build
 	Kustomization string
 
@@ -437,7 +439,7 @@ type BuildAndApplyKustomizeInput struct {
 	WaitIntervals []interface{}
 }
 
-func (input *BuildAndApplyKustomizeInput) validate() error {
+func (input *BuildAndApplyKustomizationInput) validate() error {
 	// If neither WaitForDeployment nor WatchDeploymentLogs is true, we don't need to validate the input
 	if !input.WaitForDeployment && !input.WatchDeploymentLogs {
 		return nil
@@ -454,9 +456,9 @@ func (input *BuildAndApplyKustomizeInput) validate() error {
 	return nil
 }
 
-// BuildAndApplyKustomize takes input from BuildAndApplyKustomizeInput. It builds the provided kustomization
+// BuildAndApplyKustomization takes input from BuildAndApplyKustomizationInput. It builds the provided kustomization
 // and apply it to the cluster provided by clusterProxy.
-func BuildAndApplyKustomize(ctx context.Context, input *BuildAndApplyKustomizeInput) error {
+func BuildAndApplyKustomization(ctx context.Context, input *BuildAndApplyKustomizationInput) error {
 	Expect(input.validate()).To(BeNil())
 	var err error
 	kustomization := input.Kustomization
@@ -465,6 +467,7 @@ func BuildAndApplyKustomize(ctx context.Context, input *BuildAndApplyKustomizeIn
 	if err != nil {
 		return err
 	}
+
 	err = clusterProxy.Apply(ctx, manifest)
 	if err != nil {
 		return err
@@ -518,3 +521,30 @@ func DeploymentRolledOut(ctx context.Context, clusterProxy framework.ClusterProx
 	}
 	return false
 }
+
+// KubectlDelete shells out to kubectl delete.
+func KubectlDelete(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) error {
+	aargs := append([]string{"delete", "--kubeconfig", kubeconfigPath, "-f", "-"}, args...)
+	rbytes := bytes.NewReader(resources)
+	deleteCmd := testexec.NewCommand(
+		testexec.WithCommand("kubectl"),
+		testexec.WithArgs(aargs...),
+		testexec.WithStdin(rbytes),
+	)
+
+	fmt.Printf("Running kubectl %s\n", strings.Join(aargs, " "))
+	stdout, stderr, err := deleteCmd.Run(ctx)
+	fmt.Printf("stderr:\n%s\n", string(stderr))
+	fmt.Printf("stdout:\n%s\n", string(stdout))
+	return err
+}
+
+// BuildAndRemoveKustomization builds the provided kustomization to resources and removes them from the cluster
+// provided by clusterProxy.
+func BuildAndRemoveKustomization(ctx context.Context, kustomization string, clusterProxy framework.ClusterProxy) error {
+	manifest, err := buildKustomizeManifest(kustomization)
+	if err != nil {
+		return err
+	}
+	return KubectlDelete(ctx, clusterProxy.GetKubeconfigPath(), manifest)
+}
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -129,7 +129,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	if e2eConfig.GetVariable("DEPLOY_IRONIC") != "false" {
 		// Install Ironic
 		By("Installing Ironic")
-		err := BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{
+		err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
 			Kustomization:       e2eConfig.GetVariable("IRONIC_KUSTOMIZATION"),
 			ClusterProxy:        clusterProxy,
 			WaitForDeployment:   true,
@@ -146,7 +146,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	if e2eConfig.GetVariable("DEPLOY_BMO") != "false" {
 		// Install BMO
 		By("Installing BMO")
-		err := BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{
+		err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
 			Kustomization:       e2eConfig.GetVariable("BMO_KUSTOMIZATION"),
 			ClusterProxy:        clusterProxy,
 			WaitForDeployment:   true,

diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go
@@ -175,13 +175,12 @@ var _ = Describe("BMO Upgrade", func() {
 		specName               = "upgrade"
 		secretName             = "bmc-credentials"
 		namespace              *corev1.Namespace
-		bmoIronicNamespace     string
+		bmoIronicNamespace     = "baremetal-operator-system"
 		upgradeClusterProvider bootstrap.ClusterProvider
 		upgradeClusterProxy    framework.ClusterProxy
 		bmh                    metal3api.BareMetalHost
 	)
 	BeforeEach(func() {
-		bmoIronicNamespace = "baremetal-operator-system"
 		var kubeconfigPath string
 
 		if useExistingCluster {
@@ -203,6 +202,7 @@ var _ = Describe("BMO Upgrade", func() {
 		framework.TryAddDefaultSchemes(scheme)
 		metal3api.AddToScheme(scheme)
 		upgradeClusterProxy = framework.NewClusterProxy("bmo-e2e-upgrade", kubeconfigPath, scheme)
+
 		if e2eConfig.GetVariable("UPGRADE_DEPLOY_CERT_MANAGER") != "false" {
 			By("Installing cert-manager on the upgrade cluster")
 			cmVersion := e2eConfig.GetVariable("CERT_MANAGER_VERSION")
@@ -215,12 +215,15 @@ var _ = Describe("BMO Upgrade", func() {
 			err = checkCertManagerAPI(upgradeClusterProxy)
 			Expect(err).NotTo(HaveOccurred())
 		}
+	})
 
+	It("Should upgrade BMO to latest version", func() {
 		if e2eConfig.GetVariable("UPGRADE_DEPLOY_IRONIC") != "false" {
 			// Install Ironic
+			ironicKustomization := e2eConfig.GetVariable("IRONIC_KUSTOMIZATION")
 			By("Installing Ironic on the upgrade cluster")
-			err := BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{
-				Kustomization:       e2eConfig.GetVariable("IRONIC_KUSTOMIZATION"),
+			err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
+				Kustomization:       ironicKustomization,
 				ClusterProxy:        upgradeClusterProxy,
 				WaitForDeployment:   true,
 				WatchDeploymentLogs: true,
@@ -230,23 +233,31 @@ var _ = Describe("BMO Upgrade", func() {
 				WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
 			})
 			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(func() {
+				By("Removing Ironic on the upgrade cluster")
+				BuildAndRemoveKustomization(ctx, ironicKustomization, upgradeClusterProxy)
+			})
 		}
 
 		if e2eConfig.GetVariable("UPGRADE_DEPLOY_BMO") != "false" {
-			bmoKustomization := e2eConfig.GetVariable("UPGRADE_BMO_KUSTOMIZATION_FROM")
-			bmoKustomizationName := filepath.Base(bmoKustomization)
-			By(fmt.Sprintf("Installing BMO from %s on the upgrade cluster", bmoKustomization))
-			err := BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{
-				Kustomization:       bmoKustomization,
+			bmoFromKustomization := e2eConfig.GetVariable("UPGRADE_BMO_KUSTOMIZATION_FROM")
+			bmoFromKustomizationName := filepath.Base(bmoFromKustomization)
+			By(fmt.Sprintf("Installing BMO from %s on the upgrade cluster", bmoFromKustomization))
+			err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
+				Kustomization:       bmoFromKustomization,
 				ClusterProxy:        upgradeClusterProxy,
 				WaitForDeployment:   true,
 				WatchDeploymentLogs: true,
 				DeploymentName:      "baremetal-operator-controller-manager",
 				DeploymentNamespace: bmoIronicNamespace,
-				LogPath:             filepath.Join(artifactFolder, "logs", fmt.Sprintf("%s-%s", bmoIronicNamespace, specName), fmt.Sprintf("bmo-%s", bmoKustomizationName)),
+				LogPath:             filepath.Join(artifactFolder, "logs", fmt.Sprintf("%s-%s", bmoIronicNamespace, specName), fmt.Sprintf("bmo-%s", bmoFromKustomizationName)),
 				WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
 			})
 			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(func() {
+				By(fmt.Sprintf("Removing BMO from %s on the upgrade cluster", bmoFromKustomization))
+				BuildAndRemoveKustomization(ctx, bmoFromKustomization, upgradeClusterProxy)
+			})
 		}
 
 		namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
@@ -255,9 +266,6 @@ var _ = Describe("BMO Upgrade", func() {
 			Name:      fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
 			LogFolder: artifactFolder,
 		})
-	})
-
-	It("Should upgrade BMO to latest version", func() {
 		By("Creating a secret with BMH credentials")
 		bmcCredentialsData := map[string]string{
 			"username": bmc.User,
@@ -304,7 +312,7 @@ var _ = Describe("BMO Upgrade", func() {
 		Expect(err).NotTo(HaveOccurred())
 		bmoKustomization := e2eConfig.GetVariable("BMO_KUSTOMIZATION")
 		bmoKustomizationName := filepath.Base(bmoKustomization)
-		err = BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{
+		err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
 			Kustomization:       bmoKustomization,
 			ClusterProxy:        upgradeClusterProxy,
 			WaitForDeployment:   false,
@@ -314,6 +322,10 @@ var _ = Describe("BMO Upgrade", func() {
 			LogPath:             filepath.Join(artifactFolder, "logs", fmt.Sprintf("%s-%s", bmoIronicNamespace, specName), fmt.Sprintf("bmo-%s", bmoKustomizationName)),
 			WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
 		})
+		DeferCleanup(func() {
+			By("Removing BMO main e2e deployment")
+			BuildAndRemoveKustomization(ctx, bmoKustomization, upgradeClusterProxy)
+		})
 		Expect(err).NotTo(HaveOccurred())
 		By("Waiting for BMO update to rollout")
 		Eventually(func() bool {