Add e2e test that deploys two Cass clusters and checks that Clusters …

Makefile

@@ -86,6 +86,7 @@ init-test-overlays:
 PHONY: e2e-test
 e2e-test:
 	@echo Running e2e tests
+	cd test/config/deploy_reaper_test && $(KUSTOMIZE) edit set image controller=${REV_IMAGE} && cd -
 	go test -timeout 1800s ./test/e2e/... -ginkgo.v -ginkgo.progress -test.v
 
 # Generate code

test/config/cassdc/kustomization.yaml

@@ -1,3 +1,5 @@
+namespace: deploy-reaper-test
+
 resources:
 - cassdc.yaml
 

test/config/deploy_reaper_test/kustomization.yaml

@@ -1,2 +1,8 @@
 resources:
 - base
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: controller
+  newName: docker.io/k8ssandra/reaper-operator
+  newTag: 2cfebf759df9

test/config/deploy_reaper_test/overlays/first-cluster/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - ../../../cassdc
+namePrefix: primary-

test/config/deploy_reaper_test/overlays/second-cluster/cassdc-size-patch.yaml

@@ -0,0 +1,8 @@
+apiVersion: cassandra.datastax.com/v1beta1
+kind: CassandraDatacenter
+metadata:
+  name: reaper-test
+  annotations:
+    reaper.cassandra-reaper.io/instance: cass-backend
+spec:
+  size: 1

test/config/deploy_reaper_test/overlays/second-cluster/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - ../../../cassdc
+namePrefix: secondary-
+
+patchesStrategicMerge:
+- cassdc-size-patch.yaml

test/config/deploy_reaper_test/overlays/two-clusters/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - ../first-cluster
+  - ../second-cluster

test/e2e/deploy_reaper_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"time"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	api "github.com/k8ssandra/reaper-operator/api/v1alpha1"
 	"github.com/k8ssandra/reaper-operator/test/framework"
 	. "github.com/onsi/ginkgo"
@@ -29,38 +31,23 @@ const (
 	reaperName = "cass-backend"
 )
 
-var (
-	namespaceBase = "reaper-cass-backend"
-)
-
 var _ = Describe("Deploy Reaper with Cassandra backend", func() {
 	Context("When a Cassandra cluster is deployed", func() {
 		Specify("Reaper is deployed", func() {
 			By("create namespace " + namespace)
 			err := framework.CreateNamespace(namespace)
 			Expect(err).ToNot(HaveOccurred())
 
-			By("deploy cass-operator and reaper-operator")
-			framework.KustomizeAndApply(namespace, "deploy_reaper_test")
-
-			By("wait for cass-operator to be ready")
-			err = framework.WaitForCassOperatorReady(namespace)
-			Expect(err).ToNot(HaveOccurred(), "failed waiting for cass-operator to become ready")
+			By("deploy cass-operator, reaper-operator and cassdc")
+			framework.KustomizeAndApply(namespace, "deploy_reaper_test", "k8ssandra")
+			// The first apply fails often for some reason, redo just to avoid flakiness of the test
+			framework.KustomizeAndApply(namespace, "deploy_reaper_test", "k8ssandra")
+			cassdcKey := types.NamespacedName{Namespace: namespace, Name: "reaper-test"}
 
 			By("wait for reaper-operator to be ready")
 			err = framework.WaitForReaperOperatorReady(namespace)
 			Expect(err).ToNot(HaveOccurred(), "failed waiting for reaper-operator to become ready")
 
-			By("wait for cassdc to be ready")
-			cassdcKey := types.NamespacedName{Namespace: namespace, Name: "reaper-test"}
-			cassdcRetryInterval := 15 * time.Second
-			cassdcTimeout := 7 * time.Minute
-			err = framework.WaitForCassDcReady(cassdcKey, cassdcRetryInterval, cassdcTimeout)
-			Expect(err).ToNot(HaveOccurred(), "failed waiting for cassdc to become ready")
-
-			cassdc, err := framework.GetCassDc(cassdcKey)
-			Expect(err).ToNot(HaveOccurred(), "failed to get cassdc")
-
 			By("deploy reaper")
 			reaper := &api.Reaper{
 				ObjectMeta: metav1.ObjectMeta{
@@ -73,7 +60,7 @@ var _ = Describe("Deploy Reaper with Cassandra backend", func() {
 						StorageType:       api.StorageTypeCassandra,
 						CassandraBackend: &api.CassandraBackend{
 							CassandraDatacenter: api.CassandraDatacenterRef{
-								Name: cassdc.Name,
+								Name: cassdcKey.Name,
 							},
 						},
 					},
@@ -83,16 +70,76 @@ var _ = Describe("Deploy Reaper with Cassandra backend", func() {
 			err = framework.Client.Create(context.Background(), reaper)
 			Expect(err).ToNot(HaveOccurred(), "failed to create reaper object")
 
+			By("wait for cass-operator to be ready")
+			err = framework.WaitForCassOperatorReady(namespace)
+			Expect(err).ToNot(HaveOccurred(), "failed waiting for cass-operator to become ready")
+
+			By("wait for cassdc to be ready")
+			cassdcRetryInterval := 15 * time.Second
+			cassdcTimeout := 7 * time.Minute
+			err = framework.WaitForCassDcReady(cassdcKey, cassdcRetryInterval, cassdcTimeout)
+			Expect(err).ToNot(HaveOccurred(), "failed waiting for cassdc to become ready")
+
 			By("wait for reaper to become ready")
 			reaperKey := types.NamespacedName{Namespace: reaper.Namespace, Name: reaper.Name}
 			err = framework.WaitForReaperReady(reaperKey, 10*time.Second, 3*time.Minute)
 			Expect(err).ToNot(HaveOccurred(), "failed waiting for reaper to become ready")
 
+			// This should be ready at the same time as cassdc
+
 			By("wait for the cluster to get registered with reaper")
 			err = framework.WaitForReaper(reaperKey, 10*time.Second, 3*time.Minute, func(reaper *api.Reaper) bool {
-				return len(reaper.Status.Clusters) == 1 && reaper.Status.Clusters[0] == cassdc.Spec.ClusterName
+				return len(reaper.Status.Clusters) == 1
+			})
+			Expect(err).ToNot(HaveOccurred(), "failing waiting for cluster to get registered")
+
+			// Remove Cluster and see that it's being readded
+			By("removing cluster status and expecting it to reappear")
+			err = framework.Client.Get(context.Background(), reaperKey, reaper)
+			Expect(err).ToNot(HaveOccurred(), "failed to refresh reaper object")
+
+			reaperPatch := client.MergeFrom(reaper.DeepCopy())
+			reaper.Status.Clusters = make([]string, 0)
+			err = framework.Client.Status().Patch(context.Background(), reaper, reaperPatch)
+
+			err = framework.WaitForReaper(reaperKey, 10*time.Second, 3*time.Minute, func(reaper *api.Reaper) bool {
+				return len(reaper.Status.Clusters) == 0
+			})
+			Expect(err).ToNot(HaveOccurred(), "failing waiting for clusters to get removed")
+
+			err = framework.WaitForReaper(reaperKey, 10*time.Second, 3*time.Minute, func(reaper *api.Reaper) bool {
+				return len(reaper.Status.Clusters) == 1
+			})
+			Expect(err).ToNot(HaveOccurred(), "failing waiting for clusters to get re-registered")
+		})
+
+		Specify("More clusters are created", func() {
+			// Since we didn't clean the previous resources, we don't need to redeploy everything
+			By("starting second-cluster")
+			framework.KustomizeAndApply(namespace, "deploy_reaper_test", "second-cluster")
+
+			cassdcKey := types.NamespacedName{Namespace: namespace, Name: "secondary-reaper-test"}
+
+			By("waiting for secondary-reaper-test cluster to be ready")
+			cassdcRetryInterval := 15 * time.Second
+			cassdcTimeout := 7 * time.Minute
+			err := framework.WaitForCassDcReady(cassdcKey, cassdcRetryInterval, cassdcTimeout)
+			Expect(err).ToNot(HaveOccurred(), "failed waiting for cassdc to become ready")
+
+			By("waiting for the second cluster to get registered with reaper")
+			reaperKey := types.NamespacedName{Namespace: namespace, Name: reaperName}
+			err = framework.WaitForReaper(reaperKey, 10*time.Second, 3*time.Minute, func(reaper *api.Reaper) bool {
+				return len(reaper.Status.Clusters) == 2
 			})
 			Expect(err).ToNot(HaveOccurred(), "failing waiting for cluster to get registered")
 		})
 	})
 })
+
+var _ = AfterSuite(func() {
+	// Clean up the deploy-reaper-test namespace after the tests
+	// CassandraDatacenters are deleted first so that cass-operator can remove the finalizers
+	Expect(framework.RemoveCassandraDatacenters(namespace)).Should(Succeed())
+	// Removing the namespace should be enough to remove rest of the resources (CRDs will be left behind..)
+	Expect(framework.RemoveNamespace(namespace)).Should(Succeed())
+}, 60)

test/framework/framework.go

@@ -65,22 +65,25 @@ func Init() {
 	initialized = true
 }
 
-// Runs kustomize build followed kubectl apply. dir specifies the name of a test directory.
+// KustomizeAndApply runs kustomize build followed kubectl apply. dir specifies the name of a test directory.
 // By default this function will run kustomize build on dir/overlays/k8ssandra. This will
 // result in using upstream operator images. If you are testing against a fork, then set
 // the TEST_OVERLAY environment variable to specify the fork overlay to use. When
 // TEST_OVERLAY is set this function will run kustomize build on
 // dir/overlays/forks/TEST_OVERLAY which will allow you to use a custom operator image.
-func KustomizeAndApply(namespace, dir string) {
+func KustomizeAndApply(namespace, dir, overlay string) {
 	kustomizeDir := ""
+	if overlay == "" {
+		overlay = defaultOverlay
+	}
 
 	path, err := os.Getwd()
 	Expect(err).ToNot(HaveOccurred())
 
-	if overlay, found := os.LookupEnv("TEST_OVERLAY"); found {
-		kustomizeDir = filepath.Clean(path + "/../config/" + dir + "/overlays/forks/" + overlay)
+	if forkOverlay, found := os.LookupEnv("TEST_OVERLAY"); found && overlay == defaultOverlay {
+		kustomizeDir = filepath.Clean(path + "/../config/" + dir + "/overlays/forks/" + forkOverlay)
 	} else {
-		kustomizeDir = filepath.Clean(path + "/../config/" + dir + "/overlays/" + defaultOverlay)
+		kustomizeDir = filepath.Clean(path + "/../config/" + dir + "/overlays/" + overlay)
 	}
 
 	GinkgoWriter.Write([]byte("RUNNING: kustomize build " + kustomizeDir))
@@ -108,6 +111,7 @@ func ApplyFile(namespace, file string) {
 	Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("kubectl apply failed: %s", err))
 }
 
+// CreateNamespace creates the given namespace or exits if it already exists
 func CreateNamespace(name string) error {
 	namespace := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
@@ -123,7 +127,18 @@ func CreateNamespace(name string) error {
 	return fmt.Errorf("failed to create namespace %s: %s", name, err)
 }
 
-// Blocks until .Status.ReadyReplicas == readyReplicas or until timeout is reached. An error is returned
+// RemoveNamespace deletes the given namespace
+func RemoveNamespace(name string) error {
+	namespace := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+	}
+
+	return Client.Delete(context.Background(), namespace)
+}
+
+// WaitForDeploymentReady blocks until .Status.ReadyReplicas == readyReplicas or until timeout is reached. An error is returned
 // if fetching the Deployment fails.
 func WaitForDeploymentReady(key types.NamespacedName, readyReplicas int32, retryInterval, timeout time.Duration) error {
 	return wait.Poll(retryInterval, timeout, func() (bool, error) {
@@ -139,21 +154,21 @@ func WaitForDeploymentReady(key types.NamespacedName, readyReplicas int32, retry
 	})
 }
 
-// Blocks until the cass-operator Deployment is ready. This function assumes that there will be a
+// WaitForCassOperatorReady blocks until the cass-operator Deployment is ready. This function assumes that there will be a
 // single replica in the Deployment.
 func WaitForCassOperatorReady(namespace string) error {
 	key := types.NamespacedName{Namespace: namespace, Name: "cass-operator"}
 	return WaitForDeploymentReady(key, 1, OperatorRetryInterval, OperatorTimeout)
 }
 
-// Blocks until the reaper-operator deployment is ready. This function assumes that there will be
+// WaitForReaperOperatorReady blocks until the reaper-operator deployment is ready. This function assumes that there will be
 // a single replica in the Deployment.
 func WaitForReaperOperatorReady(namespace string) error {
 	key := types.NamespacedName{Namespace: namespace, Name: "reaper-operator"}
 	return WaitForDeploymentReady(key, 1, OperatorRetryInterval, OperatorTimeout)
 }
 
-// Blocks until the CassandraDatacenter is ready as determined by
+// WaitForCassDcReady blocks until the CassandraDatacenter is ready as determined by
 // .Status.CassandraOperatorProgress == ProgressReady or until timeout is reached. An error is returned
 // is fetching the CassandraDatacenter fails.
 func WaitForCassDcReady(key types.NamespacedName, retryInterval, timeout time.Duration) error {
@@ -178,6 +193,12 @@ func GetCassDc(key types.NamespacedName) (*cassdcv1beta1.CassandraDatacenter, er
 	return cassdc, err
 }
 
+// RemoveCassandraDatacenters removes all the cassdcv1beta1.CassandraDatacenter objects
+func RemoveCassandraDatacenters(namespace string) error {
+	cassdc := &cassdcv1beta1.CassandraDatacenter{}
+	return Client.DeleteAllOf(context.Background(), cassdc, client.InNamespace(namespace))
+}
+
 func logCassDcStatus(cassdc *cassdcv1beta1.CassandraDatacenter, start time.Time) {
 	if d, err := yaml.Marshal(cassdc.Status); err == nil {
 		duration := time.Now().Sub(start)
@@ -190,16 +211,8 @@ func logCassDcStatus(cassdc *cassdcv1beta1.CassandraDatacenter, start time.Time)
 }
 
 func WaitForReaperReady(key types.NamespacedName, retryInterval, timeout time.Duration) error {
-	return wait.Poll(retryInterval, timeout, func() (bool, error) {
-		reaper := &api.Reaper{}
-		err := Client.Get(context.Background(), key, reaper)
-		if err != nil {
-			if apierrors.IsNotFound(err) {
-				return false, nil
-			}
-			return true, err
-		}
-		return reaper.Status.Ready, nil
+	return WaitForReaper(key, retryInterval, timeout, func(reaper *api.Reaper) bool {
+		return reaper.Status.Ready
 	})
 }
 
@@ -258,7 +271,7 @@ func getNodePort(service *corev1.Service, portName string) (string, error) {
 	return "", fmt.Errorf("failed to find nodeport %s", portName)
 }
 
-// Returns s with a date suffix of -yyMMddHHmmss
+// WithDateSuffix returns s with a date suffix of -yyMMddHHmmss
 func WithDateSuffix(s string) string {
 	return s + "-" + time.Now().Format("060102150405")
 }