diff --git a/Makefile b/Makefile index cb0f671..06d49bf 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,7 @@ init-test-overlays: PHONY: e2e-test e2e-test: @echo Running e2e tests + cd test/config/deploy_reaper_test && $(KUSTOMIZE) edit set image controller=${REV_IMAGE} && cd - go test -timeout 1800s ./test/e2e/... -ginkgo.v -ginkgo.progress -test.v # Generate code diff --git a/test/config/cassdc/kustomization.yaml b/test/config/cassdc/kustomization.yaml index f549b87..764f0dc 100644 --- a/test/config/cassdc/kustomization.yaml +++ b/test/config/cassdc/kustomization.yaml @@ -1,3 +1,5 @@ +namespace: deploy-reaper-test + resources: - cassdc.yaml diff --git a/test/config/deploy_reaper_test/kustomization.yaml b/test/config/deploy_reaper_test/kustomization.yaml index 36ce8fd..97b66bc 100644 --- a/test/config/deploy_reaper_test/kustomization.yaml +++ b/test/config/deploy_reaper_test/kustomization.yaml @@ -1,2 +1,8 @@ resources: - base +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: docker.io/k8ssandra/reaper-operator + newTag: 2cfebf759df9 diff --git a/test/config/deploy_reaper_test/overlays/first-cluster/kustomization.yaml b/test/config/deploy_reaper_test/overlays/first-cluster/kustomization.yaml new file mode 100644 index 0000000..2ac74c6 --- /dev/null +++ b/test/config/deploy_reaper_test/overlays/first-cluster/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - ../../../cassdc +namePrefix: primary- \ No newline at end of file diff --git a/test/config/deploy_reaper_test/overlays/second-cluster/cassdc-size-patch.yaml b/test/config/deploy_reaper_test/overlays/second-cluster/cassdc-size-patch.yaml new file mode 100644 index 0000000..3be2c0b --- /dev/null +++ b/test/config/deploy_reaper_test/overlays/second-cluster/cassdc-size-patch.yaml @@ -0,0 +1,8 @@ +apiVersion: cassandra.datastax.com/v1beta1 +kind: CassandraDatacenter +metadata: + name: reaper-test + annotations: + reaper.cassandra-reaper.io/instance: cass-backend +spec: + size: 1 diff --git a/test/config/deploy_reaper_test/overlays/second-cluster/kustomization.yaml b/test/config/deploy_reaper_test/overlays/second-cluster/kustomization.yaml new file mode 100644 index 0000000..b5ea66a --- /dev/null +++ b/test/config/deploy_reaper_test/overlays/second-cluster/kustomization.yaml @@ -0,0 +1,6 @@ +resources: + - ../../../cassdc +namePrefix: secondary- + +patchesStrategicMerge: +- cassdc-size-patch.yaml \ No newline at end of file diff --git a/test/config/deploy_reaper_test/overlays/two-clusters/kustomization.yaml b/test/config/deploy_reaper_test/overlays/two-clusters/kustomization.yaml new file mode 100644 index 0000000..efe64a7 --- /dev/null +++ b/test/config/deploy_reaper_test/overlays/two-clusters/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - ../first-cluster + - ../second-cluster diff --git a/test/e2e/deploy_reaper_test.go b/test/e2e/deploy_reaper_test.go index ee989be..4fe9489 100644 --- a/test/e2e/deploy_reaper_test.go +++ b/test/e2e/deploy_reaper_test.go @@ -4,6 +4,8 @@ import ( "context" "time" + "sigs.k8s.io/controller-runtime/pkg/client" + api "github.com/k8ssandra/reaper-operator/api/v1alpha1" "github.com/k8ssandra/reaper-operator/test/framework" . "github.com/onsi/ginkgo" @@ -29,10 +31,6 @@ const ( reaperName = "cass-backend" ) -var ( - namespaceBase = "reaper-cass-backend" -) - var _ = Describe("Deploy Reaper with Cassandra backend", func() { Context("When a Cassandra cluster is deployed", func() { Specify("Reaper is deployed", func() { @@ -40,27 +38,16 @@ var _ = Describe("Deploy Reaper with Cassandra backend", func() { err := framework.CreateNamespace(namespace) Expect(err).ToNot(HaveOccurred()) - By("deploy cass-operator and reaper-operator") - framework.KustomizeAndApply(namespace, "deploy_reaper_test") - - By("wait for cass-operator to be ready") - err = framework.WaitForCassOperatorReady(namespace) - Expect(err).ToNot(HaveOccurred(), "failed waiting for cass-operator to become ready") + By("deploy cass-operator, reaper-operator and cassdc") + framework.KustomizeAndApply(namespace, "deploy_reaper_test", "k8ssandra") + // The first apply fails often for some reason, redo just to avoid flakiness of the test + framework.KustomizeAndApply(namespace, "deploy_reaper_test", "k8ssandra") + cassdcKey := types.NamespacedName{Namespace: namespace, Name: "reaper-test"} By("wait for reaper-operator to be ready") err = framework.WaitForReaperOperatorReady(namespace) Expect(err).ToNot(HaveOccurred(), "failed waiting for reaper-operator to become ready") - By("wait for cassdc to be ready") - cassdcKey := types.NamespacedName{Namespace: namespace, Name: "reaper-test"} - cassdcRetryInterval := 15 * time.Second - cassdcTimeout := 7 * time.Minute - err = framework.WaitForCassDcReady(cassdcKey, cassdcRetryInterval, cassdcTimeout) - Expect(err).ToNot(HaveOccurred(), "failed waiting for cassdc to become ready") - - cassdc, err := framework.GetCassDc(cassdcKey) - Expect(err).ToNot(HaveOccurred(), "failed to get cassdc") - By("deploy reaper") reaper := &api.Reaper{ ObjectMeta: metav1.ObjectMeta{ @@ -73,7 +60,7 @@ var _ = Describe("Deploy Reaper with Cassandra backend", func() { StorageType: api.StorageTypeCassandra, CassandraBackend: &api.CassandraBackend{ CassandraDatacenter: api.CassandraDatacenterRef{ - Name: cassdc.Name, + Name: cassdcKey.Name, }, }, }, @@ -83,16 +70,76 @@ var _ = Describe("Deploy Reaper with Cassandra backend", func() { err = framework.Client.Create(context.Background(), reaper) Expect(err).ToNot(HaveOccurred(), "failed to create reaper object") + By("wait for cass-operator to be ready") + err = framework.WaitForCassOperatorReady(namespace) + Expect(err).ToNot(HaveOccurred(), "failed waiting for cass-operator to become ready") + + By("wait for cassdc to be ready") + cassdcRetryInterval := 15 * time.Second + cassdcTimeout := 7 * time.Minute + err = framework.WaitForCassDcReady(cassdcKey, cassdcRetryInterval, cassdcTimeout) + Expect(err).ToNot(HaveOccurred(), "failed waiting for cassdc to become ready") + By("wait for reaper to become ready") reaperKey := types.NamespacedName{Namespace: reaper.Namespace, Name: reaper.Name} err = framework.WaitForReaperReady(reaperKey, 10*time.Second, 3*time.Minute) Expect(err).ToNot(HaveOccurred(), "failed waiting for reaper to become ready") + // This should be ready at the same time as cassdc + By("wait for the cluster to get registered with reaper") err = framework.WaitForReaper(reaperKey, 10*time.Second, 3*time.Minute, func(reaper *api.Reaper) bool { - return len(reaper.Status.Clusters) == 1 && reaper.Status.Clusters[0] == cassdc.Spec.ClusterName + return len(reaper.Status.Clusters) == 1 + }) + Expect(err).ToNot(HaveOccurred(), "failing waiting for cluster to get registered") + + // Remove Cluster and see that it's being readded + By("removing cluster status and expecting it to reappear") + err = framework.Client.Get(context.Background(), reaperKey, reaper) + Expect(err).ToNot(HaveOccurred(), "failed to refresh reaper object") + + reaperPatch := client.MergeFrom(reaper.DeepCopy()) + reaper.Status.Clusters = make([]string, 0) + err = framework.Client.Status().Patch(context.Background(), reaper, reaperPatch) + + err = framework.WaitForReaper(reaperKey, 10*time.Second, 3*time.Minute, func(reaper *api.Reaper) bool { + return len(reaper.Status.Clusters) == 0 + }) + Expect(err).ToNot(HaveOccurred(), "failing waiting for clusters to get removed") + + err = framework.WaitForReaper(reaperKey, 10*time.Second, 3*time.Minute, func(reaper *api.Reaper) bool { + return len(reaper.Status.Clusters) == 1 + }) + Expect(err).ToNot(HaveOccurred(), "failing waiting for clusters to get re-registered") + }) + + Specify("More clusters are created", func() { + // Since we didn't clean the previous resources, we don't need to redeploy everything + By("starting second-cluster") + framework.KustomizeAndApply(namespace, "deploy_reaper_test", "second-cluster") + + cassdcKey := types.NamespacedName{Namespace: namespace, Name: "secondary-reaper-test"} + + By("waiting for secondary-reaper-test cluster to be ready") + cassdcRetryInterval := 15 * time.Second + cassdcTimeout := 7 * time.Minute + err := framework.WaitForCassDcReady(cassdcKey, cassdcRetryInterval, cassdcTimeout) + Expect(err).ToNot(HaveOccurred(), "failed waiting for cassdc to become ready") + + By("waiting for the second cluster to get registered with reaper") + reaperKey := types.NamespacedName{Namespace: namespace, Name: reaperName} + err = framework.WaitForReaper(reaperKey, 10*time.Second, 3*time.Minute, func(reaper *api.Reaper) bool { + return len(reaper.Status.Clusters) == 2 }) Expect(err).ToNot(HaveOccurred(), "failing waiting for cluster to get registered") }) }) }) + +var _ = AfterSuite(func() { + // Clean up the deploy-reaper-test namespace after the tests + // CassandraDatacenters are deleted first so that cass-operator can remove the finalizers + Expect(framework.RemoveCassandraDatacenters(namespace)).Should(Succeed()) + // Removing the namespace should be enough to remove rest of the resources (CRDs will be left behind..) + Expect(framework.RemoveNamespace(namespace)).Should(Succeed()) +}, 60) diff --git a/test/framework/framework.go b/test/framework/framework.go index b6fbc0a..1451a50 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -65,22 +65,25 @@ func Init() { initialized = true } -// Runs kustomize build followed kubectl apply. dir specifies the name of a test directory. +// KustomizeAndApply runs kustomize build followed kubectl apply. dir specifies the name of a test directory. // By default this function will run kustomize build on dir/overlays/k8ssandra. This will // result in using upstream operator images. If you are testing against a fork, then set // the TEST_OVERLAY environment variable to specify the fork overlay to use. When // TEST_OVERLAY is set this function will run kustomize build on // dir/overlays/forks/TEST_OVERLAY which will allow you to use a custom operator image. -func KustomizeAndApply(namespace, dir string) { +func KustomizeAndApply(namespace, dir, overlay string) { kustomizeDir := "" + if overlay == "" { + overlay = defaultOverlay + } path, err := os.Getwd() Expect(err).ToNot(HaveOccurred()) - if overlay, found := os.LookupEnv("TEST_OVERLAY"); found { - kustomizeDir = filepath.Clean(path + "/../config/" + dir + "/overlays/forks/" + overlay) + if forkOverlay, found := os.LookupEnv("TEST_OVERLAY"); found && overlay == defaultOverlay { + kustomizeDir = filepath.Clean(path + "/../config/" + dir + "/overlays/forks/" + forkOverlay) } else { - kustomizeDir = filepath.Clean(path + "/../config/" + dir + "/overlays/" + defaultOverlay) + kustomizeDir = filepath.Clean(path + "/../config/" + dir + "/overlays/" + overlay) } GinkgoWriter.Write([]byte("RUNNING: kustomize build " + kustomizeDir)) @@ -108,6 +111,7 @@ func ApplyFile(namespace, file string) { Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("kubectl apply failed: %s", err)) } +// CreateNamespace creates the given namespace or exits if it already exists func CreateNamespace(name string) error { namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -123,7 +127,18 @@ func CreateNamespace(name string) error { return fmt.Errorf("failed to create namespace %s: %s", name, err) } -// Blocks until .Status.ReadyReplicas == readyReplicas or until timeout is reached. An error is returned +// RemoveNamespace deletes the given namespace +func RemoveNamespace(name string) error { + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + + return Client.Delete(context.Background(), namespace) +} + +// WaitForDeploymentReady blocks until .Status.ReadyReplicas == readyReplicas or until timeout is reached. An error is returned // if fetching the Deployment fails. func WaitForDeploymentReady(key types.NamespacedName, readyReplicas int32, retryInterval, timeout time.Duration) error { return wait.Poll(retryInterval, timeout, func() (bool, error) { @@ -139,21 +154,21 @@ func WaitForDeploymentReady(key types.NamespacedName, readyReplicas int32, retry }) } -// Blocks until the cass-operator Deployment is ready. This function assumes that there will be a +// WaitForCassOperatorReady blocks until the cass-operator Deployment is ready. This function assumes that there will be a // single replica in the Deployment. func WaitForCassOperatorReady(namespace string) error { key := types.NamespacedName{Namespace: namespace, Name: "cass-operator"} return WaitForDeploymentReady(key, 1, OperatorRetryInterval, OperatorTimeout) } -// Blocks until the reaper-operator deployment is ready. This function assumes that there will be +// WaitForReaperOperatorReady blocks until the reaper-operator deployment is ready. This function assumes that there will be // a single replica in the Deployment. func WaitForReaperOperatorReady(namespace string) error { key := types.NamespacedName{Namespace: namespace, Name: "reaper-operator"} return WaitForDeploymentReady(key, 1, OperatorRetryInterval, OperatorTimeout) } -// Blocks until the CassandraDatacenter is ready as determined by +// WaitForCassDcReady blocks until the CassandraDatacenter is ready as determined by // .Status.CassandraOperatorProgress == ProgressReady or until timeout is reached. An error is returned // is fetching the CassandraDatacenter fails. func WaitForCassDcReady(key types.NamespacedName, retryInterval, timeout time.Duration) error { @@ -178,6 +193,12 @@ func GetCassDc(key types.NamespacedName) (*cassdcv1beta1.CassandraDatacenter, er return cassdc, err } +// RemoveCassandraDatacenters removes all the cassdcv1beta1.CassandraDatacenter objects +func RemoveCassandraDatacenters(namespace string) error { + cassdc := &cassdcv1beta1.CassandraDatacenter{} + return Client.DeleteAllOf(context.Background(), cassdc, client.InNamespace(namespace)) +} + func logCassDcStatus(cassdc *cassdcv1beta1.CassandraDatacenter, start time.Time) { if d, err := yaml.Marshal(cassdc.Status); err == nil { duration := time.Now().Sub(start) @@ -190,16 +211,8 @@ func logCassDcStatus(cassdc *cassdcv1beta1.CassandraDatacenter, start time.Time) } func WaitForReaperReady(key types.NamespacedName, retryInterval, timeout time.Duration) error { - return wait.Poll(retryInterval, timeout, func() (bool, error) { - reaper := &api.Reaper{} - err := Client.Get(context.Background(), key, reaper) - if err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - return true, err - } - return reaper.Status.Ready, nil + return WaitForReaper(key, retryInterval, timeout, func(reaper *api.Reaper) bool { + return reaper.Status.Ready }) } @@ -258,7 +271,7 @@ func getNodePort(service *corev1.Service, portName string) (string, error) { return "", fmt.Errorf("failed to find nodeport %s", portName) } -// Returns s with a date suffix of -yyMMddHHmmss +// WithDateSuffix returns s with a date suffix of -yyMMddHHmmss func WithDateSuffix(s string) string { return s + "-" + time.Now().Format("060102150405") }