Improve the logging of the HA upgrade tests and disable a test for the global synchronization mode (#2385)

Author: johscheuer

e2e/fixtures/ha_fdb_cluster.go

@@ -22,7 +22,10 @@ package fixtures
 
 import (
 	"fmt"
+	"log"
+	"math/rand"
 	"strings"
+	"time"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/v2/api/v1beta2"
 	chaosmesh "github.com/FoundationDB/fdb-kubernetes-operator/v2/e2e/chaos-mesh/api/v1alpha1"
@@ -226,8 +229,15 @@ func (haFDBCluster *HaFdbCluster) UpgradeClusterWithTimeout(
 	waitForReconciliation bool,
 	timeout int,
 ) error {
+	startTime := time.Now()
 	expectedGenerations := map[string]int64{}
-	for _, cluster := range haFDBCluster.GetAllClusters() {
+	clusters := haFDBCluster.GetAllClusters()
+	// Randomize the order of the clusters slice.
+	rand.Shuffle(len(clusters), func(i, j int) {
+		clusters[i], clusters[j] = clusters[j], clusters[i]
+	})
+
+	for _, cluster := range clusters {
 		curCluster := cluster.GetCluster()
 
 		generation := curCluster.ObjectMeta.Generation
@@ -249,8 +259,12 @@ func (haFDBCluster *HaFdbCluster) UpgradeClusterWithTimeout(
 		return nil
 	}
 
+	defer func() {
+		log.Println("upgrade took:", time.Since(startTime).String())
+	}()
+
 	g := new(errgroup.Group)
-	for _, cluster := range haFDBCluster.GetAllClusters() {
+	for _, cluster := range clusters {
 		singleCluster := cluster // https://golang.org/doc/faq#closures_and_goroutines
 		g.Go(func() error {
 			return singleCluster.WaitForReconciliation(

e2e/test_operator_ha_upgrades/operator_ha_upgrade_test.go

@@ -461,13 +461,15 @@ var _ = Describe("Operator HA Upgrades", Label("e2e", "pr"), func() {
 						coordinatorMap[pod.UID] = pod
 					}
 
+					log.Println(
+						"Cluster",
+						cluster.Name(),
+						"is running at version",
+						cluster.GetCluster().GetRunningVersion(),
+						"target version is",
+						targetVersion,
+					)
 					if cluster.GetCluster().GetRunningVersion() == targetVersion {
-						log.Println(
-							"Cluster",
-							cluster.Name(),
-							"is running at version",
-							targetVersion,
-						)
 						continue
 					}
 
@@ -479,11 +481,12 @@ var _ = Describe("Operator HA Upgrades", Label("e2e", "pr"), func() {
 					return true
 				}
 
-				randomCluster := factory.RandomPickOneCluster(clusters)
 				// Make sure we are not deleting coordinator Pods
 				var randomPod *corev1.Pod
 				g.Eventually(func() bool {
-					randomPod = factory.ChooseRandomPod(randomCluster.GetPods())
+					randomPod = factory.ChooseRandomPod(
+						factory.RandomPickOneCluster(clusters).GetPods(),
+					)
 					_, ok := coordinatorMap[randomPod.UID]
 					if ok {
 						log.Println("Skipping pod:", randomPod.Name, "as it is a coordinator")
@@ -615,6 +618,13 @@ var _ = Describe("Operator HA Upgrades", Label("e2e", "pr"), func() {
 				)
 			}
 
+			// TODO (johscheuer): https://github.com/FoundationDB/fdb-kubernetes-operator/issues/2383
+			if factory.GetSynchronizationMode() == fdbv1beta2.SynchronizationModeGlobal {
+				Skip(
+					"Test is currently disabled with the global mode because of flakiness, see: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/2383",
+				)
+			}
+
 			clusterConfig := fixtures.DefaultClusterConfigWithHaMode(
 				fixtures.HaFourZoneSingleSat,
 				false,