feat: add redisreplication status masterNode (OT-CONTAINER-KIT#849)

* feat: add redisreplication status masterNode Signed-off-by: drivebyer <[email protected]> * fix e2e Signed-off-by: drivebyer <[email protected]> * fix lint Signed-off-by: drivebyer <[email protected]> * fix lint Signed-off-by: drivebyer <[email protected]> --------- Signed-off-by: drivebyer <[email protected]> Signed-off-by: Matt Robinson <[email protected]>
mattrobinsonsre · Jul 11, 2024 · e10b2cb · e10b2cb
1 parent fb8257e
commit e10b2cb
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 37 deletions.
diff --git a/api/v1beta1/redisreplication_types.go b/api/v1beta1/redisreplication_types.go
@@ -31,7 +31,9 @@ func (cr *RedisReplicationSpec) GetReplicationCounts(t string) int32 {
 }
 
 // RedisStatus defines the observed state of Redis
-type RedisReplicationStatus struct{}
+type RedisReplicationStatus struct {
+	MasterNode string `json:"masterNode,omitempty"`
+}
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status

diff --git a/api/v1beta2/redisreplication_types.go b/api/v1beta2/redisreplication_types.go
@@ -36,7 +36,9 @@ func (cr *RedisReplicationSpec) GetReplicationCounts(t string) int32 {
 }
 
 // RedisStatus defines the observed state of Redis
-type RedisReplicationStatus struct{}
+type RedisReplicationStatus struct {
+	MasterNode string `json:"masterNode,omitempty"`
+}
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status

diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml
@@ -4127,6 +4127,9 @@ spec:
             type: object
           status:
             description: RedisStatus defines the observed state of Redis
+            properties:
+              masterNode:
+                type: string
             type: object
         required:
         - spec
@@ -8841,6 +8844,9 @@ spec:
             type: object
           status:
             description: RedisStatus defines the observed state of Redis
+            properties:
+              masterNode:
+                type: string
             type: object
         required:
         - spec

diff --git a/controllers/redisreplication_controller.go b/controllers/redisreplication_controller.go
@@ -77,19 +77,39 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{RequeueAfter: time.Second * 60}, nil
 	}
 
-	if len(k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master")) > int(leaderReplicas) {
+	var realMaster string
+	masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master")
+	if len(masterNodes) > int(leaderReplicas) {
 		reqLogger.Info("Creating redis replication by executing replication creation commands", "Replication.Ready", strconv.Itoa(int(redisReplicationInfo.Status.ReadyReplicas)))
-		masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master")
 		slaveNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "slave")
-		err := k8sutils.CreateMasterSlaveReplication(ctx, r.K8sClient, r.Log, instance, masterNodes, slaveNodes)
+		realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes)
+		if len(slaveNodes) == 0 {
+			realMaster = masterNodes[0]
+		}
+		err := k8sutils.CreateMasterSlaveReplication(ctx, r.K8sClient, r.Log, instance, masterNodes, realMaster)
 		if err != nil {
 			return ctrl.Result{RequeueAfter: time.Second * 60}, err
 		}
 	}
+	realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes)
+	if err := r.UpdateRedisReplicationMaster(ctx, instance, realMaster); err != nil {
+		return ctrl.Result{}, err
+	}
 	reqLogger.Info("Will reconcile redis operator in again 10 seconds")
 	return ctrl.Result{RequeueAfter: time.Second * 10}, nil
 }
 
+func (r *RedisReplicationReconciler) UpdateRedisReplicationMaster(ctx context.Context, instance *redisv1beta2.RedisReplication, masterNode string) error {
+	if instance.Status.MasterNode == masterNode {
+		return nil
+	}
+	instance.Status.MasterNode = masterNode
+	if err := r.Client.Status().Update(ctx, instance); err != nil {
+		return err
+	}
+	return nil
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *RedisReplicationReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).

diff --git a/controllers/redissentinel_controller.go b/controllers/redissentinel_controller.go
@@ -11,8 +11,11 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 )
 
 // RedisSentinelReconciler reconciles a RedisSentinel object
@@ -83,5 +86,14 @@ func (r *RedisSentinelReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 func (r *RedisSentinelReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&redisv1beta2.RedisSentinel{}).
+		Watches(&redisv1beta2.RedisReplication{}, &handler.Funcs{
+			CreateFunc: nil,
+			UpdateFunc: func(ctx context.Context, event event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) {
+				_ = event.ObjectNew.GetName()
+				_ = event.ObjectNew.GetNamespace()
+			},
+			DeleteFunc:  nil,
+			GenericFunc: nil,
+		}).
 		Complete(r)
 }
diff --git a/k8sutils/redis.go b/k8sutils/redis.go
@@ -554,25 +554,7 @@ func checkAttachedSlave(ctx context.Context, redisClient *redis.Client, logger l
 	return 0
 }
 
-func CreateMasterSlaveReplication(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, masterPods []string, slavePods []string) error {
-	var realMasterPod string
-
-	for _, podName := range masterPods {
-		redisClient := configureRedisReplicationClient(client, logger, cr, podName)
-		defer redisClient.Close()
-
-		if checkAttachedSlave(ctx, redisClient, logger, podName) > 0 {
-			realMasterPod = podName
-			break
-		}
-	}
-	// realMasterPod = checkAttachedSlave(ctx, client, logger, cr, masterPods)
-
-	if len(slavePods) < 1 {
-		realMasterPod = masterPods[0]
-		logger.V(1).Info("No Master Node Found with attached slave promoting the following pod to master", "pod", masterPods[0])
-	}
-
+func CreateMasterSlaveReplication(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, masterPods []string, realMasterPod string) error {
 	logger.V(1).Info("Redis Master Node is set to", "pod", realMasterPod)
 	realMasterInfo := RedisDetails{
 		PodName:   realMasterPod,
@@ -596,3 +578,15 @@ func CreateMasterSlaveReplication(ctx context.Context, client kubernetes.Interfa
 
 	return nil
 }
+
+func GetRedisReplicationRealMaster(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, masterPods []string) string {
+	for _, podName := range masterPods {
+		redisClient := configureRedisReplicationClient(client, logger, cr, podName)
+		defer redisClient.Close()
+
+		if checkAttachedSlave(ctx, redisClient, logger, podName) > 0 {
+			return podName
+		}
+	}
+	return ""
+}
diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml
@@ -15,6 +15,8 @@ spec:
         file: ready-svc.yaml
     - assert:
         file: ready-pvc.yaml
+    - assert:
+        file: ready-replication.yaml
     catch:
       - description: Redis Operator Logs
         podLogs:
@@ -51,16 +53,4 @@ spec:
          content: |
           kubectl exec --namespace ${NAMESPACE} redis-replication-0 -- redis-cli -p 6379 set foo-0 bar-0
          check:
-          ($stdout=='OK'): true
-    #  - script:
-    #      timeout: 10s
-    #      content: |
-    #       kubectl exec --namespace ${NAMESPACE} redis-replication-1 -- redis-cli -p 6379 set foo-1 bar-1
-    #      check:
-    #       ($stdout==`READONLY You can't write against a read only replica.`): true
-    #  - script:
-    #      timeout: 10s
-    #      content: |
-    #       kubectl exec --namespace ${NAMESPACE} redis-replication-2 -- redis-cli -p 6379 set foo-2 bar-2
-    #      check:
-    #       ($stdout==`READONLY You can't write against a read only replica.`): true
+          ($stdout=='OK'): true
diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml
@@ -0,0 +1,8 @@
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisReplication
+metadata:
+  name: redis-replication
+status:
+  # by default, the first pod is being selected as master
+  masterNode: redis-replication-0