csi-addons · Nikhil-Ladha · Jul 11, 2024 · Jul 8, 2024 · Jul 25, 2024
diff --git a/api/replication.storage/v1alpha1/volumegroupreplication_types.go b/api/replication.storage/v1alpha1/volumegroupreplication_types.go
@@ -21,6 +21,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	VolumeGroupReplicationNameAnnotation = "replication.storage.openshift.io/volume-group-replication-name"
+)
+
 // VolumeGroupReplicationSpec defines the desired state of VolumeGroupReplication
 type VolumeGroupReplicationSpec struct {
 	// volumeGroupReplicationClassName is the volumeGroupReplicationClass name for this VolumeGroupReplication resource

diff --git a/api/replication.storage/v1alpha1/volumegroupreplicationcontent_types.go b/api/replication.storage/v1alpha1/volumegroupreplicationcontent_types.go
@@ -21,6 +21,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	VolumeGroupReplicationContentNameAnnotation = "replication.storage.openshift.io/volumegroupreplication-content-name"
+)
+
 // VolumeGroupReplicationContentSpec defines the desired state of VolumeGroupReplicationContent
 type VolumeGroupReplicationContentSpec struct {
 	// VolumeGroupreplicationRef specifies the VolumeGroupReplication object to which this

diff --git a/cmd/manager/main.go b/cmd/manager/main.go
@@ -251,8 +251,10 @@ func main() {
 		os.Exit(1)
 	}
 	if err = (&replicationController.VolumeGroupReplicationContentReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:   mgr.GetClient(),
+		Scheme:   mgr.GetScheme(),
+		Connpool: connPool,
+		Timeout:  defaultTimeout,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "VolumeGroupReplicationContent")
 		os.Exit(1)

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -8,7 +8,6 @@ rules:
   - ""
   resources:
   - namespaces
-  - persistentvolumes
   - pods
   verbs:
   - get
@@ -30,6 +29,15 @@ rules:
   - persistentvolumeclaims/finalizers
   verbs:
   - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - coordination.k8s.io
   resources:
@@ -82,11 +90,19 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - replication.storage.openshift.io
+  resources:
+  - volumegroupreplicationclasses
+  - volumereplicationclasses
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - replication.storage.openshift.io
   resources:
   - volumegroupreplicationcontents
-  - volumegroupreplications
   verbs:
   - create
   - delete
@@ -101,7 +117,6 @@ rules:
   - volumegroupreplicationcontents/finalizers
   - volumegroupreplications/finalizers
   - volumereplications/finalizers
-  - volumereplications/status
   verbs:
   - update
 - apiGroups:
@@ -116,20 +131,32 @@ rules:
 - apiGroups:
   - replication.storage.openshift.io
   resources:
-  - volumereplicationclasses
+  - volumegroupreplications
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - replication.storage.openshift.io
   resources:
   - volumereplications
   verbs:
+  - create
+  - delete
   - get
   - list
   - update
   - watch
+- apiGroups:
+  - replication.storage.openshift.io
+  resources:
+  - volumereplications/status
+  verbs:
+  - get
+  - list
+  - update
 - apiGroups:
   - storage.k8s.io
   resources:

diff --git a/deploy/controller/rbac.yaml b/deploy/controller/rbac.yaml
@@ -101,7 +101,6 @@ rules:
   - ""
   resources:
   - namespaces
-  - persistentvolumes
   - pods
   verbs:
   - get
@@ -123,6 +122,15 @@ rules:
   - persistentvolumeclaims/finalizers
   verbs:
   - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - coordination.k8s.io
   resources:
@@ -175,11 +183,19 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - replication.storage.openshift.io
+  resources:
+  - volumegroupreplicationclasses
+  - volumereplicationclasses
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - replication.storage.openshift.io
   resources:
   - volumegroupreplicationcontents
-  - volumegroupreplications
   verbs:
   - create
   - delete
@@ -194,7 +210,6 @@ rules:
   - volumegroupreplicationcontents/finalizers
   - volumegroupreplications/finalizers
   - volumereplications/finalizers
-  - volumereplications/status
   verbs:
   - update
 - apiGroups:
@@ -209,20 +224,32 @@ rules:
 - apiGroups:
   - replication.storage.openshift.io
   resources:
-  - volumereplicationclasses
+  - volumegroupreplications
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - replication.storage.openshift.io
   resources:
   - volumereplications
   verbs:
+  - create
+  - delete
   - get
   - list
   - update
   - watch
+- apiGroups:
+  - replication.storage.openshift.io
+  resources:
+  - volumereplications/status
+  verbs:
+  - get
+  - list
+  - update
 - apiGroups:
   - storage.k8s.io
   resources:

diff --git a/docs/volumegroupreplicationclass.md b/docs/volumegroupreplicationclass.md
@@ -0,0 +1,24 @@
+# VolumeGroupReplicationClass
+
+VolumeGroupReplicationClass is a cluster scoped resource that contains driver related configuration parameters for volume group replication.
+
+`provisioner` is name of the storage provisioner.
+
+`parameters` contains key-value pairs that are passed down to the driver. Users can add their own key-value pairs. Keys with `replication.storage.openshift.io/` prefix are reserved by operator and not passed down to the driver.
+
+## Reserved parameter keys
+
+- `replication.storage.openshift.io/group-replication-secret-name`
+- `replication.storage.openshift.io/group-replication-secret-namespace`
+
+```yaml
+apiVersion: replication.storage.openshift.io/v1alpha1
+kind: VolumeGroupReplicationClass
+metadata:
+  name: volumegroupreplicationclass-sample
+spec:
+  provisioner: example.provisioner.io
+  parameters:
+    replication.storage.openshift.io/group-replication-secret-name: secret-name
+    replication.storage.openshift.io/group-replication-secret-namespace: secret-namespace
+```
diff --git a/internal/client/volumegroup-client.go b/internal/client/volumegroup-client.go
@@ -35,7 +35,7 @@ type VolumeGroup interface {
 	// CreateVolumeGroup RPC call to create a volume group.
 	CreateVolumeGroup(volumeGroupName string, volumeIDs []string, secretName, secretNamespace string, parameters map[string]string) (*proto.CreateVolumeGroupResponse, error)
 	// ModifyVolumeGroupMembership RPC call to modify the volume group.
-	ModifyVolumeGroupMembership(volumeGroupID string, volumeIDs []string, secretName, secretNamespace string) (*proto.ModifyVolumeGroupMembershipResponse, error)
+	ModifyVolumeGroupMembership(volumeGroupID string, volumeIDs []string, secretName, secretNamespace string, parameters map[string]string) (*proto.ModifyVolumeGroupMembershipResponse, error)
 	// DeleteVolumeGroup RPC call to delete the volume group.
 	DeleteVolumeGroup(volumeGroupID string, secretName, secretNamespace string) (*proto.DeleteVolumeGroupResponse, error)
 	// ControllerGetVolumeGroup RPC call to fetch the volume group.
@@ -64,12 +64,13 @@ func (vg *volumeGroupClient) CreateVolumeGroup(volumeGroupName string, volumeIDs
 }
 
 func (vg *volumeGroupClient) ModifyVolumeGroupMembership(volumeGroupID string, volumeIDs []string,
-	secretName, secretNamespace string) (*proto.ModifyVolumeGroupMembershipResponse, error) {
+	secretName, secretNamespace string, parameters map[string]string) (*proto.ModifyVolumeGroupMembershipResponse, error) {
 	req := &proto.ModifyVolumeGroupMembershipRequest{
 		VolumeGroupId:   volumeGroupID,
 		VolumeIds:       volumeIDs,
 		SecretName:      secretName,
 		SecretNamespace: secretNamespace,
+		Parameters:      parameters,
 	}
 
 	createCtx, cancel := context.WithTimeout(context.Background(), vg.timeout)

diff --git a/internal/controller/replication.storage/finalizers.go b/internal/controller/replication.storage/finalizers.go
@@ -26,6 +26,7 @@ import (
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -65,43 +66,44 @@ func (r *VolumeReplicationReconciler) removeFinalizerFromVR(logger logr.Logger,
 	return nil
 }
 
-// addFinalizerToPVC adds the VR finalizer on the PersistentVolumeClaim.
-func (r *VolumeReplicationReconciler) addFinalizerToPVC(logger logr.Logger, pvc *corev1.PersistentVolumeClaim) error {
-	if !slices.Contains(pvc.ObjectMeta.Finalizers, pvcReplicationFinalizer) {
-		logger.Info("adding finalizer to PersistentVolumeClaim object", "Finalizer", pvcReplicationFinalizer)
-		pvc.ObjectMeta.Finalizers = append(pvc.ObjectMeta.Finalizers, pvcReplicationFinalizer)
-		if err := r.Client.Update(context.TODO(), pvc); err != nil {
+// AddFinalizerToPVC adds the VR, VGR finalizer on the PersistentVolumeClaim.
+func AddFinalizerToPVC(client client.Client, logger logr.Logger, pvc *corev1.PersistentVolumeClaim,
+	replicationFinalizer string) error {
+	if !slices.Contains(pvc.ObjectMeta.Finalizers, replicationFinalizer) {
+		logger.Info("adding finalizer to PersistentVolumeClaim object", "Finalizer", replicationFinalizer)
+		pvc.ObjectMeta.Finalizers = append(pvc.ObjectMeta.Finalizers, replicationFinalizer)
+		if err := client.Update(context.TODO(), pvc); err != nil {
 			return fmt.Errorf("failed to add finalizer (%s) to PersistentVolumeClaim resource"+
 				" (%s/%s) %w",
-				pvcReplicationFinalizer, pvc.Namespace, pvc.Name, err)
+				replicationFinalizer, pvc.Namespace, pvc.Name, err)
 		}
 	}
 
 	return nil
 }
 
-// removeFinalizerFromPVC removes the VR finalizer on PersistentVolumeClaim.
-func (r *VolumeReplicationReconciler) removeFinalizerFromPVC(logger logr.Logger, pvc *corev1.PersistentVolumeClaim,
-) error {
-	if slices.Contains(pvc.ObjectMeta.Finalizers, pvcReplicationFinalizer) {
-		logger.Info("removing finalizer from PersistentVolumeClaim object", "Finalizer", pvcReplicationFinalizer)
-		pvc.ObjectMeta.Finalizers = util.RemoveFromSlice(pvc.ObjectMeta.Finalizers, pvcReplicationFinalizer)
-		if err := r.Client.Update(context.TODO(), pvc); err != nil {
+// RemoveFinalizerFromPVC removes the VR, VGR finalizer on PersistentVolumeClaim.
+func RemoveFinalizerFromPVC(client client.Client, logger logr.Logger, pvc *corev1.PersistentVolumeClaim,
+	replicationFinalizer string) error {
+	if slices.Contains(pvc.ObjectMeta.Finalizers, replicationFinalizer) {
+		logger.Info("removing finalizer from PersistentVolumeClaim object", "Finalizer", replicationFinalizer)
+		pvc.ObjectMeta.Finalizers = util.RemoveFromSlice(pvc.ObjectMeta.Finalizers, replicationFinalizer)
+		if err := client.Update(context.TODO(), pvc); err != nil {
 			return fmt.Errorf("failed to remove finalizer (%s) from PersistentVolumeClaim resource"+
 				" (%s/%s), %w",
-				pvcReplicationFinalizer, pvc.Namespace, pvc.Name, err)
+				replicationFinalizer, pvc.Namespace, pvc.Name, err)
 		}
 	}
 
 	return nil
 }
 
-// addFinalizerToVGR adds the VR finalizer on the VolumeGroupReplication.
-func (r *VolumeReplicationReconciler) addFinalizerToVGR(logger logr.Logger, vgr *replicationv1alpha1.VolumeGroupReplication) error {
+// AddFinalizerToVGR adds the VGR finalizer on the VolumeGroupReplication resource
+func AddFinalizerToVGR(client client.Client, logger logr.Logger, vgr *replicationv1alpha1.VolumeGroupReplication) error {
 	if !slices.Contains(vgr.ObjectMeta.Finalizers, vgrReplicationFinalizer) {
-		logger.Info("adding finalizer to VolumeGroupReplication object", "Finalizer", vgrReplicationFinalizer)
+		logger.Info("adding finalizer to volumeGroupReplication object", "Finalizer", vgrReplicationFinalizer)
 		vgr.ObjectMeta.Finalizers = append(vgr.ObjectMeta.Finalizers, vgrReplicationFinalizer)
-		if err := r.Client.Update(context.TODO(), vgr); err != nil {
+		if err := client.Update(context.TODO(), vgr); err != nil {
 			return fmt.Errorf("failed to add finalizer (%s) to VolumeGroupReplication resource"+
 				" (%s/%s) %w",
 				vgrReplicationFinalizer, vgr.Namespace, vgr.Name, err)
@@ -111,13 +113,18 @@ func (r *VolumeReplicationReconciler) addFinalizerToVGR(logger logr.Logger, vgr
 	return nil
 }
 
-// removeFinalizerFromVGR removes the VR finalizer on VolumeGroupReplication.
-func (r *VolumeReplicationReconciler) removeFinalizerFromVGR(logger logr.Logger, vgr *replicationv1alpha1.VolumeGroupReplication,
-) error {
+// RemoveFinalizerFromVGR removes the VGR finalizer from the VolumeGroupReplication instance.
+func RemoveFinalizerFromVGR(client client.Client, logger logr.Logger, vgr *replicationv1alpha1.VolumeGroupReplication) error {
 	if slices.Contains(vgr.ObjectMeta.Finalizers, vgrReplicationFinalizer) {
-		logger.Info("removing finalizer from VolumeGroupReplication object", "Finalizer", vgrReplicationFinalizer)
+		logger.Info("removing finalizer from volumeGroupReplication object", "Finalizer", vgrReplicationFinalizer)
+		// Check if owner annotations are removed from the VGR resource
+		if vgr.Annotations[replicationv1alpha1.VolumeGroupReplicationContentNameAnnotation] != "" ||
+			vgr.Annotations[replicationv1alpha1.VolumeReplicationNameAnnotation] != "" {
+			return fmt.Errorf("failed to remove finalizer from volumeGroupReplication object"+
+				",dependent resources are not yet deleted (%s/%s)", vgr.Namespace, vgr.Name)
+		}
 		vgr.ObjectMeta.Finalizers = util.RemoveFromSlice(vgr.ObjectMeta.Finalizers, vgrReplicationFinalizer)
-		if err := r.Client.Update(context.TODO(), vgr); err != nil {
+		if err := client.Update(context.TODO(), vgr); err != nil {
 			return fmt.Errorf("failed to remove finalizer (%s) from VolumeGroupReplication resource"+
 				" (%s/%s), %w",
 				vgrReplicationFinalizer, vgr.Namespace, vgr.Name, err)
@@ -126,3 +133,35 @@ func (r *VolumeReplicationReconciler) removeFinalizerFromVGR(logger logr.Logger,
 
 	return nil
 }
+
+// AddFinalizerToVGRContent adds the VR, VGR finalizer on the VolumeGroupReplicationContent resource
+func AddFinalizerToVGRContent(client client.Client, logger logr.Logger, vgrContent *replicationv1alpha1.VolumeGroupReplicationContent,
+	replicationFinalizer string) error {
+	if !slices.Contains(vgrContent.ObjectMeta.Finalizers, replicationFinalizer) {
+		logger.Info("adding finalizer to volumeGroupReplicationContent object", "Finalizer", replicationFinalizer)
+		vgrContent.ObjectMeta.Finalizers = append(vgrContent.ObjectMeta.Finalizers, replicationFinalizer)
+		if err := client.Update(context.TODO(), vgrContent); err != nil {
+			return fmt.Errorf("failed to add finalizer (%s) to VolumeGroupReplicationContent resource"+
+				" (%s/%s) %w",
+				replicationFinalizer, vgrContent.Namespace, vgrContent.Name, err)
+		}
+	}
+
+	return nil
+}
+
+// RemoveFinalizerFromVGRContent removes the VR, VGR finalizer from the VolumeGroupReplicationContent instance.
+func RemoveFinalizerFromVGRContent(client client.Client, logger logr.Logger, vgrContent *replicationv1alpha1.VolumeGroupReplicationContent,
+	replicationFinalizer string) error {
+	if slices.Contains(vgrContent.ObjectMeta.Finalizers, replicationFinalizer) {
+		logger.Info("removing finalizer from volumeGroupReplicationContent object", "Finalizer", replicationFinalizer)
+		vgrContent.ObjectMeta.Finalizers = util.RemoveFromSlice(vgrContent.ObjectMeta.Finalizers, replicationFinalizer)
+		if err := client.Update(context.TODO(), vgrContent); err != nil {
+			return fmt.Errorf("failed to remove finalizer (%s) from VolumeGroupReplicationContent resource"+
+				" (%s/%s), %w",
+				replicationFinalizer, vgrContent.Namespace, vgrContent.Name, err)
+		}
+	}
+
+	return nil
+}