Enhance tests (part3)

.github/workflows/e2e-multiple-k8s-clusters.yaml

@@ -24,11 +24,13 @@ jobs:
       matrix:
         package:
           - replication
+          - replicationfailure
           - changetostandalone
           - changetoprimary
           - changetosecondary
     #runs-on: "ubuntu-22.04"
     runs-on: mantle_large_runner_16core
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5

.github/workflows/e2e.yaml

@@ -14,6 +14,7 @@ on:
 jobs:
   build:
     runs-on: "ubuntu-22.04"
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5

internal/ceph/ceph.go

@@ -60,8 +60,15 @@ func NewCephCmd() CephCmd {
 	}
 }
 
-func NewCephCmdWithTools(namespace string) CephCmd {
+func NewCephCmdWithToolsAndCustomKubectl(kubectl []string, namespace string) CephCmd {
 	return &cephCmdImpl{
-		command: newCommandTools(namespace),
+		command: newCommandTools(kubectl, namespace),
+	}
+}
+
+func NewCephCmdWithTools(namespace string) CephCmd {
+	if len(envKubectlPath) == 0 {
+		panic("KUBECTL environment variable is not set")
 	}
+	return NewCephCmdWithToolsAndCustomKubectl([]string{envKubectlPath}, namespace)
 }

internal/ceph/command_using_tools.go

@@ -7,28 +7,28 @@ import (
 )
 
 var (
-	kubectlPath = os.Getenv("KUBECTL")
+	envKubectlPath = os.Getenv("KUBECTL")
 )
 
 type commandToolsImpl struct {
+	kubectl   []string
 	namespace string
 }
 
-func newCommandTools(namespace string) command {
+func newCommandTools(kubectl []string, namespace string) command {
 	return &commandToolsImpl{
+		kubectl:   kubectl,
 		namespace: namespace,
 	}
 }
 
 func (c *commandToolsImpl) execute(cephCommand ...string) ([]byte, error) {
-	if len(kubectlPath) == 0 {
-		panic("KUBECTL environment variable is not set")
-	}
-
 	var stdout bytes.Buffer
-	arg := []string{"exec", "-n", c.namespace, "deploy/rook-ceph-tools", "--"}
+	arg := []string{}
+	arg = append(arg, c.kubectl...)
+	arg = append(arg, "exec", "-n", c.namespace, "deploy/rook-ceph-tools", "--")
 	arg = append(arg, cephCommand...)
-	command := exec.Command(kubectlPath, arg...)
+	command := exec.Command(arg[0], arg[1:]...)
 	command.Stdout = &stdout
 
 	err := command.Run()

internal/controller/mantlebackup_controller.go

@@ -150,7 +150,7 @@ func (r *MantleBackupReconciler) removeRBDSnapshot(ctx context.Context, poolName
 	rmErr := r.ceph.RBDSnapRm(poolName, imageName, snapshotName)
 	if rmErr != nil {
 		_, findErr := ceph.FindRBDSnapshot(r.ceph, poolName, imageName, snapshotName)
-		if findErr != nil && errors.Is(findErr, ceph.ErrSnapshotNotFound) {
+		if findErr == nil || !errors.Is(findErr, ceph.ErrSnapshotNotFound) {
 			err := errors.Join(rmErr, findErr)
 			logger.Error(err, "failed to remove rbd snapshot", "poolName", poolName, "imageName", imageName, "snapshotName", snapshotName)
 			return fmt.Errorf("failed to remove rbd snapshot: %w", err)
@@ -487,7 +487,7 @@ func (r *MantleBackupReconciler) reconcileAsSecondary(ctx context.Context, backu
 		}
 	}
 
-	return r.secondaryCleanup(ctx, backup)
+	return r.secondaryCleanup(ctx, backup, true)
 }
 
 func scheduleExpire(_ context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) {
@@ -751,7 +751,7 @@ func (r *MantleBackupReconciler) finalizeSecondary(
 		return ctrl.Result{}, nil
 	}
 
-	result, err := r.secondaryCleanup(ctx, backup)
+	result, err := r.secondaryCleanup(ctx, backup, false)
 	if err != nil || result != (ctrl.Result{}) {
 		return result, err
 	}
@@ -1045,7 +1045,7 @@ func (r *MantleBackupReconciler) createOrUpdateExportDataPVC(ctx context.Context
 	pvcSize.Mul(2)
 
 	var pvc corev1.PersistentVolumeClaim
-	pvc.SetName(makeExportDataPVCName(target))
+	pvc.SetName(MakeExportDataPVCName(target))
 	pvc.SetNamespace(r.managedCephClusterID)
 	_, err := ctrl.CreateOrUpdate(ctx, r.Client, &pvc, func() error {
 		labels := pvc.GetLabels()
@@ -1070,35 +1070,35 @@ func (r *MantleBackupReconciler) createOrUpdateExportDataPVC(ctx context.Context
 	return err
 }
 
-func makeExportJobName(target *mantlev1.MantleBackup) string {
+func MakeExportJobName(target *mantlev1.MantleBackup) string {
 	return MantleExportJobPrefix + string(target.GetUID())
 }
 
-func makeUploadJobName(target *mantlev1.MantleBackup) string {
+func MakeUploadJobName(target *mantlev1.MantleBackup) string {
 	return MantleUploadJobPrefix + string(target.GetUID())
 }
 
-func makeExportDataPVCName(target *mantlev1.MantleBackup) string {
+func MakeExportDataPVCName(target *mantlev1.MantleBackup) string {
 	return MantleExportDataPVCPrefix + string(target.GetUID())
 }
 
-func makeObjectNameOfExportedData(name, uid string) string {
+func MakeObjectNameOfExportedData(name, uid string) string {
 	return fmt.Sprintf("%s-%s.bin", name, uid)
 }
 
-func makeImportJobName(target *mantlev1.MantleBackup) string {
+func MakeImportJobName(target *mantlev1.MantleBackup) string {
 	return MantleImportJobPrefix + string(target.GetUID())
 }
 
-func makeDiscardJobName(target *mantlev1.MantleBackup) string {
+func MakeDiscardJobName(target *mantlev1.MantleBackup) string {
 	return MantleDiscardJobPrefix + string(target.GetUID())
 }
 
-func makeDiscardPVCName(target *mantlev1.MantleBackup) string {
+func MakeDiscardPVCName(target *mantlev1.MantleBackup) string {
 	return MantleDiscardPVCPrefix + string(target.GetUID())
 }
 
-func makeDiscardPVName(target *mantlev1.MantleBackup) string {
+func MakeDiscardPVName(target *mantlev1.MantleBackup) string {
 	return MantleDiscardPVPrefix + string(target.GetUID())
 }
 
@@ -1114,7 +1114,7 @@ func (r *MantleBackupReconciler) createOrUpdateExportJob(ctx context.Context, ta
 	}
 
 	var job batchv1.Job
-	job.SetName(makeExportJobName(target))
+	job.SetName(MakeExportJobName(target))
 	job.SetNamespace(r.managedCephClusterID)
 	if _, err := ctrl.CreateOrUpdate(ctx, r.Client, &job, func() error {
 		labels := job.GetLabels()
@@ -1204,7 +1204,7 @@ func (r *MantleBackupReconciler) createOrUpdateExportJob(ctx context.Context, ta
 				Name: "volume-to-store",
 				VolumeSource: corev1.VolumeSource{
 					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-						ClaimName: makeExportDataPVCName(target),
+						ClaimName: MakeExportDataPVCName(target),
 					},
 				},
 			},
@@ -1261,7 +1261,7 @@ func (r *MantleBackupReconciler) checkIfExportJobIsCompleted(
 	if err := r.Client.Get(
 		ctx,
 		types.NamespacedName{
-			Name:      makeExportJobName(target),
+			Name:      MakeExportJobName(target),
 			Namespace: r.managedCephClusterID,
 		},
 		&job,
@@ -1279,7 +1279,7 @@ func (r *MantleBackupReconciler) checkIfExportJobIsCompleted(
 
 func (r *MantleBackupReconciler) createOrUpdateExportDataUploadJob(ctx context.Context, target *mantlev1.MantleBackup) error {
 	var job batchv1.Job
-	job.SetName(makeUploadJobName(target))
+	job.SetName(MakeUploadJobName(target))
 	job.SetNamespace(r.managedCephClusterID)
 	if _, err := ctrl.CreateOrUpdate(ctx, r.Client, &job, func() error {
 		labels := job.GetLabels()
@@ -1313,7 +1313,7 @@ func (r *MantleBackupReconciler) createOrUpdateExportDataUploadJob(ctx context.C
 				Env: []corev1.EnvVar{
 					{
 						Name:  "OBJ_NAME",
-						Value: makeObjectNameOfExportedData(target.GetName(), string(target.GetUID())),
+						Value: MakeObjectNameOfExportedData(target.GetName(), string(target.GetUID())),
 					},
 					{
 						Name:  "BUCKET_NAME",
@@ -1374,7 +1374,7 @@ func (r *MantleBackupReconciler) createOrUpdateExportDataUploadJob(ctx context.C
 				Name: "volume-to-store",
 				VolumeSource: corev1.VolumeSource{
 					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-						ClaimName: makeExportDataPVCName(target),
+						ClaimName: MakeExportDataPVCName(target),
 					},
 				},
 			},
@@ -1523,7 +1523,7 @@ func (r *MantleBackupReconciler) isExportDataAlreadyUploaded(
 ) (ctrl.Result, error) {
 	uploaded, err := r.objectStorageClient.Exists(
 		ctx,
-		makeObjectNameOfExportedData(target.GetName(), target.GetAnnotations()[annotRemoteUID]),
+		MakeObjectNameOfExportedData(target.GetName(), target.GetAnnotations()[annotRemoteUID]),
 	)
 	if err != nil {
 		return ctrl.Result{}, err
@@ -1604,7 +1604,7 @@ func (r *MantleBackupReconciler) createOrUpdateDiscardPV(
 	targetPV *corev1.PersistentVolume,
 ) error {
 	var pv corev1.PersistentVolume
-	pv.SetName(makeDiscardPVName(backup))
+	pv.SetName(MakeDiscardPVName(backup))
 	_, err := ctrl.CreateOrUpdate(ctx, r.Client, &pv, func() error {
 		labels := pv.GetLabels()
 		if labels == nil {
@@ -1650,7 +1650,7 @@ func (r *MantleBackupReconciler) createOrUpdateDiscardPVC(
 	targetPVC *corev1.PersistentVolumeClaim,
 ) error {
 	var pvc corev1.PersistentVolumeClaim
-	pvc.SetName(makeDiscardPVCName(backup))
+	pvc.SetName(MakeDiscardPVCName(backup))
 	pvc.SetNamespace(r.managedCephClusterID)
 	_, err := ctrl.CreateOrUpdate(ctx, r.Client, &pvc, func() error {
 		labels := pvc.GetLabels()
@@ -1665,7 +1665,7 @@ func (r *MantleBackupReconciler) createOrUpdateDiscardPVC(
 		pvc.Spec.StorageClassName = &storageClassName
 		pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}
 		pvc.Spec.Resources = targetPVC.Spec.Resources
-		pvc.Spec.VolumeName = makeDiscardPVName(backup)
+		pvc.Spec.VolumeName = MakeDiscardPVName(backup)
 
 		volumeMode := corev1.PersistentVolumeBlock
 		pvc.Spec.VolumeMode = &volumeMode
@@ -1680,7 +1680,7 @@ func (r *MantleBackupReconciler) createOrUpdateDiscardJob(
 	backup *mantlev1.MantleBackup,
 ) error {
 	var job batchv1.Job
-	job.SetName(makeDiscardJobName(backup))
+	job.SetName(MakeDiscardJobName(backup))
 	job.SetNamespace(r.managedCephClusterID)
 	_, err := ctrl.CreateOrUpdate(ctx, r.Client, &job, func() error {
 		labels := job.GetLabels()
@@ -1730,7 +1730,7 @@ blkdiscard /dev/discard-rbd
 				Name: "discard-rbd",
 				VolumeSource: corev1.VolumeSource{
 					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-						ClaimName: makeDiscardPVCName(backup),
+						ClaimName: MakeDiscardPVCName(backup),
 					},
 				},
 			},
@@ -1745,7 +1745,7 @@ func (r *MantleBackupReconciler) hasDiscardJobCompleted(ctx context.Context, bac
 	var job batchv1.Job
 	if err := r.Client.Get(
 		ctx,
-		types.NamespacedName{Name: makeDiscardJobName(backup), Namespace: r.managedCephClusterID},
+		types.NamespacedName{Name: MakeDiscardJobName(backup), Namespace: r.managedCephClusterID},
 		&job,
 	); err != nil {
 		if aerrors.IsNotFound(err) {
@@ -1769,7 +1769,7 @@ func (r *MantleBackupReconciler) reconcileImportJob(
 	if err := r.Client.Get(
 		ctx,
 		types.NamespacedName{
-			Name:      makeImportJobName(backup),
+			Name:      MakeImportJobName(backup),
 			Namespace: r.managedCephClusterID,
 		},
 		&job,
@@ -1819,7 +1819,7 @@ func (r *MantleBackupReconciler) createOrUpdateImportJob(
 ) error {
 	var job batchv1.Job
 
-	job.SetName(makeImportJobName(backup))
+	job.SetName(MakeImportJobName(backup))
 	job.SetNamespace(r.managedCephClusterID)
 
 	_, err := ctrl.CreateOrUpdate(ctx, r.Client, &job, func() error {
@@ -1876,7 +1876,7 @@ func (r *MantleBackupReconciler) createOrUpdateImportJob(
 				},
 				{
 					Name:  "OBJ_NAME",
-					Value: makeObjectNameOfExportedData(backup.GetName(), backup.GetAnnotations()[annotRemoteUID]),
+					Value: MakeObjectNameOfExportedData(backup.GetName(), backup.GetAnnotations()[annotRemoteUID]),
 				},
 				{
 					Name:  "BUCKET_NAME",
@@ -2028,7 +2028,7 @@ func (r *MantleBackupReconciler) primaryCleanup(
 	propagationPolicy := metav1.DeletePropagationBackground
 
 	var exportJob batchv1.Job
-	exportJob.SetName(makeExportJobName(target))
+	exportJob.SetName(MakeExportJobName(target))
 	exportJob.SetNamespace(r.managedCephClusterID)
 	if err := r.Client.Delete(ctx, &exportJob, &client.DeleteOptions{
 		PropagationPolicy: &propagationPolicy,
@@ -2037,7 +2037,7 @@ func (r *MantleBackupReconciler) primaryCleanup(
 	}
 
 	var uploadJob batchv1.Job
-	uploadJob.SetName(makeUploadJobName(target))
+	uploadJob.SetName(MakeUploadJobName(target))
 	uploadJob.SetNamespace(r.managedCephClusterID)
 	if err := r.Client.Delete(ctx, &uploadJob, &client.DeleteOptions{
 		PropagationPolicy: &propagationPolicy,
@@ -2046,7 +2046,7 @@ func (r *MantleBackupReconciler) primaryCleanup(
 	}
 
 	var exportDataPVC corev1.PersistentVolumeClaim
-	exportDataPVC.SetName(makeExportDataPVCName(target))
+	exportDataPVC.SetName(MakeExportDataPVCName(target))
 	exportDataPVC.SetNamespace(r.managedCephClusterID)
 	if err := r.Client.Delete(ctx, &exportDataPVC); err != nil && !aerrors.IsNotFound(err) {
 		return ctrl.Result{}, fmt.Errorf("failed to delete export data PVC: %w", err)
@@ -2092,6 +2092,7 @@ func (r *MantleBackupReconciler) primaryCleanup(
 func (r *MantleBackupReconciler) secondaryCleanup(
 	ctx context.Context,
 	target *mantlev1.MantleBackup,
+	deleteExportData bool,
 ) (ctrl.Result, error) {
 	diffFrom, ok := target.GetAnnotations()[annotDiffFrom]
 	if ok {
@@ -2112,7 +2113,7 @@ func (r *MantleBackupReconciler) secondaryCleanup(
 	propagationPolicy := metav1.DeletePropagationBackground
 
 	var discardJob batchv1.Job
-	discardJob.SetName(makeDiscardJobName(target))
+	discardJob.SetName(MakeDiscardJobName(target))
 	discardJob.SetNamespace(r.managedCephClusterID)
 	if err := r.Client.Delete(ctx, &discardJob, &client.DeleteOptions{
 		PropagationPolicy: &propagationPolicy,
@@ -2121,7 +2122,7 @@ func (r *MantleBackupReconciler) secondaryCleanup(
 	}
 
 	var importJob batchv1.Job
-	importJob.SetName(makeImportJobName(target))
+	importJob.SetName(MakeImportJobName(target))
 	importJob.SetNamespace(r.managedCephClusterID)
 	if err := r.Client.Delete(ctx, &importJob, &client.DeleteOptions{
 		PropagationPolicy: &propagationPolicy,
@@ -2130,24 +2131,26 @@ func (r *MantleBackupReconciler) secondaryCleanup(
 	}
 
 	var discardPVC corev1.PersistentVolumeClaim
-	discardPVC.SetName(makeDiscardPVCName(target))
+	discardPVC.SetName(MakeDiscardPVCName(target))
 	discardPVC.SetNamespace(r.managedCephClusterID)
 	if err := r.Client.Delete(ctx, &discardPVC); err != nil && !aerrors.IsNotFound(err) {
 		return ctrl.Result{}, fmt.Errorf("failed to delete discard PVC: %w", err)
 	}
 
 	var discardPV corev1.PersistentVolume
-	discardPV.SetName(makeDiscardPVName(target))
+	discardPV.SetName(MakeDiscardPVName(target))
 	discardPV.SetNamespace(r.managedCephClusterID)
 	if err := r.Client.Delete(ctx, &discardPV); err != nil && !aerrors.IsNotFound(err) {
 		return ctrl.Result{}, fmt.Errorf("failed to delete discard PV: %w", err)
 	}
 
-	if err := r.objectStorageClient.Delete(
-		ctx,
-		makeObjectNameOfExportedData(target.GetName(), target.GetAnnotations()[annotRemoteUID]),
-	); err != nil {
-		return ctrl.Result{}, fmt.Errorf("failed to delete exported data in the object storage: %w", err)
+	if deleteExportData {
+		if err := r.objectStorageClient.Delete(
+			ctx,
+			MakeObjectNameOfExportedData(target.GetName(), target.GetAnnotations()[annotRemoteUID]),
+		); err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed to delete exported data in the object storage: %w", err)
+		}
 	}
 
 	delete(target.GetAnnotations(), annotDiffFrom)