From 6685aa4fa2e5a522096c474b0268cadfaddb0b77 Mon Sep 17 00:00:00 2001 From: Joseph Date: Wed, 19 Aug 2026 13:18:13 -0700 Subject: [PATCH] e2e: add cacert tests using in-cluster minio with TLS Adds e2e coverage for BSL custom CA certificate support (OADP-641 / issue #2384). Deploys minio on-cluster with TLS using a locally-generated CA, configures a BSL pointing at it with the CA cert, and verifies: Positive test: - BSL becomes Available (Velero validates TLS using the CA) - AWS_CA_BUNDLE is set on the Velero deployment - velero-ca-bundle ConfigMap is created - A backup to the minio BSL completes successfully - Backup deletion via Velero CLI exercises the custom CA delete path Negative test: - BSL without CACert stays Unavailable against the same minio instance, confirming that the custom CA is actually required for validation New helpers in tests/e2e/lib/: - minio_helpers.go: TLS cert generation, minio Deployment/Service/Secret lifecycle, bucket creation via pod exec - backup.go: DeleteBackup for best-effort CR cleanup when Velero is gone Signed-off-by: Joseph --- tests/e2e/cacert_suite_test.go | 222 ++++++++++++++++++++++ tests/e2e/lib/backup.go | 10 + tests/e2e/lib/minio_helpers.go | 326 +++++++++++++++++++++++++++++++++ 3 files changed, 558 insertions(+) create mode 100644 tests/e2e/cacert_suite_test.go create mode 100644 tests/e2e/lib/minio_helpers.go diff --git a/tests/e2e/cacert_suite_test.go b/tests/e2e/cacert_suite_test.go new file mode 100644 index 00000000000..73538b1229a --- /dev/null +++ b/tests/e2e/cacert_suite_test.go @@ -0,0 +1,222 @@ +package e2e_test + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1" + "github.com/openshift/oadp-operator/tests/e2e/lib" +) + +var ( + cacertDpaCR *lib.DpaCustomResource + noCACertDPA *lib.DpaCustomResource +) + +var _ = ginkgo.Describe("BSL cacert with in-cluster minio", ginkgo.Ordered, ginkgo.Label("aws"), func() { + const ( + minioBSLSecretName = "minio-bsl-creds" + testBackupName = "cacert-minio-backup" + testNamespace = "cacert-minio-test-app" + ) + + var caPEM []byte + + // ── Setup ────────────────────────────────────────────────────────────────── + + ginkgo.BeforeAll(func(ctx ginkgo.SpecContext) { + // Initialise early so AfterEach can safely call Delete() even if BeforeAll fails. + cacertDpaCR = &lib.DpaCustomResource{ + Name: "ts-cacert-minio", + Namespace: namespace, + Client: runTimeClientForSuiteRun, + } + + // Clean up any resources left by a previously interrupted run. + _ = lib.DeleteBackup(runTimeClientForSuiteRun, namespace, testBackupName) + + log.Println("cacert: generating self-signed CA and server certificate") + var caKeyPEM []byte + var err error + caPEM, caKeyPEM, err = lib.GenerateSelfSignedCA() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "generating self-signed CA") + + dnsNames := []string{ + lib.MinioServiceName, + fmt.Sprintf("%s.%s", lib.MinioServiceName, namespace), + fmt.Sprintf("%s.%s.svc", lib.MinioServiceName, namespace), + fmt.Sprintf("%s.%s.svc.cluster.local", lib.MinioServiceName, namespace), + } + certPEM, keyPEM, err := lib.GenerateServerCert(caPEM, caKeyPEM, dnsNames) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "generating minio server certificate") + + log.Println("cacert: deploying minio with TLS") + minioURL, err := lib.DeployMinioWithTLS(ctx, kubernetesClientForSuiteRun, namespace, certPEM, keyPEM) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "deploying minio with TLS in namespace %s", namespace) + log.Printf("cacert: minio available at %s", minioURL) + + log.Printf("cacert: creating bucket %s", lib.MinioBucketName) + err = lib.CreateMinioBucket(ctx, kubernetesClientForSuiteRun, kubeConfig, namespace, lib.MinioBucketName) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "creating minio bucket %s", lib.MinioBucketName) + + log.Println("cacert: creating BSL credentials secret") + credsData := fmt.Sprintf("[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n", + lib.MinioAccessKey, lib.MinioSecretKey) + _, err = kubernetesClientForSuiteRun.CoreV1().Secrets(namespace).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: minioBSLSecretName, Namespace: namespace}, + Data: map[string][]byte{"cloud": []byte(credsData)}, + }, metav1.CreateOptions{}) + // Tolerate AlreadyExists: AfterAll may not have run if a previous run was killed. + if err != nil && !apierrors.IsAlreadyExists(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "creating BSL credentials secret %s", minioBSLSecretName) + } + + // kubevirt/hypershift plugins lack arm64-compatible images in some environments. + cacertDpaCR.BSLSecretName = minioBSLSecretName + cacertDpaCR.BSLProvider = dpaCR.BSLProvider + cacertDpaCR.BSLBucket = lib.MinioBucketName + cacertDpaCR.BSLBucketPrefix = "e2e" + cacertDpaCR.BSLCacert = caPEM + cacertDpaCR.BSLConfig = map[string]string{ + "s3Url": minioURL, + "s3ForcePathStyle": "true", + "region": "us-east-1", + } + cacertDpaCR.VeleroDefaultPlugins = []oadpv1alpha1.DefaultPlugin{ + oadpv1alpha1.DefaultPluginOpenShift, + oadpv1alpha1.DefaultPluginAWS, + } + cacertDpaCR.UnsupportedOverrides = dpaCR.UnsupportedOverrides + }) + + // ── Teardown ─────────────────────────────────────────────────────────────── + + ginkgo.AfterAll(func(ctx ginkgo.SpecContext) { + lib.DeleteMinioResources(ctx, kubernetesClientForSuiteRun, namespace) + _ = lib.DeleteSecret(kubernetesClientForSuiteRun, namespace, minioBSLSecretName) + }) + + ginkgo.AfterEach(func(ctx ginkgo.SpecContext) { + if !skipMustGather && ctx.SpecReport().Failed() { + _ = lib.RunMustGather(artifact_dir, cacertDpaCR.Client) + } + // Happy path: the positive It block exercised deletion via DeleteBackupRequest. + // Fallback: direct CR delete when the test failed before reaching that step. + if ctx.SpecReport().Failed() { + if err := lib.DeleteBackup(runTimeClientForSuiteRun, namespace, testBackupName); err != nil { + log.Printf("cacert: warning: could not delete backup CR %s: %v", testBackupName, err) + } + } + // Clean up whichever DPA was created this run (positive or negative test). + if noCACertDPA != nil { + if err := noCACertDPA.Delete(); err != nil { + log.Printf("cacert: warning: could not delete DPA %s: %v", noCACertDPA.Name, err) + } + noCACertDPA = nil + } + gomega.Expect(cacertDpaCR.Delete()).NotTo(gomega.HaveOccurred()) + gomega.Eventually(lib.VeleroIsDeleted(kubernetesClientForSuiteRun, namespace), 5*time.Minute, 5*time.Second).Should(gomega.BeTrue()) + }) + + // ── Tests ────────────────────────────────────────────────────────────────── + + ginkgo.It("BSL is Available and Velero gets AWS_CA_BUNDLE when using minio with custom TLS", func(ctx ginkgo.SpecContext) { + gomega.Expect(cacertDpaCR.CreateOrUpdate(cacertDpaCR.Build(lib.CSI))).NotTo(gomega.HaveOccurred()) + gomega.Eventually(cacertDpaCR.IsReconciledTrue(), 3*time.Minute, 5*time.Second).Should(gomega.BeTrue()) + gomega.Eventually(lib.VeleroPodIsRunning(kubernetesClientForSuiteRun, namespace), 3*time.Minute, 5*time.Second).Should(gomega.BeTrue()) + + // BSL Available means Velero successfully TLS-connected to minio using our CA. + log.Println("cacert: waiting for BSL to become Available") + gomega.Eventually(cacertDpaCR.BSLsAreAvailable(), 3*time.Minute, 5*time.Second).Should(gomega.BeTrue()) + + gomega.Expect(awsCABundleIsSet(ctx, "/etc/velero/ca-certs/ca-bundle.pem")).NotTo(gomega.HaveOccurred()) + + cm, err := kubernetesClientForSuiteRun.CoreV1().ConfigMaps(namespace).Get(ctx, "velero-ca-bundle", metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cm.Data).To(gomega.HaveKey("ca-bundle.pem")) + + log.Println("cacert: running backup via minio BSL") + gomega.Expect(lib.CreateNamespace(kubernetesClientForSuiteRun, testNamespace)).NotTo(gomega.HaveOccurred()) + defer func() { _ = lib.DeleteNamespace(kubernetesClientForSuiteRun, testNamespace) }() + + _, err = kubernetesClientForSuiteRun.CoreV1().ConfigMaps(testNamespace).Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cm", Namespace: testNamespace}, + Data: map[string]string{"key": "value"}, + }, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Expect(lib.CreateBackupForNamespaces(runTimeClientForSuiteRun, namespace, testBackupName, []string{testNamespace}, false, false)). + NotTo(gomega.HaveOccurred()) + gomega.Eventually(lib.IsBackupDone(runTimeClientForSuiteRun, namespace, testBackupName), 10*time.Minute, 10*time.Second). + Should(gomega.BeTrue()) + + succeeded, err := lib.IsBackupCompletedSuccessfully(kubernetesClientForSuiteRun, runTimeClientForSuiteRun, namespace, testBackupName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(succeeded).To(gomega.BeTrue(), "backup to minio with custom CA cert should complete successfully") + + // Deletion path: Velero must connect to minio via the custom CA to remove + // backup objects. The DPA stays up so Velero is still running — its controller + // must clear the backup finalizer, otherwise the CR gets stuck Terminating. + log.Println("cacert: deleting backup via minio BSL") + gomega.Expect(lib.DeleteVeleroBackupAndRestore( + runTimeClientForSuiteRun, kubernetesClientForSuiteRun, kubeConfig, + namespace, testBackupName, "", + )).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("BSL without CACert does not become Available against minio with self-signed TLS", func(ctx ginkgo.SpecContext) { + // Same minio, same bucket — but no CA cert supplied to the BSL. + // Velero cannot verify minio's self-signed cert, so the BSL must never become Available. + // AfterEach cleans up noCACertDPA alongside cacertDpaCR. + noCACertDPA = &lib.DpaCustomResource{ + Name: "ts-cacert-minio-nocert", + Namespace: namespace, + Client: runTimeClientForSuiteRun, + BSLSecretName: minioBSLSecretName, + BSLProvider: cacertDpaCR.BSLProvider, + BSLBucket: lib.MinioBucketName, + BSLBucketPrefix: "e2e-nocert", + BSLConfig: cacertDpaCR.BSLConfig, + VeleroDefaultPlugins: cacertDpaCR.VeleroDefaultPlugins, + UnsupportedOverrides: cacertDpaCR.UnsupportedOverrides, + // BSLCacert intentionally omitted + } + + gomega.Expect(noCACertDPA.CreateOrUpdate(noCACertDPA.Build(lib.CSI))).NotTo(gomega.HaveOccurred()) + gomega.Eventually(noCACertDPA.IsReconciledTrue(), 3*time.Minute, 5*time.Second).Should(gomega.BeTrue()) + gomega.Eventually(lib.VeleroPodIsRunning(kubernetesClientForSuiteRun, namespace), 3*time.Minute, 5*time.Second).Should(gomega.BeTrue()) + + gomega.Consistently(noCACertDPA.BSLsAreAvailable(), 90*time.Second, 10*time.Second).Should(gomega.BeFalse(), + "BSL without CA cert should not become Available when minio uses a self-signed certificate") + }) +}) + +// awsCABundleIsSet polls the Velero deployment until AWS_CA_BUNDLE equals wantPath. +func awsCABundleIsSet(ctx context.Context, wantPath string) error { + return wait.PollUntilContextTimeout(ctx, 5*time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + dep, err := lib.GetVeleroDeployment(kubernetesClientForSuiteRun, namespace) + if err != nil { + return false, err // surface permanent errors (RBAC, missing CRD) immediately + } + for _, c := range dep.Spec.Template.Spec.Containers { + if c.Name != "velero" { + continue + } + for _, env := range c.Env { + if env.Name == "AWS_CA_BUNDLE" && env.Value == wantPath { + return true, nil + } + } + } + return false, nil + }) +} diff --git a/tests/e2e/lib/backup.go b/tests/e2e/lib/backup.go index 98d7bff8f42..19329bbf025 100755 --- a/tests/e2e/lib/backup.go +++ b/tests/e2e/lib/backup.go @@ -377,6 +377,16 @@ func DeleteBackupRepository(c client.Client, namespace string, name string) erro return nil } +func DeleteBackup(c client.Client, namespace string, name string) error { + err := c.Delete(context.Background(), &velero.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + }) + if apierrors.IsNotFound(err) { + return nil + } + return err +} + // DeleteBackupRepositories deletes all BackupRepositories in the given namespace. func DeleteBackupRepositories(c client.Client, namespace string) error { log.Printf("Checking if backuprepository's exist in %s", namespace) diff --git a/tests/e2e/lib/minio_helpers.go b/tests/e2e/lib/minio_helpers.go new file mode 100644 index 00000000000..f37c2088d10 --- /dev/null +++ b/tests/e2e/lib/minio_helpers.go @@ -0,0 +1,326 @@ +package lib + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/utils/ptr" +) + +const ( + MinioDeploymentName = "minio-cacert-test" + MinioServiceName = "minio-cacert-test" + MinioTLSSecretName = "minio-tls-cacert-test" + MinioBucketName = "cacert-test-bucket" + MinioAccessKey = "minioadmin" + MinioSecretKey = "minioadmin" + minioPort = int32(9000) + + // minioImage pins a specific minio release. Update this when bumping minio. + // Use a digest or immutable tag to keep test runs deterministic. + minioImage = "docker.io/minio/minio:RELEASE.2025-04-22T22-12-26Z" +) + +// GenerateSelfSignedCA creates a CA certificate and private key. +func GenerateSelfSignedCA() (caPEM, caKeyPEM []byte, err error) { + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("generating CA key: %w", err) + } + + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "OADP E2E Test CA", + Organization: []string{"OADP E2E"}, + }, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + } + + caBytes, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, nil, fmt.Errorf("creating CA cert: %w", err) + } + + caPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caBytes}) + + caKeyBytes, err := x509.MarshalECPrivateKey(caKey) + if err != nil { + return nil, nil, fmt.Errorf("marshaling CA key: %w", err) + } + caKeyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: caKeyBytes}) + + return caPEM, caKeyPEM, nil +} + +// GenerateServerCert creates a TLS server certificate signed by the given CA. +func GenerateServerCert(caPEM, caKeyPEM []byte, dnsNames []string) (certPEM, keyPEM []byte, err error) { + if len(dnsNames) == 0 { + return nil, nil, fmt.Errorf("dnsNames must not be empty") + } + caBlock, _ := pem.Decode(caPEM) + if caBlock == nil { + return nil, nil, fmt.Errorf("failed to decode CA PEM") + } + caCert, err := x509.ParseCertificate(caBlock.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("parsing CA cert: %w", err) + } + + caKeyBlock, _ := pem.Decode(caKeyPEM) + if caKeyBlock == nil { + return nil, nil, fmt.Errorf("failed to decode CA key PEM") + } + caKey, err := x509.ParseECPrivateKey(caKeyBlock.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("parsing CA key: %w", err) + } + + serverKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("generating server key: %w", err) + } + + serverTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: dnsNames[0]}, + DNSNames: dnsNames, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + + certBytes, err := x509.CreateCertificate(rand.Reader, serverTemplate, caCert, &serverKey.PublicKey, caKey) + if err != nil { + return nil, nil, fmt.Errorf("creating server cert: %w", err) + } + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certBytes}) + + keyBytes, err := x509.MarshalECPrivateKey(serverKey) + if err != nil { + return nil, nil, fmt.Errorf("marshaling server key: %w", err) + } + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyBytes}) + + return certPEM, keyPEM, nil +} + +// DeployMinioWithTLS deploys minio with TLS in the given namespace and waits for it to be ready. +// Returns the in-cluster HTTPS service URL. +func DeployMinioWithTLS(ctx context.Context, c *kubernetes.Clientset, namespace string, certPEM, keyPEM []byte) (string, error) { + // Delete any leftover TLS secret from a previous interrupted run before creating + // a fresh one; a stale secret would cause Velero to fail TLS validation because + // the serving cert would not match the newly generated CA. + if err := c.CoreV1().Secrets(namespace).Delete(ctx, MinioTLSSecretName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return "", fmt.Errorf("deleting stale minio TLS secret: %w", err) + } + // Minio expects filenames public.crt and private.key in --certs-dir + tlsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: MinioTLSSecretName, + Namespace: namespace, + }, + Data: map[string][]byte{ + "public.crt": certPEM, + "private.key": keyPEM, + }, + } + if _, err := c.CoreV1().Secrets(namespace).Create(ctx, tlsSecret, metav1.CreateOptions{}); err != nil { + return "", fmt.Errorf("creating minio TLS secret: %w", err) + } + + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: MinioDeploymentName, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": MinioDeploymentName}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": MinioDeploymentName}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "minio", + Image: minioImage, + Args: []string{"server", "/data", "--certs-dir", "/certs"}, + Env: []corev1.EnvVar{ + {Name: "MINIO_ROOT_USER", Value: MinioAccessKey}, + {Name: "MINIO_ROOT_PASSWORD", Value: MinioSecretKey}, + }, + Ports: []corev1.ContainerPort{ + {ContainerPort: minioPort, Protocol: corev1.ProtocolTCP}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "certs", MountPath: "/certs", ReadOnly: true}, + {Name: "data", MountPath: "/data"}, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt32(minioPort), + }, + }, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: MinioTLSSecretName, + }, + }, + }, + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + }, + } + if _, err := c.AppsV1().Deployments(namespace).Create(ctx, dep, metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("creating minio deployment: %w", err) + } + // Deployment exists from a previous run. The TLS Secret was already replaced + // above, but the running pod won't pick up the new cert without a restart. + // Force a rollout by annotating the pod template (same as kubectl rollout restart). + patch := fmt.Sprintf(`{"spec":{"template":{"metadata":{"annotations":{"kubectl.kubernetes.io/restartedAt":%q}}}}}`, + time.Now().UTC().Format(time.RFC3339)) + if _, err := c.AppsV1().Deployments(namespace).Patch(ctx, MinioDeploymentName, + types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { + return "", fmt.Errorf("restarting minio deployment: %w", err) + } + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: MinioServiceName, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"app": MinioDeploymentName}, + Ports: []corev1.ServicePort{ + {Port: minioPort, Protocol: corev1.ProtocolTCP, TargetPort: intstr.FromInt32(minioPort)}, + }, + }, + } + if _, err := c.CoreV1().Services(namespace).Create(ctx, svc, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("creating minio service: %w", err) + } + + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + d, err := c.AppsV1().Deployments(namespace).Get(ctx, MinioDeploymentName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + return d.Status.ReadyReplicas >= 1, nil + }); err != nil { + return "", fmt.Errorf("waiting for minio to be ready: %w", err) + } + + return fmt.Sprintf("https://%s.%s.svc:%d", MinioServiceName, namespace, minioPort), nil +} + +// CreateMinioBucket execs into the running minio pod and uses the bundled mc binary to +// create the bucket. Using --insecure here is intentional — TLS validation is the job of +// Velero (the system under test), not of the bucket-creation helper. +func CreateMinioBucket(ctx context.Context, c *kubernetes.Clientset, cfg *rest.Config, namespace, bucket string) error { + // Wait until a minio pod is Running + var podName string + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + pods, err := c.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=" + MinioDeploymentName, + FieldSelector: "status.phase=Running", + }) + if err != nil || len(pods.Items) == 0 { + return false, nil + } + for _, p := range pods.Items { + if p.Status.Phase == corev1.PodRunning { + podName = p.Name + return true, nil + } + } + return false, nil + }); err != nil { + return fmt.Errorf("waiting for minio pod to be Running: %w", err) + } + + // --insecure is a global mc flag; pass it before each subcommand so both + // alias-set and mb skip TLS verification when connecting to localhost. + cmd := []string{"sh", "-c", fmt.Sprintf( + "mc --insecure alias set local https://localhost:%d %s %s && mc --insecure mb --ignore-existing local/%s", + minioPort, MinioAccessKey, MinioSecretKey, bucket, + )} + + req := c.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Command: cmd, + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(cfg, "POST", req.URL()) + if err != nil { + return fmt.Errorf("creating executor: %w", err) + } + + var stdout, stderr bytes.Buffer + if err := executor.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: &stdout, Stderr: &stderr}); err != nil { + return fmt.Errorf("exec mc mb: %w (stdout: %s, stderr: %s)", err, stdout.String(), stderr.String()) + } + return nil +} + +// DeleteMinioResources removes the minio deployment, service, and TLS secret. +func DeleteMinioResources(ctx context.Context, c *kubernetes.Clientset, namespace string) { + background := metav1.DeletePropagationBackground + deleteOpts := metav1.DeleteOptions{PropagationPolicy: &background} + _ = c.AppsV1().Deployments(namespace).Delete(ctx, MinioDeploymentName, deleteOpts) + _ = c.CoreV1().Services(namespace).Delete(ctx, MinioServiceName, metav1.DeleteOptions{}) + _ = c.CoreV1().Secrets(namespace).Delete(ctx, MinioTLSSecretName, metav1.DeleteOptions{}) +}