[Feat] Volume mount service secrets on workloads

internal/controller/reconcile-capapplicationversion.go

@@ -48,7 +48,9 @@ type DeploymentParameters struct {
 	CAV             *v1alpha1.CAPApplicationVersion
 	OwnerRef        *metav1.OwnerReference
 	WorkloadDetails v1alpha1.WorkloadDetails
-	VCAPSecretName  string
+	EnvFrom         []corev1.EnvFromSource
+	Volumes         []corev1.Volume
+	VolumeMounts    []corev1.VolumeMount
 }
 
 func (c *Controller) reconcileCAPApplicationVersion(ctx context.Context, item QueueItem, attempts int) (*ReconcileResult, error) {
@@ -270,6 +272,20 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli
 		AnnotationIstioSidecarInject: "false",
 	})
 
+	// Get ServiceInfos for consumed BTP services
+	consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services)
+
+	var envFrom []corev1.EnvFromSource
+	var serviceSecretVolumeMounts []corev1.VolumeMount
+	var serviceSecretVolumes []corev1.Volume
+
+	if useVolumeMounts(workload.JobDefinition.Env) {
+		serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos)
+		serviceSecretVolumes = getVolumes(consumedServiceInfos)
+	} else {
+		envFrom = getEnvFrom(vcapSecretName)
+	}
+
 	return &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        getContentJobName(workload.Name, cav),
@@ -297,15 +313,15 @@ func newContentDeploymentJob(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPAppli
 							Env: append([]corev1.EnvVar{
 								{Name: EnvCAPOpAppVersion, Value: cav.Spec.Version},
 							}, workload.JobDefinition.Env...),
-							EnvFrom:         getEnvFrom(vcapSecretName),
-							VolumeMounts:    workload.JobDefinition.VolumeMounts,
+							EnvFrom:         envFrom,
+							VolumeMounts:    append(workload.JobDefinition.VolumeMounts, serviceSecretVolumeMounts...),
 							Resources:       workload.JobDefinition.Resources,
 							SecurityContext: workload.JobDefinition.SecurityContext,
 						},
 					},
 					SecurityContext:           workload.JobDefinition.PodSecurityContext,
 					ServiceAccountName:        workload.JobDefinition.ServiceAccountName,
-					Volumes:                   workload.JobDefinition.Volumes,
+					Volumes:                   append(workload.JobDefinition.Volumes, serviceSecretVolumes...),
 					ImagePullSecrets:          convertToLocalObjectReferences(cav.Spec.RegistrySecrets),
 					RestartPolicy:             corev1.RestartPolicyOnFailure,
 					NodeSelector:              workload.JobDefinition.NodeSelector,
@@ -557,12 +573,25 @@ func (c *Controller) updateDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1
 
 // newDeployment creates a new generic Deployment for a CAV resource based on the type. It also sets the appropriate OwnerReferences.
 func newDeployment(ca *v1alpha1.CAPApplication, cav *v1alpha1.CAPApplicationVersion, workload *v1alpha1.WorkloadDetails, ownerRef metav1.OwnerReference, vcapSecretName string) *appsv1.Deployment {
+	// Get ServiceInfos for consumed BTP services
+	consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), ca.Spec.BTP.Services)
+
 	params := &DeploymentParameters{
 		CA:              ca,
 		CAV:             cav,
 		OwnerRef:        &ownerRef,
 		WorkloadDetails: *workload,
-		VCAPSecretName:  vcapSecretName,
+	}
+
+	if useVolumeMounts(workload.DeploymentDefinition.Env) {
+		if workload.DeploymentDefinition.Type == v1alpha1.DeploymentCAP {
+			params.VolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix)
+		} else {
+			params.VolumeMounts = getVolumeMounts(consumedServiceInfos)
+		}
+		params.Volumes = getVolumes(consumedServiceInfos)
+	} else {
+		params.EnvFrom = getEnvFrom(vcapSecretName)
 	}
 
 	return createDeployment(params)
@@ -597,7 +626,7 @@ func createDeployment(params *DeploymentParameters) *appsv1.Deployment {
 					ImagePullSecrets:          convertToLocalObjectReferences(params.CAV.Spec.RegistrySecrets),
 					Containers:                getContainer(params),
 					ServiceAccountName:        params.WorkloadDetails.DeploymentDefinition.ServiceAccountName,
-					Volumes:                   params.WorkloadDetails.DeploymentDefinition.Volumes,
+					Volumes:                   append(params.WorkloadDetails.DeploymentDefinition.Volumes, params.Volumes...),
 					SecurityContext:           params.WorkloadDetails.DeploymentDefinition.PodSecurityContext,
 					NodeSelector:              params.WorkloadDetails.DeploymentDefinition.NodeSelector,
 					NodeName:                  params.WorkloadDetails.DeploymentDefinition.NodeName,
@@ -618,8 +647,8 @@ func getContainer(params *DeploymentParameters) []corev1.Container {
 		ImagePullPolicy: params.WorkloadDetails.DeploymentDefinition.ImagePullPolicy,
 		Command:         params.WorkloadDetails.DeploymentDefinition.Command,
 		Env:             getEnv(params),
-		EnvFrom:         getEnvFrom(params.VCAPSecretName),
-		VolumeMounts:    params.WorkloadDetails.DeploymentDefinition.VolumeMounts,
+		EnvFrom:         params.EnvFrom,
+		VolumeMounts:    append(params.WorkloadDetails.DeploymentDefinition.VolumeMounts, params.VolumeMounts...),
 		LivenessProbe:   params.WorkloadDetails.DeploymentDefinition.LivenessProbe,
 		ReadinessProbe:  params.WorkloadDetails.DeploymentDefinition.ReadinessProbe,
 		Resources:       params.WorkloadDetails.DeploymentDefinition.Resources,

internal/controller/reconcile-captenantoperation.go

@@ -391,11 +391,26 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c
 		return nil, fmt.Errorf("could not find workload %s in %s %s.%s", step.Name, v1alpha1.CAPApplicationVersionKind, relatedResources.CAPApplicationVersion.Namespace, relatedResources.CAPApplicationVersion.Name)
 	}
 
-	// create VCAP secret from consumed BTP services
 	consumedServiceInfos := getConsumedServiceInfos(getConsumedServiceMap(workload.ConsumedBTPServices), relatedResources.CAPApplication.Spec.BTP.Services)
-	vcapSecretName, err := createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient)
-	if err != nil {
-		return nil, err
+
+	var envFrom []corev1.EnvFromSource
+	var serviceSecretVolumeMounts []corev1.VolumeMount
+	var serviceSecretVolumes []corev1.Volume
+
+	if (workload.JobDefinition != nil && useVolumeMounts(workload.JobDefinition.Env)) || (workload.DeploymentDefinition != nil && useVolumeMounts(workload.DeploymentDefinition.Env)) {
+		if ctop.Spec.Steps[*ctop.Status.CurrentStep-1].Type == v1alpha1.JobTenantOperation {
+			serviceSecretVolumeMounts = getCAPVolumeMounts(consumedServiceInfos, CDSVolMountPrefix)
+		} else {
+			serviceSecretVolumeMounts = getVolumeMounts(consumedServiceInfos)
+		}
+		serviceSecretVolumes = getVolumes(consumedServiceInfos)
+	} else {
+		// create VCAP secret from consumed BTP services
+		vcapSecretName, err := createVCAPSecret(ctop.Name+"-"+strings.ToLower(workload.Name), ctop.Namespace, *metav1.NewControllerRef(ctop, v1alpha1.SchemeGroupVersion.WithKind(v1alpha1.CAPTenantOperationKind)), consumedServiceInfos, c.kubeClient)
+		if err != nil {
+			return nil, err
+		}
+		envFrom = getEnvFrom(vcapSecretName)
 	}
 
 	annotations := copyMaps(workload.Annotations, map[string]string{
@@ -420,9 +435,11 @@ func (c *Controller) initiateJobForCAPTenantOperationStep(ctx context.Context, c
 		namePrefix:        relatedResources.CAPTenant.Name + "-" + workload.Name + "-",
 		labels:            labels,
 		annotations:       annotations,
-		envFromVCAPSecret: getEnvFrom(vcapSecretName),
+		envFromVCAPSecret: envFrom,
 		imagePullSecrets:  convertToLocalObjectReferences(relatedResources.CAPApplicationVersion.Spec.RegistrySecrets),
 		version:           relatedResources.CAPApplicationVersion.Spec.Version,
+		volumeMounts:      serviceSecretVolumeMounts,
+		volumes:           serviceSecretVolumes,
 	}
 
 	var job *batchv1.Job
@@ -454,6 +471,8 @@ type jobCreateParams struct {
 	imagePullSecrets  []corev1.LocalObjectReference
 	version           string
 	xsuaaInstanceName string
+	volumes           []corev1.Volume
+	volumeMounts      []corev1.VolumeMount
 }
 
 func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha1.CAPTenantOperation, workload *v1alpha1.WorkloadDetails, params *jobCreateParams) (*batchv1.Job, error) {
@@ -496,7 +515,7 @@ func (c *Controller) createTenantOperationJob(ctx context.Context, ctop *v1alpha
 					RestartPolicy:             corev1.RestartPolicyNever,
 					ImagePullSecrets:          params.imagePullSecrets,
 					Containers:                getContainers(payload, ctop, derivedWorkload, workload, params),
-					Volumes:                   derivedWorkload.volumes,
+					Volumes:                   append(derivedWorkload.volumes, params.volumes...),
 					ServiceAccountName:        derivedWorkload.serviceAccountName,
 					SecurityContext:           derivedWorkload.podSecurityContext,
 					NodeSelector:              derivedWorkload.nodeSelector,
@@ -526,7 +545,7 @@ func getContainers(payload []byte, ctop *v1alpha1.CAPTenantOperation, derivedWor
 			{Name: EnvCAPOpAppVersion, Value: params.version}, {Name: EnvCAPOpTenantID, Value: ctop.Spec.TenantId}, {Name: EnvCAPOpTenantOperation, Value: string(ctop.Spec.Operation)}, {Name: EnvCAPOpTenantSubDomain, Value: string(ctop.Spec.SubDomain)},
 		}, derivedWorkload.env...),
 		EnvFrom:         params.envFromVCAPSecret,
-		VolumeMounts:    derivedWorkload.volumeMounts,
+		VolumeMounts:    append(derivedWorkload.volumeMounts, params.volumeMounts...),
 		Resources:       derivedWorkload.resources,
 		SecurityContext: derivedWorkload.securityContext,
 	}
@@ -565,7 +584,8 @@ func getContainers(payload []byte, ctop *v1alpha1.CAPTenantOperation, derivedWor
 				{Name: "MTX_TENANT_ID", Value: ctop.Spec.TenantId},
 				{Name: "MTX_REQUEST_PAYLOAD", Value: string(payload)},
 			},
-			EnvFrom: params.envFromVCAPSecret,
+			EnvFrom:      params.envFromVCAPSecret,
+			VolumeMounts: append(derivedWorkload.volumeMounts, params.volumeMounts...),
 		},
 		*container,
 	}
@@ -641,7 +661,7 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v
 				Spec: corev1.PodSpec{
 					RestartPolicy:             corev1.RestartPolicyNever,
 					SecurityContext:           workload.JobDefinition.PodSecurityContext,
-					Volumes:                   workload.JobDefinition.Volumes,
+					Volumes:                   append(workload.JobDefinition.Volumes, params.volumes...),
 					ServiceAccountName:        workload.JobDefinition.ServiceAccountName,
 					NodeSelector:              workload.JobDefinition.NodeSelector,
 					NodeName:                  workload.JobDefinition.NodeName,
@@ -659,7 +679,7 @@ func (c *Controller) createCustomTenantOperationJob(ctx context.Context, ctop *v
 								{Name: EnvCAPOpAppVersion, Value: params.version}, {Name: EnvCAPOpTenantID, Value: ctop.Spec.TenantId}, {Name: EnvCAPOpTenantOperation, Value: string(ctop.Spec.Operation)}, {Name: EnvCAPOpTenantSubDomain, Value: string(ctop.Spec.SubDomain)},
 							}, workload.JobDefinition.Env...),
 							EnvFrom:         params.envFromVCAPSecret,
-							VolumeMounts:    workload.JobDefinition.VolumeMounts,
+							VolumeMounts:    append(workload.JobDefinition.VolumeMounts, params.volumeMounts...),
 							Command:         workload.JobDefinition.Command,
 							Resources:       workload.JobDefinition.Resources,
 							SecurityContext: workload.JobDefinition.SecurityContext,

internal/controller/reconcile.go

@@ -9,9 +9,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"path"
 
 	"github.com/sap/cap-operator/internal/util"
 	"github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1"
+	"golang.org/x/exp/slices"
 	"golang.org/x/mod/semver"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -52,6 +54,10 @@ const (
 	GardenerDNSClassIdentifier         = "dns.gardener.cloud/class"
 )
 
+const (
+	CDSVolMountPrefix = "/etc/secrets/cds"
+)
+
 const (
 	CertificateSuffix     = "certificate"
 	GardenerDNSClassValue = "garden"
@@ -79,6 +85,7 @@ const (
 	EnvCAPOpTenantSubDomain = "CAPOP_TENANT_SUBDOMAIN"
 	EnvCAPOpTenantOperation = "CAPOP_TENANT_OPERATION"
 	EnvVCAPServices         = "VCAP_SERVICES"
+	EnvUseVolumeMounts      = "USE_VOLUME_MOUNTS"
 )
 
 type JobState string
@@ -560,3 +567,34 @@ func copyMaps(originalMap map[string]string, additionalMap map[string]string) ma
 	}
 	return newMap
 }
+
+func getVolumeMounts(serviceInfos []v1alpha1.ServiceInfo) []corev1.VolumeMount {
+	volumeMounts := []corev1.VolumeMount{}
+
+	for _, serviceInfo := range serviceInfos {
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join("/etc/secrets/sapcp", serviceInfo.Class, serviceInfo.Name), ReadOnly: true})
+	}
+	return volumeMounts
+}
+
+func getCAPVolumeMounts(serviceInfos []v1alpha1.ServiceInfo, mountPrefix string) []corev1.VolumeMount {
+	volumeMounts := []corev1.VolumeMount{}
+
+	for _, serviceInfo := range serviceInfos {
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: serviceInfo.Name, MountPath: path.Join(mountPrefix, "requires", serviceInfo.Class, "credentials"), ReadOnly: true})
+	}
+	return volumeMounts
+}
+
+func getVolumes(serviceInfos []v1alpha1.ServiceInfo) []corev1.Volume {
+	volumes := []corev1.Volume{}
+
+	for _, serviceInfo := range serviceInfos {
+		volumes = append(volumes, corev1.Volume{Name: serviceInfo.Name, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: serviceInfo.Secret}}})
+	}
+	return volumes
+}
+
+func useVolumeMounts(envVars []corev1.EnvVar) bool {
+	return slices.ContainsFunc(envVars, func(env corev1.EnvVar) bool { return env.Name == EnvUseVolumeMounts && env.Value == "true" })
+}