From 6e1a6a9d8167020ee9152ffb188cb31c2d3568ab Mon Sep 17 00:00:00 2001 From: parth-gr Date: Fri, 16 Aug 2024 12:00:20 +0530 Subject: [PATCH 1/2] logrotate: add logrotate functionality for csi 1) Make main container and csi addons container log to a file(dependency on klog) 2) Add a log-rotate sidecar container, so it can rotate the logs 3) Added other volume and volumemounts as needed 4) Added the privileged option for controllerplugin 5) Add resources to the logrotate container 6) make the api change in the merge function Signed-off-by: parth-gr --- api/v1alpha1/driver_types.go | 3 + config/crd/bases/csi.ceph.io_drivers.yaml | 12 +- .../bases/csi.ceph.io_operatorconfigs.yaml | 12 +- internal/controller/defaults.go | 6 +- internal/controller/driver_controller.go | 171 +++++++++++++++++- internal/utils/csi.go | 42 +++++ 6 files changed, 231 insertions(+), 15 deletions(-) diff --git a/api/v1alpha1/driver_types.go b/api/v1alpha1/driver_types.go index 1084c1d3..62231764 100644 --- a/api/v1alpha1/driver_types.go +++ b/api/v1alpha1/driver_types.go @@ -33,8 +33,10 @@ const ( MonthlyPeriod PeriodicityType = "monthly" ) +// +kubebuilder:validation:XValidation:message="Either maxLogSize or periodicity must be set",rule="(has(self.maxLogSize)) || (has(self.periodicity))" type LogRotationSpec struct { // MaxFiles is the number of logrtoate files + // Default to 7 //+kubebuilder:validation:Optional MaxFiles int `json:"maxFiles,omitempty"` @@ -48,6 +50,7 @@ type LogRotationSpec struct { Periodicity PeriodicityType `json:"periodicity,omitempty"` // LogHostPath is the prefix directory path for the csi log files + // Default to /var/lib/cephcsi //+kubebuilder:validation:Optional LogHostPath string `json:"logHostPath,omitempty"` } diff --git a/config/crd/bases/csi.ceph.io_drivers.yaml b/config/crd/bases/csi.ceph.io_drivers.yaml index 24e904cd..e592d853 100644 --- a/config/crd/bases/csi.ceph.io_drivers.yaml +++ b/config/crd/bases/csi.ceph.io_drivers.yaml @@ -3542,11 +3542,14 @@ spec: description: log rotation for csi pods properties: logHostPath: - description: LogHostPath is the prefix directory path for - the csi log files + description: |- + LogHostPath is the prefix directory path for the csi log files + Default to /var/lib/cephcsi type: string maxFiles: - description: MaxFiles is the number of logrtoate files + description: |- + MaxFiles is the number of logrtoate files + Default to 7 type: integer maxLogSize: anyOf: @@ -3565,6 +3568,9 @@ spec: - monthly type: string type: object + x-kubernetes-validations: + - message: Either maxLogSize or periodicity must be set + rule: (has(self.maxLogSize)) || (has(self.periodicity)) verbosity: description: |- Log verbosity level for driver pods, diff --git a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml index deb9e290..109d0a53 100644 --- a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml +++ b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml @@ -3581,11 +3581,14 @@ spec: description: log rotation for csi pods properties: logHostPath: - description: LogHostPath is the prefix directory path - for the csi log files + description: |- + LogHostPath is the prefix directory path for the csi log files + Default to /var/lib/cephcsi type: string maxFiles: - description: MaxFiles is the number of logrtoate files + description: |- + MaxFiles is the number of logrtoate files + Default to 7 type: integer maxLogSize: anyOf: @@ -3605,6 +3608,9 @@ spec: - monthly type: string type: object + x-kubernetes-validations: + - message: Either maxLogSize or periodicity must be set + rule: (has(self.maxLogSize)) || (has(self.periodicity)) verbosity: description: |- Log verbosity level for driver pods, diff --git a/internal/controller/defaults.go b/internal/controller/defaults.go index 10ea404c..19ed50a4 100644 --- a/internal/controller/defaults.go +++ b/internal/controller/defaults.go @@ -38,8 +38,10 @@ var imageDefaults = map[string]string{ } const ( - defaultGRrpcTimeout = 150 - defaultKubeletDirPath = "/var/lib/kubelet" + defaultGRrpcTimeout = 150 + defaultKubeletDirPath = "/var/lib/kubelet" + defaultLogHostPath = "/var/lib/cephcsi" + defaultLogRotateMaxFiles = 7 ) var defaultLeaderElection = csiv1a1.LeaderElectionSpec{ diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index b766bf6f..c4544c88 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -230,6 +230,7 @@ func (r *driverReconcile) reconcile() error { // the desired state defined on the driver object errChan := utils.RunConcurrently( r.reconcileCsiConfigMap, + r.reconcileLogRotateConfigMap, r.reconcileK8sCsiDriver, r.reconcileControllerPluginDeployment, r.reconcileNodePluginDeamonSet, @@ -327,6 +328,63 @@ func (r *driverReconcile) LoadAndValidateDesiredState() error { return nil } +func (r *driverReconcile) reconcileLogRotateConfigMap() error { + logRotateConfigmap := &corev1.ConfigMap{} + logRotateConfigmap.Name = utils.LogRotateConfigMapName(r.driver.Name) + logRotateConfigmap.Namespace = r.driver.Namespace + + log := r.log.WithValues("logRotateConfigMap", logRotateConfigmap.Name) + log.Info("Reconciling logRotate configmap") + + logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation + if logRotationSpec != nil { + opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, logRotateConfigmap, func() error { + if _, err := utils.ToggleOwnerReference(true, logRotateConfigmap, &r.driver, r.Scheme); err != nil { + log.Error(err, "Failed adding an owner reference on the LogRotate config map") + return err + } + if logRotationSpec.Periodicity != "" && logRotationSpec.MaxLogSize.IsZero() { + err := fmt.Errorf("invalid Log.Rotation spec") + log.Error(err, "Either \"maxLogSize\" or \"periodicity\" fields must be set") + return err + } + maxFiles := cmp.Or(logRotationSpec.MaxFiles, defaultLogRotateMaxFiles) + cronLogRotateSettings := []string{ + "\tmissingok", + "\tcompress", + "\tcopytruncate", + "\tnotifempty", + fmt.Sprintf("\trotate %d", maxFiles), + } + if logRotationSpec.Periodicity != "" { + periodicity := "\t" + string(logRotationSpec.Periodicity) + cronLogRotateSettings = append(cronLogRotateSettings, periodicity) + } + if logRotationSpec.MaxLogSize.String() != "0" { + maxSize := fmt.Sprintf("\tmaxsize %s", logRotationSpec.MaxLogSize.String()) + cronLogRotateSettings = append(cronLogRotateSettings, maxSize) + } + logRotateConfigmap.Data = map[string]string{ + "csi": fmt.Sprintf( + "/csi-logs/*.log {\n%s\n}\n", + strings.Join(cronLogRotateSettings, "\n"), + ), + } + return nil + }) + + logCreateOrUpdateResult(log, "LogRotateConfigMap", logRotateConfigmap, opResult, err) + return err + } else { + // Remove the logrotate configmap if logrotate setting is removed from the driver's spec + if err := r.Delete(r.ctx, logRotateConfigmap); client.IgnoreNotFound(err) != nil { + log.Error(err, "Unable to delete LogRotate configmap") + return err + } + return nil + } +} + func (r *driverReconcile) reconcileCsiConfigMap() error { csiConfigMap := &corev1.ConfigMap{} csiConfigMap.Name = utils.CsiConfigVolume.Name @@ -454,6 +512,10 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { logVerbosity := ptr.Deref(r.driver.Spec.Log, csiv1a1.LogSpec{}).Verbosity forceKernelClient := r.isCephFsDriver() && r.driver.Spec.CephFsClientType == csiv1a1.KernelCephFsClient snPolicy := cmp.Or(r.driver.Spec.SnapshotPolicy, csiv1a1.VolumeSnapshotSnapshotPolicy) + logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation + logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) + logRotationEnabled := logRotationSpec != nil + securityContext := utils.If(logRotationEnabled, &corev1.SecurityContext{Privileged: ptr.To(true)}, nil) leaderElectionSettingsArg := []string{ utils.LeaderElectionNamespaceContainerArg(r.driver.Namespace), @@ -487,6 +549,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: fmt.Sprintf("csi-%splugin", r.driverType), Image: r.images["plugin"], ImagePullPolicy: imagePullPolicy, + SecurityContext: securityContext, Args: utils.DeleteZeroValues( []string{ utils.TypeContainerArg(string(r.driverType)), @@ -504,6 +567,13 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.CsiAddonsEndpointContainerArg, "", ), + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If( + logRotationEnabled, + utils.LogFileContainerArg(fmt.Sprintf("csi-%splugin", r.driverType)), + "", + ), }, ), Env: []corev1.EnvVar{ @@ -534,6 +604,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { if r.isRdbDriver() { mounts = append(mounts, utils.OidcTokenVolumeMount) } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } return mounts }), Resources: ptr.Deref( @@ -654,6 +727,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: "csi-addons", Image: r.images["addons"], ImagePullPolicy: imagePullPolicy, + SecurityContext: securityContext, Args: utils.DeleteZeroValues( append( slices.Clone(leaderElectionSettingsArg), @@ -664,6 +738,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.CsiAddonsAddressContainerArg, utils.ControllerPortContainerArg, utils.NamespaceContainerArg, + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""), ), ), Ports: []corev1.ContainerPort{ @@ -675,9 +752,15 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.PodNameEnvVar, utils.PodNamespaceEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.SocketDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.SocketDirVolumeMount, + } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -741,7 +824,22 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { ), }) } - + // CSI LogRotate Container + if logRotationEnabled { + resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}) + containers = append(containers, corev1.Container{ + Name: "log-rotator", + Image: r.images["plugin"], + ImagePullPolicy: imagePullPolicy, + Resources: resources, + Command: []string{"/bin/bash", "-c"}, + Args: []string{`while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done`}, + VolumeMounts: []corev1.VolumeMount{ + utils.LogsDirVolumeMount, + utils.LogRotateDirVolumeMount, + }, + }) + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -767,6 +865,13 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { volumes, utils.KmsConfigVolume(&r.driver.Spec.Encryption.ConfigMapRef)) } + if logRotationEnabled { + volumes = append( + volumes, + utils.LogsDirVolume(logHostPath, deploy.Name), + utils.LogRotateDirVolumeName(r.driver.Name), + ) + } return volumes }), }, @@ -807,6 +912,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { topology := r.isRdbDriver() && pluginSpec.Topology != nil domainLabels := cmp.Or(pluginSpec.Topology, &csiv1a1.TopologySpec{}).DomainLabels + logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation + logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) + logRotationEnabled := logRotationSpec != nil daemonSet.Spec = appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ @@ -884,6 +992,13 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.DomainLabelsContainerArg(domainLabels), "", ), + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If( + logRotationEnabled, + utils.LogFileContainerArg(fmt.Sprintf("csi-%splugin", r.driverType)), + "", + ), }, ), Env: []corev1.EnvVar{ @@ -912,6 +1027,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { if r.isRdbDriver() { mounts = append(mounts, utils.OidcTokenVolumeMount) } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } return mounts }), Resources: ptr.Deref( @@ -965,6 +1083,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { Args: utils.DeleteZeroValues( []string{ utils.CsiAddonsNodeIdContainerArg, + utils.NodeIdContainerArg, utils.LogVerbosityContainerArg(logVerbosity), utils.CsiAddonsAddressContainerArg, utils.ControllerPortContainerArg, @@ -972,6 +1091,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.NamespaceContainerArg, utils.PodUidContainerArg, utils.StagingPathContainerArg(kubeletDirPath), + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""), }, ), Ports: []corev1.ContainerPort{ @@ -983,9 +1105,15 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.PodNamespaceEnvVar, utils.PodUidEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.PluginDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.PluginDirVolumeMount, + } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -1026,6 +1154,22 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { }) } } + // CSI LogRotate Container + if logRotationEnabled { + resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}) + containers = append(containers, corev1.Container{ + Name: "log-rotator", + Image: r.images["plugin"], + ImagePullPolicy: imagePullPolicy, + Resources: resources, + Command: []string{"/bin/bash", "-c"}, + Args: []string{`while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done`}, + VolumeMounts: []corev1.VolumeMount{ + utils.LogsDirVolumeMount, + utils.LogRotateDirVolumeMount, + }, + }) + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -1067,6 +1211,13 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.OidcTokenVolume, ) } + if logRotationEnabled { + volumes = append( + volumes, + utils.LogsDirVolume(logHostPath, daemonSet.Name), + utils.LogRotateDirVolumeName(r.driver.Name), + ) + } return volumes }), }, @@ -1273,6 +1424,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if src.ControllerPlugin != nil { @@ -1328,6 +1482,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if dest.AttachRequired == nil { diff --git a/internal/utils/csi.go b/internal/utils/csi.go index 0df4ae77..4ab0521f 100644 --- a/internal/utils/csi.go +++ b/internal/utils/csi.go @@ -37,6 +37,9 @@ const ( CsiConfigMapConfigKey = "config.json" CsiConfigMapMappingKey = "cluster-mapping.json" + + logsDirVolumeName = "logs-dir" + logRotateDirVolumeName = "log-rotate-dir" ) // Ceph CSI common volumes @@ -182,6 +185,29 @@ func PodsMountDirVolume(kubeletDirPath string) corev1.Volume { }, } } +func LogsDirVolume(logHostPath, pluginName string) corev1.Volume { + return corev1.Volume{ + Name: logsDirVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: fmt.Sprintf("%s/%s", logHostPath, pluginName), + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + } +} +func LogRotateDirVolumeName(driverName string) corev1.Volume { + return corev1.Volume{ + Name: logRotateDirVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: LogRotateConfigMapName(driverName), + }, + }, + }, + } +} // Ceph CSI common volume Mounts var SocketDirVolumeMount = corev1.VolumeMount{ @@ -236,6 +262,14 @@ var EtcSelinuxVolumeMount = corev1.VolumeMount{ MountPath: "/etc/selinux", ReadOnly: true, } +var LogsDirVolumeMount = corev1.VolumeMount{ + Name: logsDirVolumeName, + MountPath: "/csi-logs", +} +var LogRotateDirVolumeMount = corev1.VolumeMount{ + Name: logRotateDirVolumeName, + MountPath: "/logrotate-config", +} func PodsMountDirVolumeMount(kubletDirPath string) corev1.VolumeMount { return corev1.VolumeMount{ @@ -335,10 +369,15 @@ var ImmediateTopologyContainerArg = "--immediate-topology=false" var RecoverVolumeExpansionFailureContainerArg = "--feature-gates=RecoverVolumeExpansionFailure=true" var EnableVolumeGroupSnapshotsContainerArg = "--enable-volume-group-snapshots=true" var ForceCephKernelClientContainerArg = "--forcecephkernelclient=true" +var LogToStdErrContainerArg = "--logtostderr=false" +var AlsoLogToStdErrContainerArg = "--alsologtostderr=true" func LogVerbosityContainerArg(level int) string { return fmt.Sprintf("--v=%d", Clamp(level, 0, 5)) } +func LogFileContainerArg(containerName string) string { + return fmt.Sprintf("--log_file=csi-logs/%s.log", containerName) +} func TypeContainerArg(t string) string { switch t { case "rbd", "cephfs", "nfs", "controller", "liveness": @@ -380,6 +419,9 @@ func KubeletRegistrationPathContainerArg(kubeletDirPath string, driverName strin func StagingPathContainerArg(kubeletDirPath string) string { return fmt.Sprintf("--stagingpath=%s/plugins/kubernetes.io/csi/", kubeletDirPath) } +func LogRotateConfigMapName(driverName string) string { + return fmt.Sprintf("%s-logrotate-config", driverName) +} func KernelMountOptionsContainerArg(options map[string]string) string { return If( len(options) > 0, From 9667889fbc53de70ac803a7e5ab7095faae5cd1a Mon Sep 17 00:00:00 2001 From: parth-gr Date: Fri, 16 Aug 2024 14:10:30 +0530 Subject: [PATCH 2/2] logrotate: use the contoller plugin privileged to set sc currently security context was just set by looking logrotate is enabled or not, but from this commit we will also check if the cntrlplugin has prviliged true Signed-off-by: parth-gr --- docs/design/logrotate.md | 34 ++++++++++--------- docs/design/operator.md | 2 +- internal/controller/driver_controller.go | 42 +++++++++++++++++------- 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/docs/design/logrotate.md b/docs/design/logrotate.md index c3773fcd..b8e9e95c 100644 --- a/docs/design/logrotate.md +++ b/docs/design/logrotate.md @@ -13,14 +13,14 @@ apiVersion: csi.ceph.io/v1alpha1 spec: log: verbosity: 1 - driverSpecDefaults: + driverSpecDefaults: log: verbosity: 5 rotation: # one of: hourly, daily, weekly, monthly periodicity: daily maxLogSize: 500M - maxFiles: 5 + maxFiles: 7 logHostPath: /var/lib/cephcsi ``` @@ -35,14 +35,14 @@ metadata: spec: log: verbosity: 1 - driverSpecDefaults: + driverSpecDefaults: log: verbosity: 5 rotation: # one of: hourly, daily, weekly, monthly periodicity: daily maxLogSize: 500M - maxFiles: 5 + maxFiles: 7 logHostPath: /var/lib/cephcsi ``` @@ -51,20 +51,24 @@ Logrotator sidecar container cpu and memory usage can configured by, `OperatorConfig CRD`: ```yaml spec: - provisioner: - logRotator: - cpu: "100m" - memory: "32Mi" - plugin: - logRotator: - cpu: "100m" - memory: "32Mi" + driverSpecDefaults: + controllerPlugin: + resources: + logRotator: + cpu: "100m" + memory: "32Mi" + nodePlugin: + resources: + logRotator: + cpu: "100m" + memory: "32Mi" ``` -For systems where SELinux is enabled (e.g. OpenShift),start plugin-controller as privileged that mount a host path. +For systems where SELinux is enabled (e.g. OpenShift), start plugin-controller as privileged that mount a host path. `OperatorConfig CRD`: ```yaml spec: - provisioner: - privileged: true + driverSpecDefaults: + controllerPlugin: + privileged: true ``` diff --git a/docs/design/operator.md b/docs/design/operator.md index 01818b18..1d4f00d7 100644 --- a/docs/design/operator.md +++ b/docs/design/operator.md @@ -92,7 +92,7 @@ spec: # one of: hourly, daily, weekly, monthly periodicity: daily maxLogSize: 500M - maxFiles: 5 + maxFiles: 7 logHostPath: /var/lib/cephcsi clusterName: 5c63ad7e-74fe-4724-a511-4ccdc560da56 enableMetadata: true diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index c4544c88..0fca903a 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -69,8 +69,11 @@ const ( NfsDriverType = "nfs" ) -// Annotation name for ownerref information -const ownerRefAnnotationKey = "csi.ceph.io/ownerref" +const ( + // Annotation name for ownerref information + ownerRefAnnotationKey = "csi.ceph.io/ownerref" + logRotateCmd = `while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done` +) // A regexp used to parse driver's prefix and type from the full name var nameRegExp, _ = regexp.Compile(fmt.Sprintf( @@ -513,9 +516,17 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { forceKernelClient := r.isCephFsDriver() && r.driver.Spec.CephFsClientType == csiv1a1.KernelCephFsClient snPolicy := cmp.Or(r.driver.Spec.SnapshotPolicy, csiv1a1.VolumeSnapshotSnapshotPolicy) logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation - logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) logRotationEnabled := logRotationSpec != nil - securityContext := utils.If(logRotationEnabled, &corev1.SecurityContext{Privileged: ptr.To(true)}, nil) + logRotateSecurityContext := utils.If( + pluginSpec.Privileged != nil && logRotationEnabled, + &corev1.SecurityContext{ + Privileged: pluginSpec.Privileged, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"All"}, + }, + }, + nil, + ) leaderElectionSettingsArg := []string{ utils.LeaderElectionNamespaceContainerArg(r.driver.Namespace), @@ -549,7 +560,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: fmt.Sprintf("csi-%splugin", r.driverType), Image: r.images["plugin"], ImagePullPolicy: imagePullPolicy, - SecurityContext: securityContext, + SecurityContext: logRotateSecurityContext, Args: utils.DeleteZeroValues( []string{ utils.TypeContainerArg(string(r.driverType)), @@ -727,7 +738,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: "csi-addons", Image: r.images["addons"], ImagePullPolicy: imagePullPolicy, - SecurityContext: securityContext, + SecurityContext: logRotateSecurityContext, Args: utils.DeleteZeroValues( append( slices.Clone(leaderElectionSettingsArg), @@ -832,8 +843,8 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Image: r.images["plugin"], ImagePullPolicy: imagePullPolicy, Resources: resources, - Command: []string{"/bin/bash", "-c"}, - Args: []string{`while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done`}, + SecurityContext: logRotateSecurityContext, + Command: []string{"/bin/bash", "-c", logRotateCmd}, VolumeMounts: []corev1.VolumeMount{ utils.LogsDirVolumeMount, utils.LogRotateDirVolumeMount, @@ -866,6 +877,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.KmsConfigVolume(&r.driver.Spec.Encryption.ConfigMapRef)) } if logRotationEnabled { + logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) volumes = append( volumes, utils.LogsDirVolume(logHostPath, deploy.Name), @@ -913,7 +925,6 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { topology := r.isRdbDriver() && pluginSpec.Topology != nil domainLabels := cmp.Or(pluginSpec.Topology, &csiv1a1.TopologySpec{}).DomainLabels logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation - logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) logRotationEnabled := logRotationSpec != nil daemonSet.Spec = appsv1.DaemonSetSpec{ @@ -1162,8 +1173,13 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { Image: r.images["plugin"], ImagePullPolicy: imagePullPolicy, Resources: resources, - Command: []string{"/bin/bash", "-c"}, - Args: []string{`while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done`}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"All"}, + }, + }, + Command: []string{"/bin/bash", "-c", logRotateCmd}, VolumeMounts: []corev1.VolumeMount{ utils.LogsDirVolumeMount, utils.LogRotateDirVolumeMount, @@ -1212,6 +1228,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { ) } if logRotationEnabled { + logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) volumes = append( volumes, utils.LogsDirVolume(logHostPath, daemonSet.Name), @@ -1461,6 +1478,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Replicas == nil { dest.Replicas = src.Replicas } + if dest.Privileged == nil { + dest.Privileged = src.Privileged + } if dest.Resources.Attacher == nil { dest.Resources.Attacher = src.Resources.Attacher }