From 4de8d13011924ad7bd815d82fa8f382695055035 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Thu, 9 Nov 2023 11:25:26 +0000 Subject: [PATCH] logserver - define one Service instead of three This change refactors Service Resource definition for the logserver. Also, it updates the function that trigger the Zuul internal tenant reconfigure in order to ensure that when the internal tenant's secret's serial value is updated then we run the reconfigure command. As we use the internal DNS name of the Service and as the Service has been renamed just 'logserver', then the logserver's Secret content changed. Then during an upgrade the reconfigure must be triggered. Change-Id: I087ffcad1cda731d47205798e68f1b18d27d9cb9 --- controllers/config.go | 53 ++++++++++++------- controllers/libs/monitoring/monitoring.go | 12 +---- controllers/logserver_controller.go | 50 ++++++++++++----- .../static/git-server/update-system-config.sh | 4 +- controllers/static/sf_operator/main.py | 4 +- controllers/utils.go | 2 +- doc/developer/howtos/index.md | 2 +- .../test-volumestats-sidecar/tasks/main.yaml | 4 +- 8 files changed, 81 insertions(+), 50 deletions(-) diff --git a/controllers/config.go b/controllers/config.go index 796e7a17..8f507786 100644 --- a/controllers/config.go +++ b/controllers/config.go @@ -23,7 +23,7 @@ var pymodSecret string var pymodMain string // SetupBaseSecrets returns true when the Job that set the zuul secret in the system-config repository is done -func (r *SFController) SetupBaseSecrets() bool { +func (r *SFController) SetupBaseSecrets(internalTenantSecretsVersion string) bool { serviceAccountName := "config-updater" serviceAccount := apiv1.ServiceAccount{} @@ -105,7 +105,9 @@ func (r *SFController) SetupBaseSecrets() bool { } var job batchv1.Job - jobName := "config-base-secret" + // We need to run a new job whenever the version of the secrets changed + // thus we include the version of the secrets + jobName := "config-base-secret-" + internalTenantSecretsVersion found := r.GetM(jobName, &job) extraCmdVars := []apiv1.EnvVar{ @@ -151,27 +153,41 @@ func (r *SFController) InstallTooling() { } func (r *SFController) SetupConfigJob() bool { + var ( + // We use the CM to store versions that can trigger internal tenant secrets update + // or zuul internal tenant reconfigure + cmName = "zs-internal-tenant-reconfigure" + zsInternalTenantReconfigure apiv1.ConfigMap + configHash = utils.Checksum([]byte(preInitScriptTemplate)) + internalTenantSecretsVersion = "1" + needReconfigureTenant = false + needCMUpdate = false + ) + + // Ensure that toolings are available in the ConfigMap r.InstallTooling() + + // Get the internal tenant version CM and evaluate if we need to trigger actions + if !r.GetM(cmName, &zsInternalTenantReconfigure) { + needReconfigureTenant = true + } else { + if configHash != zsInternalTenantReconfigure.Data["internal-tenant-config-hash"] || + internalTenantSecretsVersion != zsInternalTenantReconfigure.Data["internal-tenant-secrets-version"] { + needReconfigureTenant = true + needCMUpdate = true + } + } + // We ensure that base secrets are set in the system-config repository - if r.SetupBaseSecrets() { + baseSecretsInstalled := r.SetupBaseSecrets(internalTenantSecretsVersion) + + if baseSecretsInstalled { // We run zuul tenant-reconfigure for the 'internal' tenant, when: // - the configMap does not exists (or) // - tenant config changed + // - tenant secrets version changed // This ensures that the zuul-scheduler loaded the provisionned Zuul config // for the 'internal' tenant - var zsInternalTenantReconfigure apiv1.ConfigMap - var cmName = "zs-internal-tenant-reconfigure" - var needReconfigureTenant = false - var needConfigMapUpdate = false - var configHash = utils.Checksum([]byte(preInitScriptTemplate)) - if !r.GetM(cmName, &zsInternalTenantReconfigure) { - needReconfigureTenant = true - } else { - if configHash != zsInternalTenantReconfigure.Data["internal-tenant-config-hash"] { - needReconfigureTenant = true - needConfigMapUpdate = true - } - } if needReconfigureTenant { r.log.Info("Running tenant-reconfigure for the 'internal' tenant") if r.runZuulInternalTenantReconfigure() { @@ -181,9 +197,10 @@ func (r *SFController) SetupConfigJob() bool { Namespace: r.ns, } zsInternalTenantReconfigure.Data = map[string]string{ - "internal-tenant-config-hash": configHash, + "internal-tenant-config-hash": configHash, + "internal-tenant-secrets-version": internalTenantSecretsVersion, } - if needConfigMapUpdate { + if needCMUpdate { r.UpdateR(&zsInternalTenantReconfigure) } else { r.CreateR(&zsInternalTenantReconfigure) diff --git a/controllers/libs/monitoring/monitoring.go b/controllers/libs/monitoring/monitoring.go index 47f054eb..fe9103d9 100644 --- a/controllers/libs/monitoring/monitoring.go +++ b/controllers/libs/monitoring/monitoring.go @@ -36,7 +36,7 @@ func GetTruncatedPortName(serviceName string, suffix string) string { const NodeExporterNameSuffix = "-nodeexporter" const NodeExporterPortNameSuffix = "-ne" -const nodeExporterPort = 9100 +const NodeExporterPort = 9100 // Fun fact: arrays cannot be consts, so we define our args in this function. func getNodeExporterArgs(volumeMounts []apiv1.VolumeMount) []string { @@ -52,20 +52,12 @@ func MkNodeExporterSideCarContainer(serviceName string, volumeMounts []apiv1.Vol container := base.MkContainer(serviceName+NodeExporterNameSuffix, base.NodeExporterImage) container.Args = getNodeExporterArgs(volumeMounts) container.Ports = []apiv1.ContainerPort{ - base.MkContainerPort(nodeExporterPort, GetTruncatedPortName(serviceName, NodeExporterPortNameSuffix)), + base.MkContainerPort(NodeExporterPort, GetTruncatedPortName(serviceName, NodeExporterPortNameSuffix)), } container.VolumeMounts = volumeMounts return container } -func MkNodeExporterSideCarService(serviceName string, namespace string) apiv1.Service { - var portName = GetTruncatedPortName(serviceName, NodeExporterPortNameSuffix) - servicePorts := []int32{nodeExporterPort} - neService := base.MkService(serviceName+NodeExporterPortNameSuffix, namespace, serviceName, servicePorts, portName) - return neService - -} - // Statsd exporter utilities const statsdExporterNameSuffix = "-statsd" diff --git a/controllers/logserver_controller.go b/controllers/logserver_controller.go index 64df9508..8a6831a8 100644 --- a/controllers/logserver_controller.go +++ b/controllers/logserver_controller.go @@ -174,7 +174,36 @@ func (r *LogServerController) ensureLogserverPromRule() bool { return true } +// cleanup ensures removal of legacy resources +func (r *LogServerController) cleanup() { + // Delete apiv1.Service httpdPortName-httpdPort + r.DeleteR(&apiv1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.ns, + Name: httpdPortName, + }, + }) + // Delete apiv1.Service sshdPortName-sshdPort + r.DeleteR(&apiv1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.ns, + Name: sshdPortName, + }, + }) + // Delete apiv1.service logserverIdent-NodeExporterPortNameSuffix + r.DeleteR(&apiv1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.ns, + Name: logserverIdent + sfmonitoring.NodeExporterPortNameSuffix, + }, + }) + +} + func (r *LogServerController) DeployLogserver() sfv1.LogServerStatus { + + r.cleanup() + log := log.FromContext(r.ctx) r.EnsureSSHKeySecret(logserverIdent + "-keys") @@ -195,6 +224,12 @@ func (r *LogServerController) DeployLogserver() sfv1.LogServerStatus { r.EnsureConfigMap(logserverIdent, cmData) + // Create service exposed by logserver + servicePorts := []int32{httpdPort, sshdPort, sfmonitoring.NodeExporterPort} + svc := base.MkService( + logserverIdent, r.ns, logserverIdent, servicePorts, logserverIdent) + r.GetOrCreate(&svc) + volumeMounts := []apiv1.VolumeMount{ { Name: logserverIdent + "-config-vol", @@ -273,12 +308,6 @@ func (r *LogServerController) DeployLogserver() sfv1.LogServerStatus { "/usr/bin/" + lgEntryScriptName, } - // Create services exposed by logserver - servicePorts := []int32{httpdPort} - httpdService := base.MkService( - httpdPortName, r.ns, logserverIdent, servicePorts, httpdPortName) - r.GetOrCreate(&httpdService) - // Setup the sidecar container for sshd sshdContainer := base.MkContainer(sshdPortName, base.SSHDImage) sshdContainer.Command = []string{"bash", "/conf/run.sh"} @@ -365,13 +394,6 @@ func (r *LogServerController) DeployLogserver() sfv1.LogServerStatus { r.CreateR(&dep) } - sshdServicePorts := []int32{sshdPort} - sshdService := base.MkService(sshdPortName, r.ns, logserverIdent, sshdServicePorts, sshdPortName) - r.GetOrCreate(&sshdService) - - nodeExporterSidecarService := sfmonitoring.MkNodeExporterSideCarService(logserverIdent, r.ns) - r.GetOrCreate(&nodeExporterSidecarService) - pvcReadiness := r.reconcileExpandPVC(logserverIdent, r.cr.Spec.Settings.Storage) // refresh current deployment @@ -379,7 +401,7 @@ func (r *LogServerController) DeployLogserver() sfv1.LogServerStatus { routeReady := r.ensureHTTPSRoute( r.cr.Name+"-logserver", logserverIdent, - httpdPortName, "/", httpdPort, map[string]string{}, r.cr.Spec.FQDN, r.cr.Spec.LetsEncrypt) + logserverIdent, "/", httpdPort, map[string]string{}, r.cr.Spec.FQDN, r.cr.Spec.LetsEncrypt) // TODO(mhu) We may want to open an ingress to port 9100 for an external prometheus instance. // TODO(mhu) we may want to include monitoring objects' status in readiness computation diff --git a/controllers/static/git-server/update-system-config.sh b/controllers/static/git-server/update-system-config.sh index 77dd077e..127f090d 100755 --- a/controllers/static/git-server/update-system-config.sh +++ b/controllers/static/git-server/update-system-config.sh @@ -203,9 +203,9 @@ cat << EOF > playbooks/base/post.yaml fileserver: "{{ site_sflogs }}" - role: generate-zuul-manifest -- hosts: logserver-sshd +- hosts: logserver vars: - ansible_port: ${LOGSERVER_SSHD_SERVICE_PORT} + ansible_port: 2222 gather_facts: false tasks: - block: diff --git a/controllers/static/sf_operator/main.py b/controllers/static/sf_operator/main.py index f77f7dac..28d1542d 100644 --- a/controllers/static/sf_operator/main.py +++ b/controllers/static/sf_operator/main.py @@ -31,7 +31,7 @@ def sshkey_scan(port: int, hostname: str) -> bytes: ["ssh-keyscan", "-T", "10", "-p", str(port), hostname ]) def get_logserver_fingerprint() -> str: - return " ".join(sshkey_scan(2222, "logserver-sshd").decode().split()[0:3]) + return " ".join(sshkey_scan(2222, "logserver").decode().split()[0:3]) def mk_incluster_k8s_config(): sa = Path("/run/secrets/kubernetes.io/serviceaccount") @@ -66,7 +66,7 @@ def create_zuul_secrets(): ("ssh_private_key", os.environ["ZUUL_LOGSERVER_PRIVATE_KEY"]) ], unencrypted_items=[ - ("fqdn", "\"[logserver-sshd]:2222\""), + ("fqdn", "\"[logserver]:2222\""), ("path", "rsync"), ("ssh_known_hosts", "\"%s\"" % get_logserver_fingerprint()), ("ssh_username", "zuul") diff --git a/controllers/utils.go b/controllers/utils.go index 4eccf5f5..b557dcd8 100644 --- a/controllers/utils.go +++ b/controllers/utils.go @@ -100,7 +100,7 @@ func (r *SFUtilContext) CreateR(obj client.Object) { // DeleteR delete a resource. func (r *SFUtilContext) DeleteR(obj client.Object) { - if err := r.Client.Delete(r.ctx, obj); err != nil { + if err := r.Client.Delete(r.ctx, obj); err != nil && !errors.IsNotFound(err) { panic(err.Error()) } } diff --git a/doc/developer/howtos/index.md b/doc/developer/howtos/index.md index a8447a62..dd2e54db 100644 --- a/doc/developer/howtos/index.md +++ b/doc/developer/howtos/index.md @@ -107,7 +107,7 @@ microshift cluster inbound, then `firefox https://logserver.test.local`. To send data to the logserver, first enable the port-forward: ```shell -kubectl -n logserver port-forward service/logserver-sshd 22220:2222 +kubectl -n logserver port-forward service/logserver 22220:2222 ``` Then use rsync: diff --git a/roles/health-check/test-volumestats-sidecar/tasks/main.yaml b/roles/health-check/test-volumestats-sidecar/tasks/main.yaml index df6c1470..74e7e771 100644 --- a/roles/health-check/test-volumestats-sidecar/tasks/main.yaml +++ b/roles/health-check/test-volumestats-sidecar/tasks/main.yaml @@ -9,7 +9,7 @@ - name: Start port forwarding to logserver sidecar shell: | - kubectl port-forward service/logserver-ne 9100 -n sf + kubectl port-forward service/logserver 9100 -n sf async: 60 poll: 0 @@ -24,4 +24,4 @@ register: logserver_metrics delay: 5 retries: 10 - until: "logserver_metrics.status == 200 and 'data/rsync' in logserver_metrics.content" \ No newline at end of file + until: "logserver_metrics.status == 200 and 'data/rsync' in logserver_metrics.content"