logserver - define one Service instead of three

This change refactors Service Resource definition for the logserver.

Also, it updates the function that trigger the Zuul internal tenant reconfigure in
order to ensure that when the internal tenant's secret's serial value is updated then we run
the reconfigure command.

As we use the internal DNS name of the Service and as the Service has been renamed just 'logserver',
then the logserver's Secret content changed. Then during an upgrade the reconfigure must be
triggered.

Change-Id: I087ffcad1cda731d47205798e68f1b18d27d9cb9

controllers/config.go

@@ -23,7 +23,7 @@ var pymodSecret string
 var pymodMain string
 
 // SetupBaseSecrets returns true when the Job that set the zuul secret in the system-config repository is done
-func (r *SFController) SetupBaseSecrets() bool {
+func (r *SFController) SetupBaseSecrets(internalTenantSecretsVersion string) bool {
 
 	serviceAccountName := "config-updater"
 	serviceAccount := apiv1.ServiceAccount{}
@@ -105,7 +105,9 @@ func (r *SFController) SetupBaseSecrets() bool {
 	}
 
 	var job batchv1.Job
-	jobName := "config-base-secret"
+	// We need to run a new job whenever the version of the secrets changed
+	// thus we include the version of the secrets
+	jobName := "config-base-secret-" + internalTenantSecretsVersion
 	found := r.GetM(jobName, &job)
 
 	extraCmdVars := []apiv1.EnvVar{
@@ -151,27 +153,41 @@ func (r *SFController) InstallTooling() {
 }
 
 func (r *SFController) SetupConfigJob() bool {
+	var (
+		// We use the CM to store versions that can trigger internal tenant secrets update
+		// or zuul internal tenant reconfigure
+		cmName                       = "zs-internal-tenant-reconfigure"
+		zsInternalTenantReconfigure  apiv1.ConfigMap
+		configHash                   = utils.Checksum([]byte(preInitScriptTemplate))
+		internalTenantSecretsVersion = "1"
+		needReconfigureTenant        = false
+		needCMUpdate                 = false
+	)
+
+	// Ensure that toolings are available in the ConfigMap
 	r.InstallTooling()
+
+	// Get the internal tenant version CM and evaluate if we need to trigger actions
+	if !r.GetM(cmName, &zsInternalTenantReconfigure) {
+		needReconfigureTenant = true
+	} else {
+		if configHash != zsInternalTenantReconfigure.Data["internal-tenant-config-hash"] ||
+			internalTenantSecretsVersion != zsInternalTenantReconfigure.Data["internal-tenant-secrets-version"] {
+			needReconfigureTenant = true
+			needCMUpdate = true
+		}
+	}
+
 	// We ensure that base secrets are set in the system-config repository
-	if r.SetupBaseSecrets() {
+	baseSecretsInstalled := r.SetupBaseSecrets(internalTenantSecretsVersion)
+
+	if baseSecretsInstalled {
 		// We run zuul tenant-reconfigure for the 'internal' tenant, when:
 		// - the configMap does not exists (or)
 		// - tenant config changed
+		// - tenant secrets version changed
 		// This ensures that the zuul-scheduler loaded the provisionned Zuul config
 		// for the 'internal' tenant
-		var zsInternalTenantReconfigure apiv1.ConfigMap
-		var cmName = "zs-internal-tenant-reconfigure"
-		var needReconfigureTenant = false
-		var needConfigMapUpdate = false
-		var configHash = utils.Checksum([]byte(preInitScriptTemplate))
-		if !r.GetM(cmName, &zsInternalTenantReconfigure) {
-			needReconfigureTenant = true
-		} else {
-			if configHash != zsInternalTenantReconfigure.Data["internal-tenant-config-hash"] {
-				needReconfigureTenant = true
-				needConfigMapUpdate = true
-			}
-		}
 		if needReconfigureTenant {
 			r.log.Info("Running tenant-reconfigure for the 'internal' tenant")
 			if r.runZuulInternalTenantReconfigure() {
@@ -181,9 +197,10 @@ func (r *SFController) SetupConfigJob() bool {
 					Namespace: r.ns,
 				}
 				zsInternalTenantReconfigure.Data = map[string]string{
-					"internal-tenant-config-hash": configHash,
+					"internal-tenant-config-hash":     configHash,
+					"internal-tenant-secrets-version": internalTenantSecretsVersion,
 				}
-				if needConfigMapUpdate {
+				if needCMUpdate {
 					r.UpdateR(&zsInternalTenantReconfigure)
 				} else {
 					r.CreateR(&zsInternalTenantReconfigure)

controllers/libs/monitoring/monitoring.go

@@ -36,7 +36,7 @@ func GetTruncatedPortName(serviceName string, suffix string) string {
 
 const NodeExporterNameSuffix = "-nodeexporter"
 const NodeExporterPortNameSuffix = "-ne"
-const nodeExporterPort = 9100
+const NodeExporterPort = 9100
 
 // Fun fact: arrays cannot be consts, so we define our args in this function.
 func getNodeExporterArgs(volumeMounts []apiv1.VolumeMount) []string {
@@ -52,20 +52,12 @@ func MkNodeExporterSideCarContainer(serviceName string, volumeMounts []apiv1.Vol
 	container := base.MkContainer(serviceName+NodeExporterNameSuffix, base.NodeExporterImage)
 	container.Args = getNodeExporterArgs(volumeMounts)
 	container.Ports = []apiv1.ContainerPort{
-		base.MkContainerPort(nodeExporterPort, GetTruncatedPortName(serviceName, NodeExporterPortNameSuffix)),
+		base.MkContainerPort(NodeExporterPort, GetTruncatedPortName(serviceName, NodeExporterPortNameSuffix)),
 	}
 	container.VolumeMounts = volumeMounts
 	return container
 }
 
-func MkNodeExporterSideCarService(serviceName string, namespace string) apiv1.Service {
-	var portName = GetTruncatedPortName(serviceName, NodeExporterPortNameSuffix)
-	servicePorts := []int32{nodeExporterPort}
-	neService := base.MkService(serviceName+NodeExporterPortNameSuffix, namespace, serviceName, servicePorts, portName)
-	return neService
-
-}
-
 // Statsd exporter utilities
 
 const statsdExporterNameSuffix = "-statsd"

controllers/logserver_controller.go

@@ -174,7 +174,36 @@ func (r *LogServerController) ensureLogserverPromRule() bool {
 	return true
 }
 
+// cleanup ensures removal of legacy resources
+func (r *LogServerController) cleanup() {
+	// Delete apiv1.Service httpdPortName-httpdPort
+	r.DeleteR(&apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.ns,
+			Name:      httpdPortName,
+		},
+	})
+	// Delete apiv1.Service sshdPortName-sshdPort
+	r.DeleteR(&apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.ns,
+			Name:      sshdPortName,
+		},
+	})
+	// Delete apiv1.service logserverIdent-NodeExporterPortNameSuffix
+	r.DeleteR(&apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.ns,
+			Name:      logserverIdent + sfmonitoring.NodeExporterPortNameSuffix,
+		},
+	})
+
+}
+
 func (r *LogServerController) DeployLogserver() sfv1.LogServerStatus {
+
+	r.cleanup()
+
 	log := log.FromContext(r.ctx)
 
 	r.EnsureSSHKeySecret(logserverIdent + "-keys")
@@ -195,6 +224,12 @@ func (r *LogServerController) DeployLogserver() sfv1.LogServerStatus {
 
 	r.EnsureConfigMap(logserverIdent, cmData)
 
+	// Create service exposed by logserver
+	servicePorts := []int32{httpdPort, sshdPort, sfmonitoring.NodeExporterPort}
+	svc := base.MkService(
+		logserverIdent, r.ns, logserverIdent, servicePorts, logserverIdent)
+	r.GetOrCreate(&svc)
+
 	volumeMounts := []apiv1.VolumeMount{
 		{
 			Name:      logserverIdent + "-config-vol",
@@ -273,12 +308,6 @@ func (r *LogServerController) DeployLogserver() sfv1.LogServerStatus {
 		"/usr/bin/" + lgEntryScriptName,
 	}
 
-	// Create services exposed by logserver
-	servicePorts := []int32{httpdPort}
-	httpdService := base.MkService(
-		httpdPortName, r.ns, logserverIdent, servicePorts, httpdPortName)
-	r.GetOrCreate(&httpdService)
-
 	// Setup the sidecar container for sshd
 	sshdContainer := base.MkContainer(sshdPortName, base.SSHDImage)
 	sshdContainer.Command = []string{"bash", "/conf/run.sh"}
@@ -365,21 +394,14 @@ func (r *LogServerController) DeployLogserver() sfv1.LogServerStatus {
 		r.CreateR(&dep)
 	}
 
-	sshdServicePorts := []int32{sshdPort}
-	sshdService := base.MkService(sshdPortName, r.ns, logserverIdent, sshdServicePorts, sshdPortName)
-	r.GetOrCreate(&sshdService)
-
-	nodeExporterSidecarService := sfmonitoring.MkNodeExporterSideCarService(logserverIdent, r.ns)
-	r.GetOrCreate(&nodeExporterSidecarService)
-
 	pvcReadiness := r.reconcileExpandPVC(logserverIdent, r.cr.Spec.Settings.Storage)
 
 	// refresh current deployment
 	r.GetM(dep.GetName(), &currentDep)
 
 	routeReady := r.ensureHTTPSRoute(
 		r.cr.Name+"-logserver", logserverIdent,
-		httpdPortName, "/", httpdPort, map[string]string{}, r.cr.Spec.FQDN, r.cr.Spec.LetsEncrypt)
+		logserverIdent, "/", httpdPort, map[string]string{}, r.cr.Spec.FQDN, r.cr.Spec.LetsEncrypt)
 
 	// TODO(mhu) We may want to open an ingress to port 9100 for an external prometheus instance.
 	// TODO(mhu) we may want to include monitoring objects' status in readiness computation

controllers/static/git-server/update-system-config.sh

@@ -203,9 +203,9 @@ cat << EOF > playbooks/base/post.yaml
        fileserver: "{{ site_sflogs }}"
     -  role: generate-zuul-manifest
 
-- hosts: logserver-sshd
+- hosts: logserver
   vars:
-    ansible_port: ${LOGSERVER_SSHD_SERVICE_PORT}
+    ansible_port: 2222
   gather_facts: false
   tasks:
     - block:

controllers/static/sf_operator/main.py

@@ -31,7 +31,7 @@ def sshkey_scan(port: int, hostname: str) -> bytes:
         ["ssh-keyscan", "-T", "10", "-p", str(port), hostname ])
 
 def get_logserver_fingerprint() -> str:
-    return " ".join(sshkey_scan(2222, "logserver-sshd").decode().split()[0:3])
+    return " ".join(sshkey_scan(2222, "logserver").decode().split()[0:3])
 
 def mk_incluster_k8s_config():
     sa = Path("/run/secrets/kubernetes.io/serviceaccount")
@@ -66,7 +66,7 @@ def create_zuul_secrets():
             ("ssh_private_key", os.environ["ZUUL_LOGSERVER_PRIVATE_KEY"])
         ],
         unencrypted_items=[
-            ("fqdn", "\"[logserver-sshd]:2222\""),
+            ("fqdn", "\"[logserver]:2222\""),
             ("path", "rsync"),
             ("ssh_known_hosts", "\"%s\"" % get_logserver_fingerprint()),
             ("ssh_username", "zuul")

controllers/utils.go

@@ -100,7 +100,7 @@ func (r *SFUtilContext) CreateR(obj client.Object) {
 
 // DeleteR delete a resource.
 func (r *SFUtilContext) DeleteR(obj client.Object) {
-	if err := r.Client.Delete(r.ctx, obj); err != nil {
+	if err := r.Client.Delete(r.ctx, obj); err != nil && !errors.IsNotFound(err) {
 		panic(err.Error())
 	}
 }

doc/developer/howtos/index.md

@@ -107,7 +107,7 @@ microshift cluster inbound, then `firefox https://logserver.test.local`.
 To send data to the logserver, first enable the port-forward:
 
 ```shell
-kubectl -n logserver port-forward service/logserver-sshd 22220:2222
+kubectl -n logserver port-forward service/logserver 22220:2222
 ```
 
 Then use rsync:

roles/health-check/test-volumestats-sidecar/tasks/main.yaml

@@ -9,7 +9,7 @@
 
 - name: Start port forwarding to logserver sidecar
   shell: |
-    kubectl port-forward service/logserver-ne 9100 -n sf
+    kubectl port-forward service/logserver 9100 -n sf
   async: 60
   poll: 0
 
@@ -24,4 +24,4 @@
   register: logserver_metrics
   delay: 5
   retries: 10
-  until: "logserver_metrics.status == 200 and 'data/rsync' in logserver_metrics.content"
+  until: "logserver_metrics.status == 200 and 'data/rsync' in logserver_metrics.content"