Log Forwarding: enable forwarding nodepool build logs

If log forwarding is enabled, create a fluent bit sidecar on the
nodepool-builder statefulset that forwards logs a json payloads,
ie the way Nodepool and Zuul process do too.

Include a simple Ansible output callback plugin to timestamp
each line in ansible-playbook's output, which allows these log
lines to be queried in order in the log collector.

Change-Id: I5b4779b58f72aab41de3dd3262f0e1d8f90d5566

Author: mhuin

controllers/libs/base/images.go

@@ -32,6 +32,7 @@ var (
 	HTTPDImage          = ImageToString(Image{Path: "registry.access.redhat.com/ubi8/httpd-24", Version: "1-284.1696531168"})
 	NodeExporterImage   = ImageToString(Image{Path: "quay.io/prometheus/node-exporter", Version: "v1.6.1"})
 	StatsdExporterImage = ImageToString(Image{Path: "quay.io/prometheus/statsd-exporter", Version: "v0.24.0"})
+	FluentBitImage      = ImageToString(Image{Path: "cr.fluentbit.io/fluent/fluent-bit", Version: "2.1.10-debug"})
 )
 
 func ZuulImage(service string) string {

controllers/libs/logging/logging.go

@@ -53,3 +53,17 @@ func SetupLogForwarding(serviceName string, forwarderSpec *v1.FluentBitForwarder
 		return []apiv1.EnvVar{}
 	}
 }
+
+func CreateFluentBitSideCarContainer(serviceName string, extraLabels []FluentBitLabel, volumeMounts []apiv1.VolumeMount) apiv1.Container {
+	container := base.MkContainer("fluentbit", base.FluentBitImage)
+	container.Env = CreateForwarderEnvVars(serviceName, extraLabels)
+	ports := []apiv1.ContainerPort{
+		{
+			Name:          "fb-http-server",
+			ContainerPort: 2020,
+		},
+	}
+	container.Ports = ports
+	container.VolumeMounts = volumeMounts
+	return container
+}

controllers/nodepool.go

@@ -41,6 +41,20 @@ var nodepoolStatsdMappingConfigTemplate string
 //go:embed static/nodepool/httpd-build-logs-dir.conf
 var httpdBuildLogsDirConfig string
 
+//go:embed static/nodepool/fluentbit/parsers.conf
+var fluentBitForwarderParsersConfig string
+
+//go:embed static/nodepool/fluentbit/fluent-bit.conf.tmpl
+var fluentBitForwarderConfig string
+
+// ansible.cfg and the timestamp callback could be hard-coded in the nodepool builder container.
+//
+//go:embed static/nodepool/ansible/ansible.cfg
+var ansibleConfiguration string
+
+//go:embed static/nodepool/ansible/timestamp.py
+var timestampOutputCallback string
+
 const (
 	nodepoolIdent                = "nodepool"
 	launcherIdent                = nodepoolIdent + "-launcher"
@@ -69,11 +83,47 @@ var nodepoolFluentBitLabels = []logging.FluentBitLabel{
 	},
 }
 
+func createImageBuildLogForwarderSidecar(r *SFController, annotations map[string]string) (apiv1.Volume, apiv1.Container) {
+	fbForwarderConfig := make(map[string]string)
+	fbForwarderConfig["fluent-bit.conf"], _ = utils.ParseString(
+		fluentBitForwarderConfig,
+		struct {
+			ExtraKeys              []logging.FluentBitLabel
+			FluentBitHTTPInputHost string
+			FluentBitHTTPInputPort string
+		}{[]logging.FluentBitLabel{}, r.cr.Spec.FluentBitLogForwarding.HTTPInputHost, strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort))})
+	fbForwarderConfig["parsers.conf"] = fluentBitForwarderParsersConfig
+	r.EnsureConfigMap("fluentbit-dib-cfg", fbForwarderConfig)
+
+	volume := base.MkVolumeCM("dib-log-forwarder-config",
+		"fluentbit-dib-cfg-config-map")
+
+	volumeMounts := []apiv1.VolumeMount{
+		{
+			Name:      builderIdent,
+			SubPath:   "builds",
+			MountPath: "/watch/",
+		},
+		{
+			Name:      "dib-log-forwarder-config",
+			MountPath: "/fluent-bit/etc/",
+		},
+	}
+	sidecar := logging.CreateFluentBitSideCarContainer("diskimage-builder", nodepoolFluentBitLabels, volumeMounts)
+	annotations["dib-fluent-bit.conf"] = utils.Checksum([]byte(fbForwarderConfig["fluent-bit.conf"]))
+	annotations["dib-fluent-bit-parser"] = utils.Checksum([]byte(fbForwarderConfig["parsers.conf"]))
+	annotations["dib-fluent-bit-image"] = base.FluentBitImage
+	return volume, sidecar
+
+}
+
 func (r *SFController) setNodepoolTooling() {
 	toolingData := make(map[string]string)
 	toolingData["generate-config.sh"] = generateConfigScript
 	toolingData["dib-ansible.py"] = dibAnsibleWrapper
 	toolingData["ssh_config"] = builderSSHConfig
+	toolingData["timestamp.py"] = timestampOutputCallback
+	toolingData["ansible.cfg"] = ansibleConfiguration
 	r.EnsureConfigMap("nodepool-tooling", toolingData)
 }
 
@@ -437,6 +487,18 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
 			MountPath: "/etc/nodepool-logging/logging.yaml",
 			ReadOnly:  true,
 		},
+		{
+			Name:      "nodepool-tooling-vol",
+			SubPath:   "ansible.cfg",
+			MountPath: "/etc/ansible/ansible.cfg",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "nodepool-tooling-vol",
+			SubPath:   "timestamp.py",
+			MountPath: "/usr/share/ansible/plugins/callback/timestamp.py",
+			ReadOnly:  true,
+		},
 	}
 
 	nodepoolProvidersSecrets, volumeMount, ready := r.setProviderSecretsVolumeMounts(volumeMount)
@@ -488,6 +550,11 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
 
 	extraLoggingEnvVars := logging.SetupLogForwarding("nodepool-builder", r.cr.Spec.FluentBitLogForwarding, nodepoolFluentBitLabels, annotations)
 	nb.Spec.Template.Spec.Containers[0].Env = append(nb.Spec.Template.Spec.Containers[0].Env, extraLoggingEnvVars...)
+	if r.cr.Spec.FluentBitLogForwarding != nil {
+		fbVolume, fbSidecar := createImageBuildLogForwarderSidecar(r, annotations)
+		nb.Spec.Template.Spec.Containers = append(nb.Spec.Template.Spec.Containers, fbSidecar)
+		nb.Spec.Template.Spec.Volumes = append(nb.Spec.Template.Spec.Volumes, fbVolume)
+	}
 
 	nb.Spec.Template.ObjectMeta.Annotations = annotations
 

controllers/static/nodepool/ansible/ansible.cfg

@@ -0,0 +1,2 @@
+[defaults]
+stdout_callback=timestamp