Merge "Zuul, Nodepool: Enable log forwarding with Fluent Bit"

api/v1/softwarefactory_types.go

@@ -33,6 +33,14 @@ type LetsEncryptSpec struct {
 	Server LEServer `json:"server"`
 }
 
+type FluentBitForwarderSpec struct {
+	// The Host for the Fluent Bit HTTP Input to forward logs to.
+	HTTPInputHost string `json:"httpInputHost"`
+	// The (optional) port on which to forward logs to, defaults to 80.
+	// +kubebuilder:default:=80
+	HTTPInputPort int32 `json:"httpInputPort,omitempty"`
+}
+
 type StorageSpec struct {
 	// Storage space to allocate to the resource, expressed as a [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/)
 	Size resource.Quantity `json:"size"`
@@ -385,6 +393,9 @@ type SoftwareFactorySpec struct {
 	// LetsEncrypt settings for enabling using LetsEncrypt for Routes/TLS
 	LetsEncrypt *LetsEncryptSpec `json:"letsEncrypt,omitempty"`
 
+	// Enable log forwarding to a [Fluent Bit HTTP input](https://docs.fluentbit.io/manual/pipeline/inputs/http)
+	FluentBitLogForwarding *FluentBitForwarderSpec `json:"FluentBitLogForwarding,omitempty"`
+
 	// Default storage class to use by Persistent Volume Claims
 	StorageClassName string `json:"storageClassName,omitempty"`
 

config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml

@@ -48,6 +48,22 @@ spec:
           spec:
             description: SoftwareFactorySpec defines the desired state of SoftwareFactory
             properties:
+              FluentBitLogForwarding:
+                description: Enable log forwarding to a [Fluent Bit HTTP input](https://docs.fluentbit.io/manual/pipeline/inputs/http)
+                properties:
+                  httpInputHost:
+                    description: The Host for the Fluent Bit HTTP Input to forward
+                      logs to.
+                    type: string
+                  httpInputPort:
+                    default: 80
+                    description: The (optional) port on which to forward logs to,
+                      defaults to 80.
+                    format: int32
+                    type: integer
+                required:
+                - httpInputHost
+                type: object
               config-location:
                 description: Config repository spec
                 properties:

controllers/libs/base/base.go

@@ -129,6 +129,17 @@ func MkEnvVar(env string, value string) apiv1.EnvVar {
 	}
 }
 
+func MkEnvVarFromFieldRef(env string, fieldPath string) apiv1.EnvVar {
+	return apiv1.EnvVar{
+		Name: env,
+		ValueFrom: &apiv1.EnvVarSource{
+			FieldRef: &apiv1.ObjectFieldSelector{
+				FieldPath: fieldPath,
+			},
+		},
+	}
+}
+
 // MkSecretFromFunc produces a Secret where data is the result of getData
 func MkSecretFromFunc(name string, namespace string, getData func() string) apiv1.Secret {
 	return apiv1.Secret{

controllers/libs/base/images.go

@@ -14,8 +14,8 @@ func ImageToString(i Image) string {
 }
 
 const (
-	nodepoolImageVersion = "9.0.0-6"
-	zuulImageVersion     = "9.2.0-1"
+	nodepoolImageVersion = "9.0.0-8"
+	zuulImageVersion     = "9.2.0-3"
 )
 
 var (

controllers/libs/logging/logging.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2023 Red Hat
+// SPDX-License-Identifier: Apache-2.0
+
+/*
+Package logging provides various utility functions regarding
+optional service log collection for the sf-operator:
+
+* create fluent bit sidecar
+*/
+package logging
+
+import (
+	_ "embed"
+	"strconv"
+
+	v1 "github.com/softwarefactory-project/sf-operator/api/v1"
+	"github.com/softwarefactory-project/sf-operator/controllers/libs/base"
+	apiv1 "k8s.io/api/core/v1"
+)
+
+type FluentBitLabel struct {
+	Key   string
+	Value string
+}
+
+type PythonTemplateLoggingParams struct {
+	LogLevel    string
+	ForwardLogs bool
+	BaseURL     string
+}
+
+func CreateForwarderEnvVars(name string, extraLabels []FluentBitLabel) []apiv1.EnvVar {
+	forwarderEnvVars := []apiv1.EnvVar{
+		base.MkEnvVarFromFieldRef("K8S_NODENAME", "spec.nodeName"),
+		base.MkEnvVarFromFieldRef("K8S_PODNAME", "metadata.name"),
+		base.MkEnvVarFromFieldRef("K8S_NAMESPACE", "metadata.namespace"),
+		base.MkEnvVarFromFieldRef("K8S_PODIP", "status.podIP"),
+		base.MkEnvVar("K8S_LABELS_RUN", name),
+	}
+	for i := range extraLabels {
+		var v = base.MkEnvVar("K8S_"+extraLabels[i].Key, extraLabels[i].Value)
+		forwarderEnvVars = append(forwarderEnvVars, v)
+	}
+	return forwarderEnvVars
+}
+
+func SetupLogForwarding(serviceName string, forwarderSpec *v1.FluentBitForwarderSpec, extraLabels []FluentBitLabel, annotations map[string]string) []apiv1.EnvVar {
+	if forwarderSpec != nil {
+		annotations["log-forwarding"] = forwarderSpec.HTTPInputHost + ":" + strconv.Itoa(int(forwarderSpec.HTTPInputPort))
+		return CreateForwarderEnvVars(serviceName, extraLabels)
+	} else {
+		annotations["log-forwarding"] = "disabled"
+		return []apiv1.EnvVar{}
+	}
+}

controllers/nodepool.go

@@ -12,6 +12,7 @@ import (
 	v1 "github.com/softwarefactory-project/sf-operator/api/v1"
 	"github.com/softwarefactory-project/sf-operator/controllers/libs/base"
 	"github.com/softwarefactory-project/sf-operator/controllers/libs/conds"
+	logging "github.com/softwarefactory-project/sf-operator/controllers/libs/logging"
 	"github.com/softwarefactory-project/sf-operator/controllers/libs/monitoring"
 	"github.com/softwarefactory-project/sf-operator/controllers/libs/utils"
 
@@ -61,6 +62,13 @@ var configScriptVolumeMount = apiv1.VolumeMount{
 	ReadOnly:  true,
 }
 
+var nodepoolFluentBitLabels = []logging.FluentBitLabel{
+	{
+		Key:   "SERVICE",
+		Value: "nodepool",
+	},
+}
+
 func (r *SFController) setNodepoolTooling() {
 	toolingData := make(map[string]string)
 	toolingData["generate-config.sh"] = generateConfigScript
@@ -104,15 +112,33 @@ func (r *SFController) getNodepoolConfigEnvs() []apiv1.EnvVar {
 	return nodepoolEnvVars
 }
 
-func mkLoggingTemplate(logLevel v1.LogLevel) (string, error) {
+func (r *SFController) mkLoggingTemplate(serviceName string) (string, error) {
 	// Unfortunatly I'm unable to leverage default value set at API validation
 	selectedLogLevel := v1.InfoLogLevel
+	var logLevel v1.LogLevel
+	if serviceName == "builder" {
+		logLevel = r.cr.Spec.Nodepool.Builder.LogLevel
+	} else {
+		logLevel = r.cr.Spec.Nodepool.Launcher.LogLevel
+	}
 	if logLevel != "" {
 		selectedLogLevel = logLevel
 	}
 
+	var forwardLogs = false
+	var inputBaseURL = ""
+	if r.cr.Spec.FluentBitLogForwarding != nil {
+		forwardLogs = true
+		inputBaseURL = "http://" + r.cr.Spec.FluentBitLogForwarding.HTTPInputHost + ":" + strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort))
+	}
+
 	loggingConfig, err := utils.ParseString(
-		loggingConfigTemplate, struct{ LogLevel string }{LogLevel: string(selectedLogLevel)})
+		loggingConfigTemplate,
+		logging.PythonTemplateLoggingParams{
+			LogLevel:    string(selectedLogLevel),
+			ForwardLogs: forwardLogs,
+			BaseURL:     inputBaseURL,
+		})
 
 	return loggingConfig, err
 }
@@ -322,7 +348,7 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
 
 	r.setNodepoolTooling()
 
-	loggingConfig, _ := mkLoggingTemplate(r.cr.Spec.Nodepool.Builder.LogLevel)
+	loggingConfig, _ := r.mkLoggingTemplate("builder")
 
 	builderExtraConfigData := make(map[string]string)
 	builderExtraConfigData["logging.yaml"] = loggingConfig
@@ -453,14 +479,18 @@ func (r *SFController) DeployNodepoolBuilder(statsdExporterVolume apiv1.Volume,
 		builderIdent, base.NodepoolBuilderImage, r.getStorageConfOrDefault(r.cr.Spec.Nodepool.Builder.Storage),
 		replicas, apiv1.ReadWriteOnce)
 
-	nb.Spec.Template.ObjectMeta.Annotations = annotations
 	nb.Spec.Template.Spec.InitContainers = []apiv1.Container{initContainer}
 	nb.Spec.Template.Spec.Volumes = volumes
 	nb.Spec.Template.Spec.Containers[0].Command = []string{"/usr/local/bin/dumb-init", "--",
 		"/usr/local/bin/nodepool-builder", "-f", "-l", "/etc/nodepool-logging/logging.yaml"}
 	nb.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMount
 	nb.Spec.Template.Spec.Containers[0].Env = r.getNodepoolConfigEnvs()
 
+	extraLoggingEnvVars := logging.SetupLogForwarding("nodepool-builder", r.cr.Spec.FluentBitLogForwarding, nodepoolFluentBitLabels, annotations)
+	nb.Spec.Template.Spec.Containers[0].Env = append(nb.Spec.Template.Spec.Containers[0].Env, extraLoggingEnvVars...)
+
+	nb.Spec.Template.ObjectMeta.Annotations = annotations
+
 	// Append statsd exporter sidecar
 	nb.Spec.Template.Spec.Containers = append(nb.Spec.Template.Spec.Containers,
 		monitoring.MkStatsdExporterSideCarContainer(shortIdent, "statsd-config", relayAddress),
@@ -523,7 +553,7 @@ func (r *SFController) DeployNodepoolLauncher(statsdExporterVolume apiv1.Volume,
 
 	r.setNodepoolTooling()
 
-	loggingConfig, _ := mkLoggingTemplate(r.cr.Spec.Nodepool.Launcher.LogLevel)
+	loggingConfig, _ := r.mkLoggingTemplate("launcher")
 
 	// get statsd relay if defined
 	var relayAddress *string
@@ -595,6 +625,9 @@ func (r *SFController) DeployNodepoolLauncher(statsdExporterVolume apiv1.Volume,
 		"/usr/local/bin/nodepool-launcher", "-f", "-l", "/etc/nodepool-logging/logging.yaml"}
 	container.Env = r.getNodepoolConfigEnvs()
 
+	extraLoggingEnvVars := logging.SetupLogForwarding("nodepool-launcher", r.cr.Spec.FluentBitLogForwarding, nodepoolFluentBitLabels, annotations)
+	container.Env = append(container.Env, extraLoggingEnvVars...)
+
 	initContainer := base.MkContainer("nodepool-launcher-init", base.BusyboxImage)
 
 	initContainer.Command = []string{"/usr/local/bin/generate-config.sh"}

controllers/static/nodepool/logging.yaml.tmpl

@@ -2,20 +2,38 @@ version: 1
 formatters:
   console:
     class: 'nodepool.logconfig.MultiLineFormatter'
-    format: "%(asctime)s %(levelname)7s %(name)s: %(message)s"
+    format: "%(asctime)s %(levelname)s %(name)s: %(message)s"
+{{- if .ForwardLogs }}
+  logforward:
+    format: '%(asctime)s %(levelname)s %(name)s: %(message)s'
+{{ end }}
 handlers:
   console:
     class: logging.StreamHandler
     formatter: console
     level: {{.LogLevel}}
     stream: ext://sys.stdout
+{{- if .ForwardLogs }}
+  logforward:
+    class: sfExtras.SimpleFluentBitHTTPInputHandler
+    level: {{ .LogLevel }}
+    formatter:
+    url: '{{ .BaseURL }}/nodepool'
+    env_prefix: K8S_
+{{ end }}
 loggers:
   nodepool:
     handlers:
       - console
+{{- if .ForwardLogs }}
+      - logforward
+{{ end }}
     level: {{.LogLevel}}
     propagate: 0
 root:
   handlers:
     - console
+{{- if .ForwardLogs }}
+    - logforward
+{{ end }}
   level: {{.LogLevel}}

controllers/static/zuul/logging.yaml.tmpl

@@ -2,36 +2,63 @@ version: 1
 formatters:
   console:
     class: 'zuul.lib.logutil.MultiLineFormatter'
-    format: '%(levelname)7s %(name)s: %(message)s'
+    format: '%(asctime)s %(levelname)s %(name)s: %(message)s'
+{{- if .ForwardLogs }}
+  logforward:
+    format: '%(asctime)s %(levelname)s %(name)s: %(message)s'
+{{ end }}
 handlers:
   console:
     class: logging.StreamHandler
     formatter: console
     level: {{.LogLevel}}
     stream: ext://sys.stdout
+{{- if .ForwardLogs }}
+  logforward:
+    class: sfExtras.SimpleFluentBitHTTPInputHandler
+    level: {{ .LogLevel }}
+    formatter: logforward
+    url: '{{ .BaseURL }}/zuul'
+    env_prefix: K8S_
+{{ end }}
 loggers:
   zuul.GerritConnection.io:
     handlers:
       - console
+{{- if .ForwardLogs }}
+      - logforward
+{{ end }}
     level: {{ .LogLevel }}
     propagate: 0
   sqlalchemy.engine:
     handlers:
       - console
+{{- if .ForwardLogs }}
+      - logforward
+{{ end }}
     level: {{ .LogLevel }}
     propagate: 0
   connection:
     handlers:
       - console
+{{- if .ForwardLogs }}
+      - logforward
+{{ end }}
     level: {{ .LogLevel }}
     propagate: 0
   zuul:
     handlers:
       - console
+ {{- if .ForwardLogs }}
+      - logforward
+{{ end }}
     level: {{ .LogLevel }}
     propagate: 0
 root:
   handlers:
     - console
+{{- if .ForwardLogs }}
+    - logforward
+{{ end }}
   level: {{ .LogLevel }}