Merge "log forwarding: add debug toggle for fluentbit sidecars"

CHANGELOG.md

@@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file.
 ## [in development]
 
 ### Added
+
+- "Debug" toggle for fluent bit sidecars
+
 ### Changed
 ### Deprecated
 ### Removed

api/v1/softwarefactory_types.go

@@ -39,6 +39,10 @@ type FluentBitForwarderSpec struct {
 	// The (optional) port on which to forward logs to, defaults to 80.
 	// +kubebuilder:default:=80
 	HTTPInputPort int32 `json:"httpInputPort,omitempty"`
+	// +kubebuilder:default:=false
+	// +optional
+	// Run fluent bit sidecars in debug mode. This will output forwarded payloads and additional info in the sidecar's logs. Defaults to false.
+	Debug *bool `json:"debug,omitempty"`
 }
 
 type StorageSpec struct {

config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml

@@ -51,6 +51,12 @@ spec:
               FluentBitLogForwarding:
                 description: Enable log forwarding to a [Fluent Bit HTTP input](https://docs.fluentbit.io/manual/pipeline/inputs/http)
                 properties:
+                  debug:
+                    default: false
+                    description: Run fluent bit sidecars in debug mode. This will
+                      output forwarded payloads and additional info in the sidecar's
+                      logs. Defaults to false.
+                    type: boolean
                   httpInputHost:
                     description: The Host for the Fluent Bit HTTP Input to forward
                       logs to.

controllers/libs/base/images.go

@@ -32,7 +32,7 @@ var (
 	HTTPDImage          = ImageToString(Image{Path: "registry.access.redhat.com/ubi8/httpd-24", Version: "1-284.1696531168"})
 	NodeExporterImage   = ImageToString(Image{Path: "quay.io/prometheus/node-exporter", Version: "v1.6.1"})
 	StatsdExporterImage = ImageToString(Image{Path: "quay.io/prometheus/statsd-exporter", Version: "v0.24.0"})
-	FluentBitImage      = ImageToString(Image{Path: "cr.fluentbit.io/fluent/fluent-bit", Version: "2.1.10-debug"})
+	FluentBitImage      = ImageToString(Image{Path: "cr.fluentbit.io/fluent/fluent-bit", Version: "2.1.10"})
 )
 
 func ZuulImage(service string) string {

controllers/libs/logging/logging.go

@@ -54,8 +54,12 @@ func SetupLogForwarding(serviceName string, forwarderSpec *v1.FluentBitForwarder
 	}
 }
 
-func CreateFluentBitSideCarContainer(serviceName string, extraLabels []FluentBitLabel, volumeMounts []apiv1.VolumeMount) apiv1.Container {
-	container := base.MkContainer("fluentbit", base.FluentBitImage)
+func CreateFluentBitSideCarContainer(serviceName string, extraLabels []FluentBitLabel, volumeMounts []apiv1.VolumeMount, debug bool) apiv1.Container {
+	var img = base.FluentBitImage
+	if debug {
+		img += "-debug"
+	}
+	container := base.MkContainer("fluentbit", img)
 	container.Env = CreateForwarderEnvVars(serviceName, extraLabels)
 	ports := []apiv1.ContainerPort{
 		{

controllers/mariadb.go

@@ -44,13 +44,18 @@ type ZuulDBOpts struct {
 func createLogForwarderSidecar(r *SFController, annotations map[string]string) (apiv1.Volume, apiv1.Container) {
 
 	fbForwarderConfig := make(map[string]string)
+	var fbLogLevel = "info"
+	if r.cr.Spec.FluentBitLogForwarding.Debug != nil && *r.cr.Spec.FluentBitLogForwarding.Debug {
+		fbLogLevel = "debug"
+	}
 	fbForwarderConfig["fluent-bit.conf"], _ = utils.ParseString(
 		mariadbFluentBitForwarderConfig,
 		struct {
 			ExtraKeys              []logging.FluentBitLabel
 			FluentBitHTTPInputHost string
 			FluentBitHTTPInputPort string
-		}{[]logging.FluentBitLabel{}, r.cr.Spec.FluentBitLogForwarding.HTTPInputHost, strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort))})
+			LogLevel               string
+		}{[]logging.FluentBitLabel{}, r.cr.Spec.FluentBitLogForwarding.HTTPInputHost, strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort)), fbLogLevel})
 	r.EnsureConfigMap("fluentbit-mariadb-cfg", fbForwarderConfig)
 
 	volume := base.MkVolumeCM("mariadb-log-forwarder-config",
@@ -66,9 +71,13 @@ func createLogForwarderSidecar(r *SFController, annotations map[string]string) (
 			MountPath: "/fluent-bit/etc/",
 		},
 	}
-	sidecar := logging.CreateFluentBitSideCarContainer(MariaDBIdent, []logging.FluentBitLabel{}, volumeMounts)
+	var fluentbitDebug = false
+	if r.cr.Spec.FluentBitLogForwarding.Debug != nil {
+		fluentbitDebug = *r.cr.Spec.FluentBitLogForwarding.Debug
+	}
+	sidecar := logging.CreateFluentBitSideCarContainer(MariaDBIdent, []logging.FluentBitLabel{}, volumeMounts, fluentbitDebug)
 	annotations["mariadb-fluent-bit.conf"] = utils.Checksum([]byte(fbForwarderConfig["fluent-bit.conf"]))
-	annotations["mariadb-fluent-bit-image"] = base.FluentBitImage
+	annotations["mariadb-fluent-bit-image"] = sidecar.Image
 	return volume, sidecar
 }
 

controllers/nodepool.go

@@ -94,14 +94,19 @@ var nodepoolFluentBitLabels = []logging.FluentBitLabel{
 }
 
 func createImageBuildLogForwarderSidecar(r *SFController, annotations map[string]string) (apiv1.Volume, apiv1.Container) {
+	var fbLogLevel = "info"
+	if r.cr.Spec.FluentBitLogForwarding.Debug != nil && *r.cr.Spec.FluentBitLogForwarding.Debug {
+		fbLogLevel = "debug"
+	}
 	fbForwarderConfig := make(map[string]string)
 	fbForwarderConfig["fluent-bit.conf"], _ = utils.ParseString(
 		fluentBitForwarderConfig,
 		struct {
 			ExtraKeys              []logging.FluentBitLabel
 			FluentBitHTTPInputHost string
 			FluentBitHTTPInputPort string
-		}{[]logging.FluentBitLabel{}, r.cr.Spec.FluentBitLogForwarding.HTTPInputHost, strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort))})
+			LogLevel               string
+		}{[]logging.FluentBitLabel{}, r.cr.Spec.FluentBitLogForwarding.HTTPInputHost, strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort)), fbLogLevel})
 	fbForwarderConfig["parsers.conf"] = fluentBitForwarderParsersConfig
 	r.EnsureConfigMap("fluentbit-dib-cfg", fbForwarderConfig)
 
@@ -119,11 +124,15 @@ func createImageBuildLogForwarderSidecar(r *SFController, annotations map[string
 			MountPath: "/fluent-bit/etc/",
 		},
 	}
+	var fluentbitDebug = false
+	if r.cr.Spec.FluentBitLogForwarding.Debug != nil {
+		fluentbitDebug = *r.cr.Spec.FluentBitLogForwarding.Debug
+	}
 	builderFluentBitLabels := append(nodepoolFluentBitLabels, logging.FluentBitLabel{Key: "CONTAINER", Value: BuilderIdent})
-	sidecar := logging.CreateFluentBitSideCarContainer("diskimage-builder", builderFluentBitLabels, volumeMounts)
+	sidecar := logging.CreateFluentBitSideCarContainer("diskimage-builder", builderFluentBitLabels, volumeMounts, fluentbitDebug)
 	annotations["dib-fluent-bit.conf"] = utils.Checksum([]byte(fbForwarderConfig["fluent-bit.conf"]))
 	annotations["dib-fluent-bit-parser"] = utils.Checksum([]byte(fbForwarderConfig["parsers.conf"]))
-	annotations["dib-fluent-bit-image"] = base.FluentBitImage
+	annotations["dib-fluent-bit-image"] = sidecar.Image
 	return volume, sidecar
 
 }

controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl

@@ -1,7 +1,7 @@
 [SERVICE]
     http_server On
     http_port   2020
-    log_level   debug
+    log_level   {{ .LogLevel }}
 [INPUT]
     name                tail
     tag                 ${K8S_NAMESPACE}.${K8S_NODENAME}.${K8S_PODNAME}.mariadb

controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl

@@ -1,7 +1,7 @@
 [SERVICE]
     http_server On
     http_port   2020
-    log_level   debug
+    log_level   {{ .LogLevel }}
     parsers_file    parsers.conf
 [INPUT]
     name                tail

controllers/static/zookeeper/fluent-bit.conf.tmpl

@@ -1,7 +1,7 @@
 [SERVICE]
     http_server On
     http_port   2020
-    log_level   debug
+    log_level   {{ .LogLevel }}
 [INPUT]
     name                tail
     tag                 ${K8S_NAMESPACE}.${K8S_NODENAME}.${K8S_PODNAME}.zookeeper

controllers/zookeeper.go

@@ -48,15 +48,19 @@ const ZookeeperIdent = "zookeeper"
 const zkPIMountPath = "/config-scripts"
 
 func createZKLogForwarderSidecar(r *SFController, annotations map[string]string) (apiv1.Volume, apiv1.Container) {
-
+	var fbLogLevel = "info"
+	if r.cr.Spec.FluentBitLogForwarding.Debug != nil && *r.cr.Spec.FluentBitLogForwarding.Debug {
+		fbLogLevel = "debug"
+	}
 	fbForwarderConfig := make(map[string]string)
 	fbForwarderConfig["fluent-bit.conf"], _ = utils.ParseString(
 		zkFluentBitForwarderConfig,
 		struct {
 			ExtraKeys              []logging.FluentBitLabel
 			FluentBitHTTPInputHost string
 			FluentBitHTTPInputPort string
-		}{[]logging.FluentBitLabel{}, r.cr.Spec.FluentBitLogForwarding.HTTPInputHost, strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort))})
+			LogLevel               string
+		}{[]logging.FluentBitLabel{}, r.cr.Spec.FluentBitLogForwarding.HTTPInputHost, strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort)), fbLogLevel})
 	r.EnsureConfigMap("fluentbit-zk-cfg", fbForwarderConfig)
 
 	volume := base.MkVolumeCM("zk-log-forwarder-config",
@@ -72,8 +76,13 @@ func createZKLogForwarderSidecar(r *SFController, annotations map[string]string)
 			MountPath: "/fluent-bit/etc/",
 		},
 	}
-	sidecar := logging.CreateFluentBitSideCarContainer("zookeeper", []logging.FluentBitLabel{}, volumeMounts)
+	var fluentbitDebug = false
+	if r.cr.Spec.FluentBitLogForwarding.Debug != nil {
+		fluentbitDebug = *r.cr.Spec.FluentBitLogForwarding.Debug
+	}
+	sidecar := logging.CreateFluentBitSideCarContainer("zookeeper", []logging.FluentBitLabel{}, volumeMounts, fluentbitDebug)
 	annotations["zk-fluent-bit.conf"] = utils.Checksum([]byte(fbForwarderConfig["fluent-bit.conf"]))
+	annotations["zk-fluent-bit-image"] = sidecar.Image
 	return volume, sidecar
 }
 

doc/reference/api/index.md

@@ -82,6 +82,7 @@ _Appears in:_
 | --- | --- | --- |
 | `httpInputHost` _string_ | The Host for the Fluent Bit HTTP Input to forward logs to. | -|
 | `httpInputPort` _integer_ | The (optional) port on which to forward logs to, defaults to 80. | {80}|
+| `debug` _boolean_ | Run fluent bit sidecars in debug mode. This will output forwarded payloads and additional info in the sidecar's logs. Defaults to false. | {false}|
 
 
 #### GerritConnection

roles/post/get-k8s-resources/tasks/main.yaml

@@ -59,16 +59,19 @@
     done
   ignore_errors: true
 
-- name: Create Pods Logs directory
-  ansible.builtin.file:
-    path: "{{ dest_dir }}/pod"
-    state: directory
-    mode: "0755"
+- name: Collect pod logs in namespace
+  when: namespace != "sf"
+  block:
+  - name: Create Pods Logs directory
+    ansible.builtin.file:
+      path: "{{ dest_dir }}/pod"
+      state: directory
+      mode: "0755"
 
-- name: Collect namespace pods logs
-  ansible.builtin.shell: |
-    for pod in $(kubectl {{ namespace_opt }} get pods -o name); do
-      for container in $(kubectl {{ namespace_opt }} get $pod -o jsonpath='{.spec.containers[*].name}'); do
-        kubectl {{ namespace_opt }} logs {{ logs_arguments }} $pod -c $container > {{ dest_dir }}/${pod}-${container}-logs.txt
+  - name: Collect namespace pods logs
+    ansible.builtin.shell: |
+      for pod in $(kubectl {{ namespace_opt }} get pods -o name); do
+        for container in $(kubectl {{ namespace_opt }} get $pod -o jsonpath='{.spec.containers[*].name}'); do
+          kubectl {{ namespace_opt }} logs {{ logs_arguments }} $pod -c $container > {{ dest_dir }}/${pod}-${container}-logs.txt
+        done
       done
-    done

tools/loki/test-fluentbit.yaml

@@ -39,7 +39,8 @@ metadata:
   name: test-fluentbit
 spec:
   containers:
-  - image: cr.fluentbit.io/fluent/fluent-bit:2.1.10-debug
+  # append "-debug" to image name to run in debug mode, in case you run into CrashLoopBackOff errors.
+  - image: cr.fluentbit.io/fluent/fluent-bit:2.1.10
     name: test-fluentbit
     resources: {}
     securityContext: