Enable fluent bit Forward Output

Change-Id: I678089a224c1b8dd3dace9e4af8a864b82abb3a4

api/v1/softwarefactory_types.go

@@ -35,7 +35,7 @@ type LetsEncryptSpec struct {
 
 type FluentBitForwarderSpec struct {
 	// The Host for the Fluent Bit HTTP Input to forward logs to.
-	HTTPInputHost string `json:"httpInputHost"`
+	HTTPInputHost string `json:"httpInputHost,omitempty"`
 	// The (optional) port on which to forward logs to, defaults to 80.
 	// +kubebuilder:default:=80
 	HTTPInputPort int32 `json:"httpInputPort,omitempty"`

config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml

@@ -82,8 +82,6 @@ spec:
                       defaults to 80.
                     format: int32
                     type: integer
-                required:
-                - httpInputHost
                 type: object
               config-location:
                 description: Config repository spec

controllers/libs/logging/logging.go

@@ -52,6 +52,24 @@ func CreateForwarderEnvVars(name string, extraLabels []FluentBitLabel) []apiv1.E
 	return forwarderEnvVars
 }
 
+func CreateBaseLoggingExtraKeys(name string, component string) []FluentBitLabel {
+	baseExtraKeys := []FluentBitLabel{
+		{
+			Key:   "labels_app",
+			Value: "sf",
+		},
+		{
+			Key:   "labels_run",
+			Value: name,
+		},
+		{
+			Key:   "component",
+			Value: component,
+		},
+	}
+	return baseExtraKeys
+}
+
 func CreateForwarderConfigTemplateParams(tag string, forwarderSpec *v1.FluentBitForwarderSpec) TemplateLoggingParams {
 	var httpInputParams = TemplateInputParams{
 		InUse: false,

controllers/nodepool.go

@@ -194,6 +194,7 @@ func (r *SFController) mkLoggingTemplate(serviceName string) (string, error) {
 	}
 
 	var loggingParams = logging.CreateForwarderConfigTemplateParams("nodepool."+serviceName, r.cr.Spec.FluentBitLogForwarding)
+	var loggingExtraKeys = logging.CreateBaseLoggingExtraKeys("nodepool-"+serviceName, "nodepool")
 	// Change logLevel to what we actually want
 	loggingParams.LogLevel = string(selectedLogLevel)
 
@@ -202,7 +203,7 @@ func (r *SFController) mkLoggingTemplate(serviceName string) (string, error) {
 		struct {
 			ExtraKeys     []logging.FluentBitLabel
 			LoggingParams logging.TemplateLoggingParams
-		}{[]logging.FluentBitLabel{}, loggingParams})
+		}{loggingExtraKeys, loggingParams})
 
 	return loggingConfig, err
 }

controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl

@@ -4,8 +4,8 @@
     log_level   {{ .LoggingParams.LogLevel }}
 [INPUT]
     name                tail
-    tag                 mariadb
-    path                /watch/*.log
+    tag                 mariadb.error
+    path                /watch/error.log
     path_key            full_path
     refresh_interval    5
     read_from_head      True
@@ -24,6 +24,10 @@
 {{- range .ExtraKeys }}
     add     {{ .Key }} ${K8S_{{ .Value -}}}
 {{- end }}
+[FILTER]
+    name    modify
+    match   mariadb.error
+    add     level ERROR
 {{- if eq .LoggingParams.LogLevel "debug" }}
 [OUTPUT]
     name    stdout
@@ -38,4 +42,12 @@
     format  json
     host    {{ .LoggingParams.HTTPInputConfig.Host }}
     port    {{ .LoggingParams.HTTPInputConfig.Port }}
-{{ end }}
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+[OUTPUT]
+    name    forward
+    match   *
+    tag     {{ .LoggingParams.Tag }}
+    host    {{ .LoggingParams.ForwardInputConfig.Host }}
+    port    {{ .LoggingParams.ForwardInputConfig.Port }}
+{{ end }}

controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl

@@ -48,7 +48,7 @@
     Reserve_Data On
 [FILTER]
     Name modify
-    Match *dib
+    Match diskimage-builder
     Remove full_path
 {{- if eq .LoggingParams.LogLevel "debug" }}
 [OUTPUT]
@@ -64,4 +64,12 @@
     format  json
     host    {{ .LoggingParams.HTTPInputConfig.Host }}
     port    {{ .LoggingParams.HTTPInputConfig.Port }}
-{{ end }}
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+[OUTPUT]
+    name    forward
+    match   *
+    tag     {{ .LoggingParams.Tag }}
+    host    {{ .LoggingParams.ForwardInputConfig.Host }}
+    port    {{ .LoggingParams.ForwardInputConfig.Port }}
+{{ end }}

controllers/static/nodepool/logging.yaml.tmpl

@@ -7,6 +7,16 @@ formatters:
   fluenthttp:
     format: '%(asctime)s %(levelname)s %(name)s: %(message)s'
 {{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+  fluentforward:
+    '()': fluent.handler.FluentRecordFormatter
+    format:
+      level: '%(levelname)s'
+      log: '%(name)s: %(message)s'
+{{- range .ExtraKeys }}
+      {{ .Key }}: '{{ .Value -}}'
+{{- end }}
+{{ end }}
 handlers:
   console:
     class: logging.StreamHandler
@@ -21,12 +31,25 @@ handlers:
     url: 'http://{{ .LoggingParams.HTTPInputConfig.Host }}:{{ .LoggingParams.HTTPInputConfig.Port }}/{{ .LoggingParams.Tag }}'
     env_prefix: K8S_
 {{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+  fluentforward:
+    class: fluent.handler.FluentHandler
+    level: {{ .LoggingParams.LogLevel }}
+    formatter: fluentforward
+#    buffer_overflow_handler: overflow_handler
+    host: {{ .LoggingParams.ForwardInputConfig.Host }}
+    port: {{ .LoggingParams.ForwardInputConfig.Port }}
+    tag: {{ .LoggingParams.Tag }}
+{{ end }}
 loggers:
   nodepool:
     handlers:
       - console
 {{- if .LoggingParams.HTTPInputConfig.InUse }}
       - fluenthttp
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+      - fluentforward
 {{ end }}
     level: {{.LoggingParams.LogLevel}}
     propagate: 0
@@ -35,5 +58,8 @@ root:
     - console
 {{- if .LoggingParams.HTTPInputConfig.InUse }}
     - fluenthttp
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+    - fluentforward
 {{ end }}
   level: {{.LoggingParams.LogLevel}}

controllers/static/zookeeper/fluent-bit.conf.tmpl

@@ -38,4 +38,12 @@
     format  json
     host    {{ .LoggingParams.HTTPInputConfig.Host }}
     port    {{ .LoggingParams.HTTPInputConfig.Port }}
-{{ end }}
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+[OUTPUT]
+    name    forward
+    match   *
+    tag     {{ .LoggingParams.Tag }}
+    host    {{ .LoggingParams.ForwardInputConfig.Host }}
+    port    {{ .LoggingParams.ForwardInputConfig.Port }}
+{{ end }}

controllers/static/zuul/logging.yaml.tmpl

@@ -7,6 +7,16 @@ formatters:
   fluenthttp:
     format: '%(asctime)s %(levelname)s %(name)s: %(message)s'
 {{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+  fluentforward:
+    '()': fluent.handler.FluentRecordFormatter
+    format:
+      level: '%(levelname)s'
+      log: '%(name)s: %(message)s'
+{{- range .ExtraKeys }}
+      {{ .Key }}: '{{ .Value -}}'
+{{- end }}
+{{ end }}
 handlers:
   console:
     class: logging.StreamHandler
@@ -21,12 +31,25 @@ handlers:
     url: 'http://{{ .LoggingParams.HTTPInputConfig.Host }}:{{ .LoggingParams.HTTPInputConfig.Port }}/{{ .LoggingParams.Tag }}'
     env_prefix: K8S_
 {{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+  fluentforward:
+    class: fluent.handler.FluentHandler
+    level: {{ .LoggingParams.LogLevel }}
+    formatter: fluentforward
+#    buffer_overflow_handler: overflow_handler
+    host: {{ .LoggingParams.ForwardInputConfig.Host }}
+    port: {{ .LoggingParams.ForwardInputConfig.Port }}
+    tag: {{ .LoggingParams.Tag }}
+{{ end }}
 loggers:
   zuul.GerritConnection.ssh:
     handlers:
       - console
 {{- if .LoggingParams.HTTPInputConfig.InUse }}
       - fluenthttp
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+      - fluentforward
 {{ end }}
     level: INFO
     propagate: 0
@@ -35,6 +58,9 @@ loggers:
       - console
 {{- if .LoggingParams.HTTPInputConfig.InUse }}
       - fluenthttp
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+      - fluentforward
 {{ end }}
     level: {{ .LoggingParams.LogLevel }}
     propagate: 0
@@ -43,6 +69,9 @@ loggers:
       - console
 {{- if .LoggingParams.HTTPInputConfig.InUse }}
       - fluenthttp
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+      - fluentforward
 {{ end }}
     level: {{ .LoggingParams.LogLevel }}
     propagate: 0
@@ -51,6 +80,9 @@ loggers:
       - console
 {{- if .LoggingParams.HTTPInputConfig.InUse }}
       - fluenthttp
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+      - fluentforward
 {{ end }}
     level: {{ .LoggingParams.LogLevel }}
     propagate: 0
@@ -59,6 +91,8 @@ root:
     - console
 {{- if .LoggingParams.HTTPInputConfig.InUse }}
     - fluenthttp
+{{ end }}
+{{- if .LoggingParams.ForwardInputConfig.InUse }}
+    - fluentforward
 {{ end }}
   level: {{ .LoggingParams.LogLevel }}
-

controllers/zuul.go

@@ -385,44 +385,48 @@ func (r *SFController) computeLoggingConfig() map[string]string {
 	}
 
 	var zeloggingParams = logging.CreateForwarderConfigTemplateParams("zuul.executor", r.cr.Spec.FluentBitLogForwarding)
+	var zeloggingExtraKeys = logging.CreateBaseLoggingExtraKeys("zuul-executor", "zuul")
 	// Change logLevel to what we actually want
 	zeloggingParams.LogLevel = string(zuulExecutorLogLevel)
 	loggingData["zuul-executor-logging.yaml"], _ = utils.ParseString(
 		zuulLoggingConfig,
 		struct {
 			ExtraKeys     []logging.FluentBitLabel
 			LoggingParams logging.TemplateLoggingParams
-		}{[]logging.FluentBitLabel{}, zeloggingParams})
+		}{zeloggingExtraKeys, zeloggingParams})
 
 	var zsloggingParams = logging.CreateForwarderConfigTemplateParams("zuul.scheduler", r.cr.Spec.FluentBitLogForwarding)
+	var zsloggingExtraKeys = logging.CreateBaseLoggingExtraKeys("zuul-scheduler", "zuul")
 	// Change logLevel to what we actually want
 	zsloggingParams.LogLevel = string(zuulSchedulerLogLevel)
 	loggingData["zuul-scheduler-logging.yaml"], _ = utils.ParseString(
 		zuulLoggingConfig,
 		struct {
 			ExtraKeys     []logging.FluentBitLabel
 			LoggingParams logging.TemplateLoggingParams
-		}{[]logging.FluentBitLabel{}, zsloggingParams})
+		}{zsloggingExtraKeys, zsloggingParams})
 
 	var zwloggingParams = logging.CreateForwarderConfigTemplateParams("zuul.web", r.cr.Spec.FluentBitLogForwarding)
+	var zwloggingExtraKeys = logging.CreateBaseLoggingExtraKeys("zuul-web", "zuul")
 	// Change logLevel to what we actually want
 	zwloggingParams.LogLevel = string(zuulWebLogLevel)
 	loggingData["zuul-web-logging.yaml"], _ = utils.ParseString(
 		zuulLoggingConfig,
 		struct {
 			ExtraKeys     []logging.FluentBitLabel
 			LoggingParams logging.TemplateLoggingParams
-		}{[]logging.FluentBitLabel{}, zwloggingParams})
+		}{zwloggingExtraKeys, zwloggingParams})
 
 	var zmloggingParams = logging.CreateForwarderConfigTemplateParams("zuul.merger", r.cr.Spec.FluentBitLogForwarding)
+	var zmloggingExtraKeys = logging.CreateBaseLoggingExtraKeys("zuul-merger", "zuul")
 	// Change logLevel to what we actually want
 	zmloggingParams.LogLevel = string(zuulMergerLogLevel)
 	loggingData["zuul-merger-logging.yaml"], _ = utils.ParseString(
 		zuulLoggingConfig,
 		struct {
 			ExtraKeys     []logging.FluentBitLabel
 			LoggingParams logging.TemplateLoggingParams
-		}{[]logging.FluentBitLabel{}, zmloggingParams})
+		}{zmloggingExtraKeys, zmloggingParams})
 
 	return loggingData
 }

doc/deployment/logging.md

@@ -39,9 +39,14 @@ and the most straightforward to implement as an application deployer.
 If **however** no such solution is available, or if the default solution does not suit your
 needs, the SF-Operator can set up Zuul and Nodepool to forward application logs to a
 [Fluent Bit](https://fluentbit.io/) collector. The collector must be configured to expose an
-[HTTP input](https://docs.fluentbit.io/manual/pipeline/inputs/http) that your deployment
+[forward input](https://docs.fluentbit.io/manual/pipeline/inputs/forward) that your deployment
 will be able to reach.
 
+!!! warning
+    The initially supported [HTTP input](https://docs.fluentbit.io/manual/pipeline/inputs/http)
+    will be deprecated in a future version of the operator. Make sure to transition to the
+    forward input if you have been using the HTTP input.
+
 ### Enabling log forwarding on a deployment
 
 In the spec of your Software Factory deployment, edit the following:
@@ -50,10 +55,10 @@ In the spec of your Software Factory deployment, edit the following:
 [...]
 spec:
   FluentBitLogForwarding:
-    httpInputHost: "fluentbit-http-input-service"
+    forwardInputHost: "fluentbit-forward-input-service"
 ```
 
-The **httpInputHost** is the host of the HTTP input to which logs should be sent.
+The **forwardInputHost** is the host of the HTTP input to which logs should be sent.
 
 ### Pre-defined labels
 
@@ -127,8 +132,8 @@ Then add or edit the following section in the spec:
 [...]
 spec:
   FluentBitLogForwarding:
-    httpInputHost: fluent-bit-http-input
-    httpInputPort: 5140
+    httpInputHost: fluent-bit-forward-input
+    forwardInputPort: 24224
 ```
 
 

doc/reference/CHANGELOG.md

@@ -7,6 +7,10 @@ All notable changes to this project will be documented in this file.
 ### Added
 ### Removed
 ### Changed
+
+- log forwarding - Added support for the [forward input](https://docs.fluentbit.io/manual/pipeline/inputs/forward).
+  The HTTP input is deprecated and support for it will be removed at a later point.
+
 ### Fixed
 ### Security
 
@@ -23,7 +27,7 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
-- zookeeper - certificates duration bump of version v0.0.42 was partially handled due to a missing removal of the corresponding `Secrets` resources. 
+- zookeeper - certificates duration bump of version v0.0.42 was partially handled due to a missing removal of the corresponding `Secrets` resources.
 
 ### Security
 

roles/health-check/enable-log-forward/tasks/main.yaml

@@ -5,5 +5,5 @@
   vars:
     cr_spec:
       FluentBitLogForwarding:
-        httpInputHost: fluent-bit-http-input
-        httpInputPort: 5140
+        forwardInputHost: fluent-bit-forward-input
+        forwardInputPort: 24224