diff --git a/api/v1/softwarefactory_types.go b/api/v1/softwarefactory_types.go index 2f85bd35..9269c708 100644 --- a/api/v1/softwarefactory_types.go +++ b/api/v1/softwarefactory_types.go @@ -35,7 +35,7 @@ type LetsEncryptSpec struct { type FluentBitForwarderSpec struct { // The Host for the Fluent Bit HTTP Input to forward logs to. - HTTPInputHost string `json:"httpInputHost"` + HTTPInputHost string `json:"httpInputHost,omitempty"` // The (optional) port on which to forward logs to, defaults to 80. // +kubebuilder:default:=80 HTTPInputPort int32 `json:"httpInputPort,omitempty"` diff --git a/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml b/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml index 6fd6270b..58b1407e 100644 --- a/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml +++ b/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml @@ -82,8 +82,6 @@ spec: defaults to 80. format: int32 type: integer - required: - - httpInputHost type: object config-location: description: Config repository spec diff --git a/controllers/libs/logging/logging.go b/controllers/libs/logging/logging.go index 594c964d..2a0e74d4 100644 --- a/controllers/libs/logging/logging.go +++ b/controllers/libs/logging/logging.go @@ -52,6 +52,24 @@ func CreateForwarderEnvVars(name string, extraLabels []FluentBitLabel) []apiv1.E return forwarderEnvVars } +func CreateBaseLoggingExtraKeys(name string, component string) []FluentBitLabel { + baseExtraKeys := []FluentBitLabel{ + { + Key: "labels_app", + Value: "sf", + }, + { + Key: "labels_run", + Value: name, + }, + { + Key: "component", + Value: component, + }, + } + return baseExtraKeys +} + func CreateForwarderConfigTemplateParams(tag string, forwarderSpec *v1.FluentBitForwarderSpec) TemplateLoggingParams { var httpInputParams = TemplateInputParams{ InUse: false, diff --git a/controllers/nodepool.go b/controllers/nodepool.go index 36e26b44..706fc988 100644 --- a/controllers/nodepool.go +++ b/controllers/nodepool.go @@ -194,6 +194,7 @@ func (r *SFController) mkLoggingTemplate(serviceName string) (string, error) { } var loggingParams = logging.CreateForwarderConfigTemplateParams("nodepool."+serviceName, r.cr.Spec.FluentBitLogForwarding) + var loggingExtraKeys = logging.CreateBaseLoggingExtraKeys("nodepool-"+serviceName, "nodepool") // Change logLevel to what we actually want loggingParams.LogLevel = string(selectedLogLevel) @@ -202,7 +203,7 @@ func (r *SFController) mkLoggingTemplate(serviceName string) (string, error) { struct { ExtraKeys []logging.FluentBitLabel LoggingParams logging.TemplateLoggingParams - }{[]logging.FluentBitLabel{}, loggingParams}) + }{loggingExtraKeys, loggingParams}) return loggingConfig, err } diff --git a/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl b/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl index ea577d8a..ab5016c6 100644 --- a/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl +++ b/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl @@ -4,8 +4,8 @@ log_level {{ .LoggingParams.LogLevel }} [INPUT] name tail - tag mariadb - path /watch/*.log + tag mariadb.error + path /watch/error.log path_key full_path refresh_interval 5 read_from_head True @@ -24,6 +24,10 @@ {{- range .ExtraKeys }} add {{ .Key }} ${K8S_{{ .Value -}}} {{- end }} +[FILTER] + name modify + match mariadb.error + add level ERROR {{- if eq .LoggingParams.LogLevel "debug" }} [OUTPUT] name stdout @@ -38,4 +42,12 @@ format json host {{ .LoggingParams.HTTPInputConfig.Host }} port {{ .LoggingParams.HTTPInputConfig.Port }} -{{ end }} \ No newline at end of file +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} +[OUTPUT] + name forward + match * + tag {{ .LoggingParams.Tag }} + host {{ .LoggingParams.ForwardInputConfig.Host }} + port {{ .LoggingParams.ForwardInputConfig.Port }} +{{ end }} diff --git a/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl b/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl index a6263060..5ceb723c 100644 --- a/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl +++ b/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl @@ -48,7 +48,7 @@ Reserve_Data On [FILTER] Name modify - Match *dib + Match diskimage-builder Remove full_path {{- if eq .LoggingParams.LogLevel "debug" }} [OUTPUT] @@ -64,4 +64,12 @@ format json host {{ .LoggingParams.HTTPInputConfig.Host }} port {{ .LoggingParams.HTTPInputConfig.Port }} -{{ end }} \ No newline at end of file +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} +[OUTPUT] + name forward + match * + tag {{ .LoggingParams.Tag }} + host {{ .LoggingParams.ForwardInputConfig.Host }} + port {{ .LoggingParams.ForwardInputConfig.Port }} +{{ end }} diff --git a/controllers/static/nodepool/logging.yaml.tmpl b/controllers/static/nodepool/logging.yaml.tmpl index cb638532..2e916edf 100644 --- a/controllers/static/nodepool/logging.yaml.tmpl +++ b/controllers/static/nodepool/logging.yaml.tmpl @@ -7,6 +7,16 @@ formatters: fluenthttp: format: '%(asctime)s %(levelname)s %(name)s: %(message)s' {{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + fluentforward: + '()': fluent.handler.FluentRecordFormatter + format: + level: '%(levelname)s' + log: '%(name)s: %(message)s' +{{- range .ExtraKeys }} + {{ .Key }}: '{{ .Value -}}' +{{- end }} +{{ end }} handlers: console: class: logging.StreamHandler @@ -21,12 +31,25 @@ handlers: url: 'http://{{ .LoggingParams.HTTPInputConfig.Host }}:{{ .LoggingParams.HTTPInputConfig.Port }}/{{ .LoggingParams.Tag }}' env_prefix: K8S_ {{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + fluentforward: + class: fluent.handler.FluentHandler + level: {{ .LoggingParams.LogLevel }} + formatter: fluentforward +# buffer_overflow_handler: overflow_handler + host: {{ .LoggingParams.ForwardInputConfig.Host }} + port: {{ .LoggingParams.ForwardInputConfig.Port }} + tag: {{ .LoggingParams.Tag }} +{{ end }} loggers: nodepool: handlers: - console {{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + - fluentforward {{ end }} level: {{.LoggingParams.LogLevel}} propagate: 0 @@ -35,5 +58,8 @@ root: - console {{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + - fluentforward {{ end }} level: {{.LoggingParams.LogLevel}} diff --git a/controllers/static/zookeeper/fluent-bit.conf.tmpl b/controllers/static/zookeeper/fluent-bit.conf.tmpl index ba67a2b7..6ef292c7 100644 --- a/controllers/static/zookeeper/fluent-bit.conf.tmpl +++ b/controllers/static/zookeeper/fluent-bit.conf.tmpl @@ -38,4 +38,12 @@ format json host {{ .LoggingParams.HTTPInputConfig.Host }} port {{ .LoggingParams.HTTPInputConfig.Port }} -{{ end }} \ No newline at end of file +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} +[OUTPUT] + name forward + match * + tag {{ .LoggingParams.Tag }} + host {{ .LoggingParams.ForwardInputConfig.Host }} + port {{ .LoggingParams.ForwardInputConfig.Port }} +{{ end }} diff --git a/controllers/static/zuul/logging.yaml.tmpl b/controllers/static/zuul/logging.yaml.tmpl index fdbcd17d..9fcebd36 100644 --- a/controllers/static/zuul/logging.yaml.tmpl +++ b/controllers/static/zuul/logging.yaml.tmpl @@ -7,6 +7,16 @@ formatters: fluenthttp: format: '%(asctime)s %(levelname)s %(name)s: %(message)s' {{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + fluentforward: + '()': fluent.handler.FluentRecordFormatter + format: + level: '%(levelname)s' + log: '%(name)s: %(message)s' +{{- range .ExtraKeys }} + {{ .Key }}: '{{ .Value -}}' +{{- end }} +{{ end }} handlers: console: class: logging.StreamHandler @@ -21,12 +31,25 @@ handlers: url: 'http://{{ .LoggingParams.HTTPInputConfig.Host }}:{{ .LoggingParams.HTTPInputConfig.Port }}/{{ .LoggingParams.Tag }}' env_prefix: K8S_ {{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + fluentforward: + class: fluent.handler.FluentHandler + level: {{ .LoggingParams.LogLevel }} + formatter: fluentforward +# buffer_overflow_handler: overflow_handler + host: {{ .LoggingParams.ForwardInputConfig.Host }} + port: {{ .LoggingParams.ForwardInputConfig.Port }} + tag: {{ .LoggingParams.Tag }} +{{ end }} loggers: zuul.GerritConnection.ssh: handlers: - console {{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + - fluentforward {{ end }} level: INFO propagate: 0 @@ -35,6 +58,9 @@ loggers: - console {{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + - fluentforward {{ end }} level: {{ .LoggingParams.LogLevel }} propagate: 0 @@ -43,6 +69,9 @@ loggers: - console {{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + - fluentforward {{ end }} level: {{ .LoggingParams.LogLevel }} propagate: 0 @@ -51,6 +80,9 @@ loggers: - console {{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + - fluentforward {{ end }} level: {{ .LoggingParams.LogLevel }} propagate: 0 @@ -59,6 +91,8 @@ root: - console {{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp +{{ end }} +{{- if .LoggingParams.ForwardInputConfig.InUse }} + - fluentforward {{ end }} level: {{ .LoggingParams.LogLevel }} - diff --git a/controllers/zuul.go b/controllers/zuul.go index 816873b3..0053620a 100644 --- a/controllers/zuul.go +++ b/controllers/zuul.go @@ -385,6 +385,7 @@ func (r *SFController) computeLoggingConfig() map[string]string { } var zeloggingParams = logging.CreateForwarderConfigTemplateParams("zuul.executor", r.cr.Spec.FluentBitLogForwarding) + var zeloggingExtraKeys = logging.CreateBaseLoggingExtraKeys("zuul-executor", "zuul") // Change logLevel to what we actually want zeloggingParams.LogLevel = string(zuulExecutorLogLevel) loggingData["zuul-executor-logging.yaml"], _ = utils.ParseString( @@ -392,9 +393,10 @@ func (r *SFController) computeLoggingConfig() map[string]string { struct { ExtraKeys []logging.FluentBitLabel LoggingParams logging.TemplateLoggingParams - }{[]logging.FluentBitLabel{}, zeloggingParams}) + }{zeloggingExtraKeys, zeloggingParams}) var zsloggingParams = logging.CreateForwarderConfigTemplateParams("zuul.scheduler", r.cr.Spec.FluentBitLogForwarding) + var zsloggingExtraKeys = logging.CreateBaseLoggingExtraKeys("zuul-scheduler", "zuul") // Change logLevel to what we actually want zsloggingParams.LogLevel = string(zuulSchedulerLogLevel) loggingData["zuul-scheduler-logging.yaml"], _ = utils.ParseString( @@ -402,9 +404,10 @@ func (r *SFController) computeLoggingConfig() map[string]string { struct { ExtraKeys []logging.FluentBitLabel LoggingParams logging.TemplateLoggingParams - }{[]logging.FluentBitLabel{}, zsloggingParams}) + }{zsloggingExtraKeys, zsloggingParams}) var zwloggingParams = logging.CreateForwarderConfigTemplateParams("zuul.web", r.cr.Spec.FluentBitLogForwarding) + var zwloggingExtraKeys = logging.CreateBaseLoggingExtraKeys("zuul-web", "zuul") // Change logLevel to what we actually want zwloggingParams.LogLevel = string(zuulWebLogLevel) loggingData["zuul-web-logging.yaml"], _ = utils.ParseString( @@ -412,9 +415,10 @@ func (r *SFController) computeLoggingConfig() map[string]string { struct { ExtraKeys []logging.FluentBitLabel LoggingParams logging.TemplateLoggingParams - }{[]logging.FluentBitLabel{}, zwloggingParams}) + }{zwloggingExtraKeys, zwloggingParams}) var zmloggingParams = logging.CreateForwarderConfigTemplateParams("zuul.merger", r.cr.Spec.FluentBitLogForwarding) + var zmloggingExtraKeys = logging.CreateBaseLoggingExtraKeys("zuul-merger", "zuul") // Change logLevel to what we actually want zmloggingParams.LogLevel = string(zuulMergerLogLevel) loggingData["zuul-merger-logging.yaml"], _ = utils.ParseString( @@ -422,7 +426,7 @@ func (r *SFController) computeLoggingConfig() map[string]string { struct { ExtraKeys []logging.FluentBitLabel LoggingParams logging.TemplateLoggingParams - }{[]logging.FluentBitLabel{}, zmloggingParams}) + }{zmloggingExtraKeys, zmloggingParams}) return loggingData } diff --git a/doc/deployment/logging.md b/doc/deployment/logging.md index c1061983..fd64cdf2 100644 --- a/doc/deployment/logging.md +++ b/doc/deployment/logging.md @@ -39,9 +39,14 @@ and the most straightforward to implement as an application deployer. If **however** no such solution is available, or if the default solution does not suit your needs, the SF-Operator can set up Zuul and Nodepool to forward application logs to a [Fluent Bit](https://fluentbit.io/) collector. The collector must be configured to expose an -[HTTP input](https://docs.fluentbit.io/manual/pipeline/inputs/http) that your deployment +[forward input](https://docs.fluentbit.io/manual/pipeline/inputs/forward) that your deployment will be able to reach. +!!! warning + The initially supported [HTTP input](https://docs.fluentbit.io/manual/pipeline/inputs/http) + will be deprecated in a future version of the operator. Make sure to transition to the + forward input if you have been using the HTTP input. + ### Enabling log forwarding on a deployment In the spec of your Software Factory deployment, edit the following: @@ -50,10 +55,10 @@ In the spec of your Software Factory deployment, edit the following: [...] spec: FluentBitLogForwarding: - httpInputHost: "fluentbit-http-input-service" + forwardInputHost: "fluentbit-forward-input-service" ``` -The **httpInputHost** is the host of the HTTP input to which logs should be sent. +The **forwardInputHost** is the host of the HTTP input to which logs should be sent. ### Pre-defined labels @@ -127,8 +132,8 @@ Then add or edit the following section in the spec: [...] spec: FluentBitLogForwarding: - httpInputHost: fluent-bit-http-input - httpInputPort: 5140 + httpInputHost: fluent-bit-forward-input + forwardInputPort: 24224 ``` diff --git a/doc/reference/CHANGELOG.md b/doc/reference/CHANGELOG.md index 2e0bc964..461929ed 100644 --- a/doc/reference/CHANGELOG.md +++ b/doc/reference/CHANGELOG.md @@ -7,6 +7,10 @@ All notable changes to this project will be documented in this file. ### Added ### Removed ### Changed + +- log forwarding - Added support for the [forward input](https://docs.fluentbit.io/manual/pipeline/inputs/forward). + The HTTP input is deprecated and support for it will be removed at a later point. + ### Fixed ### Security @@ -23,7 +27,7 @@ All notable changes to this project will be documented in this file. ### Fixed -- zookeeper - certificates duration bump of version v0.0.42 was partially handled due to a missing removal of the corresponding `Secrets` resources. +- zookeeper - certificates duration bump of version v0.0.42 was partially handled due to a missing removal of the corresponding `Secrets` resources. ### Security diff --git a/roles/health-check/enable-log-forward/tasks/main.yaml b/roles/health-check/enable-log-forward/tasks/main.yaml index 402810c2..1d503871 100644 --- a/roles/health-check/enable-log-forward/tasks/main.yaml +++ b/roles/health-check/enable-log-forward/tasks/main.yaml @@ -5,5 +5,5 @@ vars: cr_spec: FluentBitLogForwarding: - httpInputHost: fluent-bit-http-input - httpInputPort: 5140 \ No newline at end of file + forwardInputHost: fluent-bit-forward-input + forwardInputPort: 24224 \ No newline at end of file diff --git a/roles/health-check/test-log-forwarding/tasks/main.yaml b/roles/health-check/test-log-forwarding/tasks/main.yaml index 70bf5ed1..0b4a2577 100644 --- a/roles/health-check/test-log-forwarding/tasks/main.yaml +++ b/roles/health-check/test-log-forwarding/tasks/main.yaml @@ -22,6 +22,7 @@ - zuul - nodepool - mariadb + - zookeeper - name: Ensure DIB logs are collected by loki ansible.builtin.shell: "~/bin/logcli query '{labels_run=\"dib\"}' --quiet --limit 5" diff --git a/tools/loki/test-fluentbit.yaml b/tools/loki/test-fluentbit.yaml index e6426249..9f31979a 100644 --- a/tools/loki/test-fluentbit.yaml +++ b/tools/loki/test-fluentbit.yaml @@ -8,8 +8,8 @@ data: log_level debug [INPUT] - name http - port 5140 + name forward + port 24224 [OUTPUT] name stdout @@ -40,7 +40,7 @@ metadata: spec: containers: # append "-debug" to image name to run in debug mode, in case you run into CrashLoopBackOff errors. - - image: cr.fluentbit.io/fluent/fluent-bit:2.1.10 + - image: cr.fluentbit.io/fluent/fluent-bit:latest name: test-fluentbit resources: {} securityContext: @@ -58,8 +58,8 @@ spec: ports: - name: fb-http-server containerPort: 2020 - - name: fb-http-input - containerPort: 5140 + - name: fb-fw-input + containerPort: 24224 dnsPolicy: ClusterFirst restartPolicy: Always volumes: @@ -70,16 +70,16 @@ spec: apiVersion: v1 kind: Service metadata: - name: fluent-bit-http-input + name: fluent-bit-forward-input labels: run: fluentbit app: demo-logging spec: ports: - - name: fb-http-input - port: 5140 + - name: fb-fw-input + port: 24224 protocol: TCP - targetPort: 5140 + targetPort: 24224 selector: run: fluentbit app: demo-logging