From 45c35167b131dec171fc51d893c1fe4f2859392d Mon Sep 17 00:00:00 2001 From: Matthieu Huin Date: Wed, 9 Oct 2024 14:50:54 +0200 Subject: [PATCH] logging: remove HTTP Input The feature was deprecated as of v0.0.44. It'll be formally removed in the following release. Change-Id: I8e5bd2f1a8a3dea42c19d8f9ef123768205dedac --- api/v1/softwarefactory_types.go | 5 ---- cli/cmd/initialize.go | 4 +-- ...efactory-project.io_softwarefactories.yaml | 10 ------- controllers/libs/base/static/images.yaml | 24 ++++++++--------- controllers/libs/logging/logging.go | 20 +------------- .../mariadb/fluentbit/fluent-bit.conf.tmpl | 9 ------- .../nodepool/fluentbit/fluent-bit.conf.tmpl | 9 ------- controllers/static/nodepool/logging.yaml.tmpl | 18 ------------- .../static/zookeeper/fluent-bit.conf.tmpl | 9 ------- controllers/static/zuul/logging.yaml.tmpl | 27 ------------------- controllers/zuul.go | 15 ----------- doc/deployment/logging.md | 7 +---- doc/reference/CHANGELOG.md | 3 +++ doc/reference/api/index.md | 2 -- 14 files changed, 19 insertions(+), 143 deletions(-) diff --git a/api/v1/softwarefactory_types.go b/api/v1/softwarefactory_types.go index 9269c70..1121523 100644 --- a/api/v1/softwarefactory_types.go +++ b/api/v1/softwarefactory_types.go @@ -34,11 +34,6 @@ type LetsEncryptSpec struct { } type FluentBitForwarderSpec struct { - // The Host for the Fluent Bit HTTP Input to forward logs to. - HTTPInputHost string `json:"httpInputHost,omitempty"` - // The (optional) port on which to forward logs to, defaults to 80. - // +kubebuilder:default:=80 - HTTPInputPort int32 `json:"httpInputPort,omitempty"` // +kubebuilder:default:=false // +optional // Run fluent bit sidecars in debug mode. This will output forwarded payloads and additional info in the sidecar's logs. Defaults to false. diff --git a/cli/cmd/initialize.go b/cli/cmd/initialize.go index 316b5a4..073656a 100644 --- a/cli/cmd/initialize.go +++ b/cli/cmd/initialize.go @@ -110,8 +110,8 @@ func initializeSFManifest(withAuth bool, withBuilder bool, full bool, connection if full { fbSpec := sfv1.FluentBitForwarderSpec{ - HTTPInputHost: "fluentbit", - HTTPInputPort: 5140, + ForwardInputHost: "fluentbit", + ForwardInputPort: 24224, } manifest.Spec.FluentBitLogForwarding = &fbSpec diff --git a/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml b/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml index 58b1407..5d825af 100644 --- a/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml +++ b/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml @@ -72,16 +72,6 @@ spec: to 24224. format: int32 type: integer - httpInputHost: - description: The Host for the Fluent Bit HTTP Input to forward - logs to. - type: string - httpInputPort: - default: 80 - description: The (optional) port on which to forward logs to, - defaults to 80. - format: int32 - type: integer type: object config-location: description: Config repository spec diff --git a/controllers/libs/base/static/images.yaml b/controllers/libs/base/static/images.yaml index b1c0548..f71fe7d 100644 --- a/controllers/libs/base/static/images.yaml +++ b/controllers/libs/base/static/images.yaml @@ -4,28 +4,28 @@ images: - name: zuul-scheduler container: quay.io/software-factory/zuul-scheduler - version: 11.1.0-2 - source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-scheduler.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de + version: 11.1.0-3 + source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-scheduler.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5 - name: zuul-executor container: quay.io/software-factory/zuul-executor - version: 11.1.0-2 - source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-executor.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de + version: 11.1.0-3 + source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-executor.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5 - name: zuul-merger container: quay.io/software-factory/zuul-merger - version: 11.1.0-2 - source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-merger.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de + version: 11.1.0-3 + source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-merger.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5 - name: zuul-web container: quay.io/software-factory/zuul-web - version: 11.1.0-2 - source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-web.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de + version: 11.1.0-3 + source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-web.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5 - name: nodepool-launcher container: quay.io/software-factory/nodepool-launcher - version: 10.0.0-5 - source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/nodepool-launcher.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de + version: 10.0.0-6 + source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/nodepool-launcher.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5 - name: nodepool-builder container: quay.io/software-factory/nodepool-builder - version: 10.0.0-5 - source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/nodepool-builder.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de + version: 10.0.0-6 + source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/nodepool-builder.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5 - name: zookeeper container: quay.io/software-factory/zookeeper version: 3.8.4-2 diff --git a/controllers/libs/logging/logging.go b/controllers/libs/logging/logging.go index 092b276..6344b55 100644 --- a/controllers/libs/logging/logging.go +++ b/controllers/libs/logging/logging.go @@ -32,15 +32,9 @@ type TemplateInputParams struct { type TemplateLoggingParams struct { Tag string LogLevel string - HTTPInputConfig TemplateInputParams ForwardInputConfig TemplateInputParams } -var ( - //go:embed static/sfExtras.py - SFExtrasPythonModule string -) - func CreateForwarderEnvVars(name string, extraLabels []FluentBitLabel) []apiv1.EnvVar { forwarderEnvVars := []apiv1.EnvVar{ base.MkEnvVarFromFieldRef("K8S_NODENAME", "spec.nodeName"), @@ -84,11 +78,6 @@ func CreateBaseLoggingExtraKeys(name string, component string, container string, } func CreateForwarderConfigTemplateParams(tag string, forwarderSpec *v1.FluentBitForwarderSpec) TemplateLoggingParams { - var httpInputParams = TemplateInputParams{ - InUse: false, - Host: "", - Port: "", - } var forwardInputParams = TemplateInputParams{ InUse: false, Host: "", @@ -97,15 +86,9 @@ func CreateForwarderConfigTemplateParams(tag string, forwarderSpec *v1.FluentBit var loggingParams = TemplateLoggingParams{ Tag: tag, LogLevel: "info", - HTTPInputConfig: httpInputParams, ForwardInputConfig: forwardInputParams, } if forwarderSpec != nil { - if forwarderSpec.HTTPInputHost != "" { - loggingParams.HTTPInputConfig.InUse = true - loggingParams.HTTPInputConfig.Host = forwarderSpec.HTTPInputHost - loggingParams.HTTPInputConfig.Port = strconv.Itoa(int(forwarderSpec.HTTPInputPort)) - } if forwarderSpec.ForwardInputHost != "" { loggingParams.ForwardInputConfig.InUse = true loggingParams.ForwardInputConfig.Host = forwarderSpec.ForwardInputHost @@ -121,8 +104,7 @@ func CreateForwarderConfigTemplateParams(tag string, forwarderSpec *v1.FluentBit func SetupLogForwarding(serviceName string, forwarderSpec *v1.FluentBitForwarderSpec, extraLabels []FluentBitLabel, annotations map[string]string) []apiv1.EnvVar { if forwarderSpec != nil { - annotations["log-forwarding"] = forwarderSpec.HTTPInputHost + ":" + strconv.Itoa(int(forwarderSpec.HTTPInputPort)) - annotations["log-forwarding"] += forwarderSpec.ForwardInputHost + ":" + strconv.Itoa(int(forwarderSpec.ForwardInputPort)) + annotations["log-forwarding"] = forwarderSpec.ForwardInputHost + ":" + strconv.Itoa(int(forwarderSpec.ForwardInputPort)) return CreateForwarderEnvVars(serviceName, extraLabels) } else { annotations["log-forwarding"] = "disabled" diff --git a/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl b/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl index 004686f..1a7ab75 100644 --- a/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl +++ b/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl @@ -40,15 +40,6 @@ match * format json_lines {{ end }} -{{- if .LoggingParams.HTTPInputConfig.InUse }} -[OUTPUT] - name http - match * - uri /{{ .LoggingParams.Tag }} - format json - host {{ .LoggingParams.HTTPInputConfig.Host }} - port {{ .LoggingParams.HTTPInputConfig.Port }} -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} [OUTPUT] name forward diff --git a/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl b/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl index 52d3742..07e3b1c 100644 --- a/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl +++ b/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl @@ -62,15 +62,6 @@ match * format json_lines {{ end }} -{{- if .LoggingParams.HTTPInputConfig.InUse }} -[OUTPUT] - name http - match * - uri /{{ .LoggingParams.Tag }} - format json - host {{ .LoggingParams.HTTPInputConfig.Host }} - port {{ .LoggingParams.HTTPInputConfig.Port }} -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} [OUTPUT] name forward diff --git a/controllers/static/nodepool/logging.yaml.tmpl b/controllers/static/nodepool/logging.yaml.tmpl index 2e916ed..776f240 100644 --- a/controllers/static/nodepool/logging.yaml.tmpl +++ b/controllers/static/nodepool/logging.yaml.tmpl @@ -3,10 +3,6 @@ formatters: console: class: 'nodepool.logconfig.MultiLineFormatter' format: "%(asctime)s %(levelname)s %(name)s: %(message)s" -{{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp: - format: '%(asctime)s %(levelname)s %(name)s: %(message)s' -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} fluentforward: '()': fluent.handler.FluentRecordFormatter @@ -23,14 +19,6 @@ handlers: formatter: console level: {{ .LoggingParams.LogLevel}} stream: ext://sys.stdout -{{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp: - class: sfExtras.SimpleFluentBitHTTPInputHandler - level: {{ .LoggingParams.LogLevel }} - formatter: fluenthttp - url: 'http://{{ .LoggingParams.HTTPInputConfig.Host }}:{{ .LoggingParams.HTTPInputConfig.Port }}/{{ .LoggingParams.Tag }}' - env_prefix: K8S_ -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} fluentforward: class: fluent.handler.FluentHandler @@ -45,9 +33,6 @@ loggers: nodepool: handlers: - console -{{- if .LoggingParams.HTTPInputConfig.InUse }} - - fluenthttp -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} - fluentforward {{ end }} @@ -56,9 +41,6 @@ loggers: root: handlers: - console -{{- if .LoggingParams.HTTPInputConfig.InUse }} - - fluenthttp -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} - fluentforward {{ end }} diff --git a/controllers/static/zookeeper/fluent-bit.conf.tmpl b/controllers/static/zookeeper/fluent-bit.conf.tmpl index bcb1f45..61e7df6 100644 --- a/controllers/static/zookeeper/fluent-bit.conf.tmpl +++ b/controllers/static/zookeeper/fluent-bit.conf.tmpl @@ -36,15 +36,6 @@ match * format json_lines {{ end }} -{{- if .LoggingParams.HTTPInputConfig.InUse }} -[OUTPUT] - name http - match * - uri /{{ .LoggingParams.Tag }} - format json - host {{ .LoggingParams.HTTPInputConfig.Host }} - port {{ .LoggingParams.HTTPInputConfig.Port }} -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} [OUTPUT] name forward diff --git a/controllers/static/zuul/logging.yaml.tmpl b/controllers/static/zuul/logging.yaml.tmpl index 9fcebd3..7434c82 100644 --- a/controllers/static/zuul/logging.yaml.tmpl +++ b/controllers/static/zuul/logging.yaml.tmpl @@ -3,10 +3,6 @@ formatters: console: class: 'zuul.lib.logutil.MultiLineFormatter' format: '%(asctime)s %(levelname)s %(name)s: %(message)s' -{{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp: - format: '%(asctime)s %(levelname)s %(name)s: %(message)s' -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} fluentforward: '()': fluent.handler.FluentRecordFormatter @@ -23,14 +19,6 @@ handlers: formatter: console level: {{ .LoggingParams.LogLevel }} stream: ext://sys.stdout -{{- if .LoggingParams.HTTPInputConfig.InUse }} - fluenthttp: - class: sfExtras.SimpleFluentBitHTTPInputHandler - level: {{ .LoggingParams.LogLevel }} - formatter: fluenthttp - url: 'http://{{ .LoggingParams.HTTPInputConfig.Host }}:{{ .LoggingParams.HTTPInputConfig.Port }}/{{ .LoggingParams.Tag }}' - env_prefix: K8S_ -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} fluentforward: class: fluent.handler.FluentHandler @@ -45,9 +33,6 @@ loggers: zuul.GerritConnection.ssh: handlers: - console -{{- if .LoggingParams.HTTPInputConfig.InUse }} - - fluenthttp -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} - fluentforward {{ end }} @@ -56,9 +41,6 @@ loggers: zuul.GerritConnection.io: handlers: - console -{{- if .LoggingParams.HTTPInputConfig.InUse }} - - fluenthttp -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} - fluentforward {{ end }} @@ -67,9 +49,6 @@ loggers: connection: handlers: - console -{{- if .LoggingParams.HTTPInputConfig.InUse }} - - fluenthttp -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} - fluentforward {{ end }} @@ -78,9 +57,6 @@ loggers: zuul: handlers: - console -{{- if .LoggingParams.HTTPInputConfig.InUse }} - - fluenthttp -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} - fluentforward {{ end }} @@ -89,9 +65,6 @@ loggers: root: handlers: - console -{{- if .LoggingParams.HTTPInputConfig.InUse }} - - fluenthttp -{{ end }} {{- if .LoggingParams.ForwardInputConfig.InUse }} - fluentforward {{ end }} diff --git a/controllers/zuul.go b/controllers/zuul.go index 34266f8..597a346 100644 --- a/controllers/zuul.go +++ b/controllers/zuul.go @@ -190,12 +190,6 @@ func (r *SFController) mkZuulContainer(service string, corporateCMExists bool) a Name: "zuul-ca", MountPath: "/etc/pki/ca-trust/extracted", }, - { - Name: "extra-python-module", - MountPath: "/usr/local/lib/python3.11/site-packages/sfExtras.py", - SubPath: "sfExtras.py", - ReadOnly: true, - }, } envs := []apiv1.EnvVar{ base.MkEnvVar("REQUESTS_CA_BUNDLE", "/etc/ssl/certs/ca-bundle.crt"), @@ -301,11 +295,6 @@ func mkZuulVolumes(service string, r *SFController, corporateCMExists bool) []ap // Install the logging settings config map resource r.EnsureConfigMap("zuul-logging", r.computeLoggingConfig()) - // Install extra python module - r.EnsureConfigMap("extra-python-module", map[string]string{ - "sfExtras.py": logging.SFExtrasPythonModule, - }) - volumes := []apiv1.Volume{ base.MkVolumeSecret("ca-cert"), base.MkVolumeSecret("zuul-config"), @@ -470,7 +459,6 @@ func (r *SFController) EnsureZuulScheduler(cfg *ini.File) bool { "zuul-extra": utils.Checksum([]byte(sshConfig)), "zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")), "corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists), - "extras-python-module": utils.Checksum([]byte(logging.SFExtrasPythonModule)), } if r.isConfigRepoSet() { @@ -598,7 +586,6 @@ func (r *SFController) EnsureZuulExecutor(cfg *ini.File) bool { "zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-executor"))), "zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")), "corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists), - "extras-python-module": utils.Checksum([]byte(logging.SFExtrasPythonModule)), } // TODO Add the zk-port-forward-kube-config secret resource version in the annotation if enabled @@ -676,7 +663,6 @@ func (r *SFController) EnsureZuulMerger(cfg *ini.File) bool { "zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")), "zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-merger"))), "corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists), - "extras-python-module": utils.Checksum([]byte(logging.SFExtrasPythonModule)), } zm := r.mkHeadlessSatefulSet(service, "", r.getStorageConfOrDefault(r.cr.Spec.Zuul.Merger.Storage), apiv1.ReadWriteOnce, r.cr.Spec.ExtraLabels) @@ -742,7 +728,6 @@ func (r *SFController) EnsureZuulWeb(cfg *ini.File) bool { "zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-web"))), "zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")), "corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists), - "extras-python-module": utils.Checksum([]byte(logging.SFExtrasPythonModule)), } zw := base.MkDeployment("zuul-web", r.ns, "", r.cr.Spec.ExtraLabels) diff --git a/doc/deployment/logging.md b/doc/deployment/logging.md index 241af1f..20a2655 100644 --- a/doc/deployment/logging.md +++ b/doc/deployment/logging.md @@ -42,11 +42,6 @@ needs, the SF-Operator can set up Zuul and Nodepool to forward application logs [forward input](https://docs.fluentbit.io/manual/pipeline/inputs/forward) that your deployment will be able to reach. -!!! warning - The initially supported [HTTP input](https://docs.fluentbit.io/manual/pipeline/inputs/http) - will be deprecated in a future version of the operator. Make sure to transition to the - forward input if you have been using the HTTP input. - ### Enabling log forwarding on a deployment In the spec of your Software Factory deployment, edit the following: @@ -132,7 +127,7 @@ Then add or edit the following section in the spec: [...] spec: FluentBitLogForwarding: - httpInputHost: fluent-bit-forward-input + forwardInputHost: fluent-bit-forward-input forwardInputPort: 24224 ``` diff --git a/doc/reference/CHANGELOG.md b/doc/reference/CHANGELOG.md index 5a7a9c0..c999519 100644 --- a/doc/reference/CHANGELOG.md +++ b/doc/reference/CHANGELOG.md @@ -6,6 +6,9 @@ All notable changes to this project will be documented in this file. ### Added ### Removed + +- log forwarding - remove support for the HTTP input. + ### Changed ### Fixed ### Security diff --git a/doc/reference/api/index.md b/doc/reference/api/index.md index 74f601c..26ad561 100644 --- a/doc/reference/api/index.md +++ b/doc/reference/api/index.md @@ -82,8 +82,6 @@ _Appears in:_ | Field | Description | Default Value | | --- | --- | --- | -| `httpInputHost` _string_ | The Host for the Fluent Bit HTTP Input to forward logs to. | -| -| `httpInputPort` _integer_ | The (optional) port on which to forward logs to, defaults to 80. | {80}| | `debug` _boolean_ | Run fluent bit sidecars in debug mode. This will output forwarded payloads and additional info in the sidecar's logs. Defaults to false. | {false}| | `forwardInputHost` _string_ | The Host for the Fluent Bit Forward Input to forward logs to. | -| | `forwardInputPort` _integer_ | The (optional) port of the forward input, defaults to 24224. | {24224}|