Merge "logging: remove HTTP Input"

softwarefactory-project · Oct 15, 2024 · ca94665 · ca94665
2 parents 483753d + 45c3516
commit ca94665
Show file tree

Hide file tree

Showing 14 changed files with 19 additions and 143 deletions.
diff --git a/api/v1/softwarefactory_types.go b/api/v1/softwarefactory_types.go
@@ -34,11 +34,6 @@ type LetsEncryptSpec struct {
 }
 
 type FluentBitForwarderSpec struct {
-	// The Host for the Fluent Bit HTTP Input to forward logs to.
-	HTTPInputHost string `json:"httpInputHost,omitempty"`
-	// The (optional) port on which to forward logs to, defaults to 80.
-	// +kubebuilder:default:=80
-	HTTPInputPort int32 `json:"httpInputPort,omitempty"`
 	// +kubebuilder:default:=false
 	// +optional
 	// Run fluent bit sidecars in debug mode. This will output forwarded payloads and additional info in the sidecar's logs. Defaults to false.

diff --git a/cli/cmd/initialize.go b/cli/cmd/initialize.go
@@ -110,8 +110,8 @@ func initializeSFManifest(withAuth bool, withBuilder bool, full bool, connection
 
 	if full {
 		fbSpec := sfv1.FluentBitForwarderSpec{
-			HTTPInputHost: "fluentbit",
-			HTTPInputPort: 5140,
+			ForwardInputHost: "fluentbit",
+			ForwardInputPort: 24224,
 		}
 		manifest.Spec.FluentBitLogForwarding = &fbSpec
 

diff --git a/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml b/config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml
@@ -72,16 +72,6 @@ spec:
                       to 24224.
                     format: int32
                     type: integer
-                  httpInputHost:
-                    description: The Host for the Fluent Bit HTTP Input to forward
-                      logs to.
-                    type: string
-                  httpInputPort:
-                    default: 80
-                    description: The (optional) port on which to forward logs to,
-                      defaults to 80.
-                    format: int32
-                    type: integer
                 type: object
               config-location:
                 description: Config repository spec

diff --git a/controllers/libs/base/static/images.yaml b/controllers/libs/base/static/images.yaml
@@ -4,28 +4,28 @@
 images:
     - name: zuul-scheduler
       container: quay.io/software-factory/zuul-scheduler
-      version: 11.1.0-2
-      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-scheduler.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de
+      version: 11.1.0-3
+      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-scheduler.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5
     - name: zuul-executor
       container: quay.io/software-factory/zuul-executor
-      version: 11.1.0-2
-      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-executor.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de
+      version: 11.1.0-3
+      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-executor.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5
     - name: zuul-merger
       container: quay.io/software-factory/zuul-merger
-      version: 11.1.0-2
-      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-merger.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de
+      version: 11.1.0-3
+      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-merger.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5
     - name: zuul-web
       container: quay.io/software-factory/zuul-web
-      version: 11.1.0-2
-      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-web.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de
+      version: 11.1.0-3
+      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/zuul-web.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5
     - name: nodepool-launcher
       container: quay.io/software-factory/nodepool-launcher
-      version: 10.0.0-5
-      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/nodepool-launcher.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de
+      version: 10.0.0-6
+      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/nodepool-launcher.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5
     - name: nodepool-builder
       container: quay.io/software-factory/nodepool-builder
-      version: 10.0.0-5
-      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/nodepool-builder.container?id=11f7a8ed25f18e880d05d795f5d8a52fd75dd4de
+      version: 10.0.0-6
+      source: https://softwarefactory-project.io/cgit/containers/tree/images-sf/master/containers/rendered/nodepool-builder.container?id=d3b760e620710d0cef12e840b76bff5327eb75d5
     - name: zookeeper
       container: quay.io/software-factory/zookeeper
       version: 3.8.4-2

diff --git a/controllers/libs/logging/logging.go b/controllers/libs/logging/logging.go
@@ -32,15 +32,9 @@ type TemplateInputParams struct {
 type TemplateLoggingParams struct {
 	Tag                string
 	LogLevel           string
-	HTTPInputConfig    TemplateInputParams
 	ForwardInputConfig TemplateInputParams
 }
 
-var (
-	//go:embed static/sfExtras.py
-	SFExtrasPythonModule string
-)
-
 func CreateForwarderEnvVars(name string, extraLabels []FluentBitLabel) []apiv1.EnvVar {
 	forwarderEnvVars := []apiv1.EnvVar{
 		base.MkEnvVarFromFieldRef("K8S_NODENAME", "spec.nodeName"),
@@ -84,11 +78,6 @@ func CreateBaseLoggingExtraKeys(name string, component string, container string,
 }
 
 func CreateForwarderConfigTemplateParams(tag string, forwarderSpec *v1.FluentBitForwarderSpec) TemplateLoggingParams {
-	var httpInputParams = TemplateInputParams{
-		InUse: false,
-		Host:  "",
-		Port:  "",
-	}
 	var forwardInputParams = TemplateInputParams{
 		InUse: false,
 		Host:  "",
@@ -97,15 +86,9 @@ func CreateForwarderConfigTemplateParams(tag string, forwarderSpec *v1.FluentBit
 	var loggingParams = TemplateLoggingParams{
 		Tag:                tag,
 		LogLevel:           "info",
-		HTTPInputConfig:    httpInputParams,
 		ForwardInputConfig: forwardInputParams,
 	}
 	if forwarderSpec != nil {
-		if forwarderSpec.HTTPInputHost != "" {
-			loggingParams.HTTPInputConfig.InUse = true
-			loggingParams.HTTPInputConfig.Host = forwarderSpec.HTTPInputHost
-			loggingParams.HTTPInputConfig.Port = strconv.Itoa(int(forwarderSpec.HTTPInputPort))
-		}
 		if forwarderSpec.ForwardInputHost != "" {
 			loggingParams.ForwardInputConfig.InUse = true
 			loggingParams.ForwardInputConfig.Host = forwarderSpec.ForwardInputHost
@@ -121,8 +104,7 @@ func CreateForwarderConfigTemplateParams(tag string, forwarderSpec *v1.FluentBit
 
 func SetupLogForwarding(serviceName string, forwarderSpec *v1.FluentBitForwarderSpec, extraLabels []FluentBitLabel, annotations map[string]string) []apiv1.EnvVar {
 	if forwarderSpec != nil {
-		annotations["log-forwarding"] = forwarderSpec.HTTPInputHost + ":" + strconv.Itoa(int(forwarderSpec.HTTPInputPort))
-		annotations["log-forwarding"] += forwarderSpec.ForwardInputHost + ":" + strconv.Itoa(int(forwarderSpec.ForwardInputPort))
+		annotations["log-forwarding"] = forwarderSpec.ForwardInputHost + ":" + strconv.Itoa(int(forwarderSpec.ForwardInputPort))
 		return CreateForwarderEnvVars(serviceName, extraLabels)
 	} else {
 		annotations["log-forwarding"] = "disabled"

diff --git a/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl b/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl
@@ -40,15 +40,6 @@
     match   *
     format  json_lines
 {{ end }}
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-[OUTPUT]
-    name    http
-    match   *
-    uri     /{{ .LoggingParams.Tag }}
-    format  json
-    host    {{ .LoggingParams.HTTPInputConfig.Host }}
-    port    {{ .LoggingParams.HTTPInputConfig.Port }}
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
 [OUTPUT]
     name                        forward

diff --git a/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl b/controllers/static/nodepool/fluentbit/fluent-bit.conf.tmpl
@@ -62,15 +62,6 @@
     match   *
     format  json_lines
 {{ end }}
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-[OUTPUT]
-    name    http
-    match   *
-    uri     /{{ .LoggingParams.Tag }}
-    format  json
-    host    {{ .LoggingParams.HTTPInputConfig.Host }}
-    port    {{ .LoggingParams.HTTPInputConfig.Port }}
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
 [OUTPUT]
     name                        forward

diff --git a/controllers/static/nodepool/logging.yaml.tmpl b/controllers/static/nodepool/logging.yaml.tmpl
@@ -3,10 +3,6 @@ formatters:
   console:
     class: 'nodepool.logconfig.MultiLineFormatter'
     format: "%(asctime)s %(levelname)s %(name)s: %(message)s"
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-  fluenthttp:
-    format: '%(asctime)s %(levelname)s %(name)s: %(message)s'
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
   fluentforward:
     '()': fluent.handler.FluentRecordFormatter
@@ -23,14 +19,6 @@ handlers:
     formatter: console
     level: {{ .LoggingParams.LogLevel}}
     stream: ext://sys.stdout
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-  fluenthttp:
-    class: sfExtras.SimpleFluentBitHTTPInputHandler
-    level: {{ .LoggingParams.LogLevel }}
-    formatter: fluenthttp
-    url: 'http://{{ .LoggingParams.HTTPInputConfig.Host }}:{{ .LoggingParams.HTTPInputConfig.Port }}/{{ .LoggingParams.Tag }}'
-    env_prefix: K8S_
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
   fluentforward:
     class: fluent.handler.FluentHandler
@@ -45,9 +33,6 @@ loggers:
   nodepool:
     handlers:
       - console
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-      - fluenthttp
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
       - fluentforward
 {{ end }}
@@ -56,9 +41,6 @@ loggers:
 root:
   handlers:
     - console
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-    - fluenthttp
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
     - fluentforward
 {{ end }}

diff --git a/controllers/static/zookeeper/fluent-bit.conf.tmpl b/controllers/static/zookeeper/fluent-bit.conf.tmpl
@@ -36,15 +36,6 @@
     match   *
     format  json_lines
 {{ end }}
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-[OUTPUT]
-    name    http
-    match   *
-    uri     /{{ .LoggingParams.Tag }}
-    format  json
-    host    {{ .LoggingParams.HTTPInputConfig.Host }}
-    port    {{ .LoggingParams.HTTPInputConfig.Port }}
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
 [OUTPUT]
     name                        forward

diff --git a/controllers/static/zuul/logging.yaml.tmpl b/controllers/static/zuul/logging.yaml.tmpl
@@ -3,10 +3,6 @@ formatters:
   console:
     class: 'zuul.lib.logutil.MultiLineFormatter'
     format: '%(asctime)s %(levelname)s %(name)s: %(message)s'
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-  fluenthttp:
-    format: '%(asctime)s %(levelname)s %(name)s: %(message)s'
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
   fluentforward:
     '()': fluent.handler.FluentRecordFormatter
@@ -23,14 +19,6 @@ handlers:
     formatter: console
     level: {{ .LoggingParams.LogLevel }}
     stream: ext://sys.stdout
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-  fluenthttp:
-    class: sfExtras.SimpleFluentBitHTTPInputHandler
-    level: {{ .LoggingParams.LogLevel }}
-    formatter: fluenthttp
-    url: 'http://{{ .LoggingParams.HTTPInputConfig.Host }}:{{ .LoggingParams.HTTPInputConfig.Port }}/{{ .LoggingParams.Tag }}'
-    env_prefix: K8S_
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
   fluentforward:
     class: fluent.handler.FluentHandler
@@ -45,9 +33,6 @@ loggers:
   zuul.GerritConnection.ssh:
     handlers:
       - console
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-      - fluenthttp
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
       - fluentforward
 {{ end }}
@@ -56,9 +41,6 @@ loggers:
   zuul.GerritConnection.io:
     handlers:
       - console
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-      - fluenthttp
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
       - fluentforward
 {{ end }}
@@ -67,9 +49,6 @@ loggers:
   connection:
     handlers:
       - console
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-      - fluenthttp
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
       - fluentforward
 {{ end }}
@@ -78,9 +57,6 @@ loggers:
   zuul:
     handlers:
       - console
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-      - fluenthttp
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
       - fluentforward
 {{ end }}
@@ -89,9 +65,6 @@ loggers:
 root:
   handlers:
     - console
-{{- if .LoggingParams.HTTPInputConfig.InUse }}
-    - fluenthttp
-{{ end }}
 {{- if .LoggingParams.ForwardInputConfig.InUse }}
     - fluentforward
 {{ end }}

diff --git a/controllers/zuul.go b/controllers/zuul.go
@@ -190,12 +190,6 @@ func (r *SFController) mkZuulContainer(service string, corporateCMExists bool) a
 			Name:      "zuul-ca",
 			MountPath: "/etc/pki/ca-trust/extracted",
 		},
-		{
-			Name:      "extra-python-module",
-			MountPath: "/usr/local/lib/python3.11/site-packages/sfExtras.py",
-			SubPath:   "sfExtras.py",
-			ReadOnly:  true,
-		},
 	}
 	envs := []apiv1.EnvVar{
 		base.MkEnvVar("REQUESTS_CA_BUNDLE", "/etc/ssl/certs/ca-bundle.crt"),
@@ -301,11 +295,6 @@ func mkZuulVolumes(service string, r *SFController, corporateCMExists bool) []ap
 	// Install the logging settings config map resource
 	r.EnsureConfigMap("zuul-logging", r.computeLoggingConfig())
 
-	// Install extra python module
-	r.EnsureConfigMap("extra-python-module", map[string]string{
-		"sfExtras.py": logging.SFExtrasPythonModule,
-	})
-
 	volumes := []apiv1.Volume{
 		base.MkVolumeSecret("ca-cert"),
 		base.MkVolumeSecret("zuul-config"),
@@ -470,7 +459,6 @@ func (r *SFController) EnsureZuulScheduler(cfg *ini.File) bool {
 		"zuul-extra":                 utils.Checksum([]byte(sshConfig)),
 		"zuul-connections":           utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
 		"corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists),
-		"extras-python-module":       utils.Checksum([]byte(logging.SFExtrasPythonModule)),
 	}
 
 	if r.isConfigRepoSet() {
@@ -598,7 +586,6 @@ func (r *SFController) EnsureZuulExecutor(cfg *ini.File) bool {
 		"zuul-logging":               utils.Checksum([]byte(r.getZuulLoggingString("zuul-executor"))),
 		"zuul-connections":           utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
 		"corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists),
-		"extras-python-module":       utils.Checksum([]byte(logging.SFExtrasPythonModule)),
 	}
 	// TODO Add the zk-port-forward-kube-config secret resource version in the annotation if enabled
 
@@ -676,7 +663,6 @@ func (r *SFController) EnsureZuulMerger(cfg *ini.File) bool {
 		"zuul-connections":           utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
 		"zuul-logging":               utils.Checksum([]byte(r.getZuulLoggingString("zuul-merger"))),
 		"corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists),
-		"extras-python-module":       utils.Checksum([]byte(logging.SFExtrasPythonModule)),
 	}
 
 	zm := r.mkHeadlessSatefulSet(service, "", r.getStorageConfOrDefault(r.cr.Spec.Zuul.Merger.Storage), apiv1.ReadWriteOnce, r.cr.Spec.ExtraLabels)
@@ -742,7 +728,6 @@ func (r *SFController) EnsureZuulWeb(cfg *ini.File) bool {
 		"zuul-logging":               utils.Checksum([]byte(r.getZuulLoggingString("zuul-web"))),
 		"zuul-connections":           utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")),
 		"corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists),
-		"extras-python-module":       utils.Checksum([]byte(logging.SFExtrasPythonModule)),
 	}
 
 	zw := base.MkDeployment("zuul-web", r.ns, "", r.cr.Spec.ExtraLabels)

diff --git a/doc/deployment/logging.md b/doc/deployment/logging.md
@@ -42,11 +42,6 @@ needs, the SF-Operator can set up Zuul and Nodepool to forward application logs
 [forward input](https://docs.fluentbit.io/manual/pipeline/inputs/forward) that your deployment
 will be able to reach.
 
-!!! warning
-    The initially supported [HTTP input](https://docs.fluentbit.io/manual/pipeline/inputs/http)
-    will be deprecated in a future version of the operator. Make sure to transition to the
-    forward input if you have been using the HTTP input.
-
 ### Enabling log forwarding on a deployment
 
 In the spec of your Software Factory deployment, edit the following:
@@ -132,7 +127,7 @@ Then add or edit the following section in the spec:
 [...]
 spec:
   FluentBitLogForwarding:
-    httpInputHost: fluent-bit-forward-input
+    forwardInputHost: fluent-bit-forward-input
     forwardInputPort: 24224
 ```
 

diff --git a/doc/reference/CHANGELOG.md b/doc/reference/CHANGELOG.md
@@ -6,6 +6,9 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 ### Removed
+
+- log forwarding - remove support for the HTTP input.
+
 ### Changed
 ### Fixed