From 8a6f0bb7b7efea83eb2e37b0c40363415a90c6ef Mon Sep 17 00:00:00 2001 From: Matthieu Huin Date: Fri, 4 Oct 2024 18:04:11 +0200 Subject: [PATCH] manage sfExtras.py module in sf-operator This is much easier to manage than at container build time. Change-Id: Ifd327c99b0a90cb98348936d75e28a3a581e7881 --- controllers/libs/logging/logging.go | 5 ++++ controllers/libs/logging/static/sfExtras.py | 32 +++++++++++++++++++++ controllers/zuul.go | 16 +++++++++++ 3 files changed, 53 insertions(+) create mode 100644 controllers/libs/logging/static/sfExtras.py diff --git a/controllers/libs/logging/logging.go b/controllers/libs/logging/logging.go index 5ba255f..32ff1ac 100644 --- a/controllers/libs/logging/logging.go +++ b/controllers/libs/logging/logging.go @@ -29,6 +29,11 @@ type PythonTemplateLoggingParams struct { BaseURL string } +var ( + //go:embed static/sfExtras.py + SFExtrasPythonModule string +) + func CreateForwarderEnvVars(name string, extraLabels []FluentBitLabel) []apiv1.EnvVar { forwarderEnvVars := []apiv1.EnvVar{ base.MkEnvVarFromFieldRef("K8S_NODENAME", "spec.nodeName"), diff --git a/controllers/libs/logging/static/sfExtras.py b/controllers/libs/logging/static/sfExtras.py new file mode 100644 index 0000000..64afcd2 --- /dev/null +++ b/controllers/libs/logging/static/sfExtras.py @@ -0,0 +1,32 @@ +# Copyright (C) 2023 Red Hat +# SPDX-License-Identifier: Apache-2.0 + + +import os +import logging + +import requests + + +class SimpleFluentBitHTTPInputHandler(logging.Handler): + """A minimal handler for sending logs to the HTTP Input + of a Fluent Bit collector.""" + def __init__(self, url, env_prefix=None): + logging.Handler.__init__(self) + self.url = url + self.env_prefix = env_prefix + + def emit(self, record): + d = { + 'log': self.format(record) + } + if self.env_prefix is not None: + for envvar in os.environ: + if envvar.startswith(self.env_prefix): + key = envvar[len(self.env_prefix):].lower() + d[key] = os.environ[envvar] + try: + req = requests.post(self.url, json=d) + req.raise_for_status() + except requests.HTTPError as e: + self.handleError(record) \ No newline at end of file diff --git a/controllers/zuul.go b/controllers/zuul.go index 3a7142b..16abfa9 100644 --- a/controllers/zuul.go +++ b/controllers/zuul.go @@ -190,6 +190,12 @@ func (r *SFController) mkZuulContainer(service string, corporateCMExists bool) a Name: "zuul-ca", MountPath: "/etc/pki/ca-trust/extracted", }, + { + Name: "extra-python-module", + MountPath: "/usr/local/lib/python3.11/site-packages/sfExtras.py", + SubPath: "sfExtras.py", + ReadOnly: true, + }, } envs := []apiv1.EnvVar{ base.MkEnvVar("REQUESTS_CA_BUNDLE", "/etc/ssl/certs/ca-bundle.crt"), @@ -295,6 +301,11 @@ func mkZuulVolumes(service string, r *SFController, corporateCMExists bool) []ap // Install the logging settings config map resource r.EnsureConfigMap("zuul-logging", r.computeLoggingConfig()) + // Install extra python module + r.EnsureConfigMap("extra-python-module", map[string]string{ + "sfExtras.py": logging.SFExtrasPythonModule, + }) + volumes := []apiv1.Volume{ base.MkVolumeSecret("ca-cert"), base.MkVolumeSecret("zuul-config"), @@ -311,6 +322,7 @@ func mkZuulVolumes(service string, r *SFController, corporateCMExists bool) []ap }, base.MkVolumeCM("statsd-config", "zuul-statsd-config-map"), base.MkVolumeCM("extra-config", "zuul-extra-config-map"), + base.MkVolumeCM("extra-python-module", "extra-python-module-config-map"), base.MkEmptyDirVolume("zuul-ca"), } if !isStatefulset(service) { @@ -452,6 +464,7 @@ func (r *SFController) EnsureZuulScheduler(cfg *ini.File) bool { "zuul-extra": utils.Checksum([]byte(sshConfig)), "zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")), "corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists), + "extras-python-module": utils.Checksum([]byte(logging.SFExtrasPythonModule)), } if r.isConfigRepoSet() { @@ -579,6 +592,7 @@ func (r *SFController) EnsureZuulExecutor(cfg *ini.File) bool { "zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-executor"))), "zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")), "corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists), + "extras-python-module": utils.Checksum([]byte(logging.SFExtrasPythonModule)), } // TODO Add the zk-port-forward-kube-config secret resource version in the annotation if enabled @@ -656,6 +670,7 @@ func (r *SFController) EnsureZuulMerger(cfg *ini.File) bool { "zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")), "zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-merger"))), "corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists), + "extras-python-module": utils.Checksum([]byte(logging.SFExtrasPythonModule)), } zm := r.mkHeadlessSatefulSet(service, "", r.getStorageConfOrDefault(r.cr.Spec.Zuul.Merger.Storage), apiv1.ReadWriteOnce, r.cr.Spec.ExtraLabels) @@ -721,6 +736,7 @@ func (r *SFController) EnsureZuulWeb(cfg *ini.File) bool { "zuul-logging": utils.Checksum([]byte(r.getZuulLoggingString("zuul-web"))), "zuul-connections": utils.IniSectionsChecksum(cfg, utils.IniGetSectionNamesByPrefix(cfg, "connection")), "corporate-ca-certs-version": getCMVersion(corporateCM, corporateCMExists), + "extras-python-module": utils.Checksum([]byte(logging.SFExtrasPythonModule)), } zw := base.MkDeployment("zuul-web", r.ns, "", r.cr.Spec.ExtraLabels)