From 82eab55e2a09547fab7a76e4512af746df4f9b65 Mon Sep 17 00:00:00 2001 From: Matthieu Huin Date: Wed, 15 Nov 2023 16:17:22 +0100 Subject: [PATCH] Log Forwarding: enable forwarding of mariadb error logs Change-Id: Ifcc2397e3de6125cc219c61f17a590cd84b22580 --- controllers/mariadb.go | 64 ++++++++++++++++++- .../mariadb/fluentbit/fluent-bit.conf.tmpl | 35 ++++++++++ .../test-log-forwarding/tasks/main.yaml | 10 ++- 3 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl diff --git a/controllers/mariadb.go b/controllers/mariadb.go index 9a366169..1006a1b5 100644 --- a/controllers/mariadb.go +++ b/controllers/mariadb.go @@ -6,13 +6,16 @@ package controllers import ( + _ "embed" "fmt" "strconv" "github.com/go-sql-driver/mysql" "github.com/softwarefactory-project/sf-operator/controllers/libs/base" "github.com/softwarefactory-project/sf-operator/controllers/libs/conds" + logging "github.com/softwarefactory-project/sf-operator/controllers/libs/logging" "github.com/softwarefactory-project/sf-operator/controllers/libs/utils" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,6 +28,9 @@ const ( zuulDBConfigSecret = "zuul-db-connection" ) +//go:embed static/mariadb/fluentbit/fluent-bit.conf.tmpl +var mariadbFluentBitForwarderConfig string + type ZuulDBOpts struct { Username string Password string @@ -34,6 +40,37 @@ type ZuulDBOpts struct { Params map[string]string } +func createLogForwarderSidecar(r *SFController, annotations map[string]string) (apiv1.Volume, apiv1.Container) { + + fbForwarderConfig := make(map[string]string) + fbForwarderConfig["fluent-bit.conf"], _ = utils.ParseString( + mariadbFluentBitForwarderConfig, + struct { + ExtraKeys []logging.FluentBitLabel + FluentBitHTTPInputHost string + FluentBitHTTPInputPort string + }{[]logging.FluentBitLabel{}, r.cr.Spec.FluentBitLogForwarding.HTTPInputHost, strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort))}) + r.EnsureConfigMap("fluentbit-mariadb-cfg", fbForwarderConfig) + + volume := base.MkVolumeCM("mariadb-log-forwarder-config", + "fluentbit-mariadb-cfg-config-map") + + volumeMounts := []apiv1.VolumeMount{ + { + Name: "mariadb-logs", + MountPath: "/watch/", + }, + { + Name: "mariadb-log-forwarder-config", + MountPath: "/fluent-bit/etc/", + }, + } + sidecar := logging.CreateFluentBitSideCarContainer("mariadb", []logging.FluentBitLabel{}, volumeMounts) + annotations["mariadb-fluent-bit.conf"] = utils.Checksum([]byte(fbForwarderConfig["fluent-bit.conf"])) + annotations["mariadb-fluent-bit-image"] = base.FluentBitImage + return volume, sidecar +} + func (r *SFController) CreateDBInitContainer(username string, password string, dbname string) apiv1.Container { c := "CREATE DATABASE IF NOT EXISTS " + dbname + " CHARACTER SET utf8 COLLATE utf8_general_ci; " g := "GRANT ALL PRIVILEGES ON " + dbname + ".* TO '" + username + "'@'%' IDENTIFIED BY '${USER_PASSWORD}' WITH GRANT OPTION; FLUSH PRIVILEGES;" @@ -159,7 +196,28 @@ func (r *SFController) DeployMariadb() bool { base.MkEmptyDirVolume("mariadb-run"), } - r.GetOrCreate(&sts) + annotations := map[string]string{ + "serial": "1", + } + if r.cr.Spec.FluentBitLogForwarding != nil { + fbVolume, fbSidecar := createLogForwarderSidecar(r, annotations) + sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, fbSidecar) + sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, fbVolume) + } + sts.Spec.Template.ObjectMeta.Annotations = annotations + + current := appsv1.StatefulSet{} + if r.GetM(mariadbIdent, ¤t) { + if !utils.MapEquals(¤t.Spec.Template.ObjectMeta.Annotations, &annotations) { + r.log.V(1).Info("mariaDB changed, rollout pods ...") + current.Spec = sts.DeepCopy().Spec + r.UpdateR(¤t) + return false + } + } else { + current := sts + r.CreateR(¤t) + } servicePorts := []int32{mariadbPort} srv := base.MkServicePod(mariadbIdent, r.ns, "mariadb-0", servicePorts, mariaDBPortName) @@ -167,7 +225,7 @@ func (r *SFController) DeployMariadb() bool { var zuulDBSecret apiv1.Secret - if r.IsStatefulSetReady(&sts) { + if r.IsStatefulSetReady(¤t) { zuulDBSecret = apiv1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: zuulDBConfigSecret, @@ -181,7 +239,7 @@ func (r *SFController) DeployMariadb() bool { } } - isReady := r.IsStatefulSetReady(&sts) && zuulDBSecret.Data != nil + isReady := r.IsStatefulSetReady(¤t) && zuulDBSecret.Data != nil conds.UpdateConditions(&r.cr.Status.Conditions, mariadbIdent, isReady) diff --git a/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl b/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl new file mode 100644 index 00000000..4c54a772 --- /dev/null +++ b/controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl @@ -0,0 +1,35 @@ +[SERVICE] + http_server On + http_port 2020 + log_level debug +[INPUT] + name tail + tag ${K8S_NAMESPACE}.${K8S_NODENAME}.${K8S_PODNAME}.mariadb + path /watch/*.log + path_key full_path + refresh_interval 5 + read_from_head True + db /watch/mariadb_fluentbit.db +[FILTER] + name modify + match * + add namespace ${K8S_NAMESPACE} + add nodeName ${K8S_NODENAME} + add podName ${K8S_PODNAME} + add ip ${K8S_PODIP} + add labels_run mariadb + add component mariadb +{{- range .ExtraKeys }} + add {{ .Key }} ${K8S_{{ .Value -}}} +{{- end }} +[OUTPUT] + name stdout + match * + format json_lines +[OUTPUT] + name http + match * + uri /${K8S_NAMESPACE}.${K8S_NODENAME}.${K8S_PODNAME}.mariadb + format json + host {{ .FluentBitHTTPInputHost }} + port {{ .FluentBitHTTPInputPort }} \ No newline at end of file diff --git a/roles/health-check/test-log-forwarding/tasks/main.yaml b/roles/health-check/test-log-forwarding/tasks/main.yaml index 990225fb..3b4f2c14 100644 --- a/roles/health-check/test-log-forwarding/tasks/main.yaml +++ b/roles/health-check/test-log-forwarding/tasks/main.yaml @@ -15,9 +15,13 @@ until: "loki_buildinfo.status == 200 and '2.9.2' in loki_buildinfo.content" - name: Aggregate logs collected by loki - ansible.builtin.shell: "~/bin/logcli query '{application=\"zuul\"}' --quiet --limit 5" - register: zuul_logs - failed_when: zuul_logs.stdout|length < 1 + ansible.builtin.shell: "~/bin/logcli query '{application=\"{{ item }}\"}' --quiet --limit 5" + register: xxx_logs + failed_when: xxx_logs.stdout|length < 1 + loop: + - zuul + - nodepool + - mariadb - name: Ensure DIB logs are collected by loki ansible.builtin.shell: "~/bin/logcli query '{labels_run=\"dib\"}' --quiet --limit 5"