Log Forwarding: enable forwarding of mariadb error logs

Change-Id: Ifcc2397e3de6125cc219c61f17a590cd84b22580

controllers/mariadb.go

@@ -6,13 +6,16 @@
 package controllers
 
 import (
+	_ "embed"
 	"fmt"
 	"strconv"
 
 	"github.com/go-sql-driver/mysql"
 	"github.com/softwarefactory-project/sf-operator/controllers/libs/base"
 	"github.com/softwarefactory-project/sf-operator/controllers/libs/conds"
+	logging "github.com/softwarefactory-project/sf-operator/controllers/libs/logging"
 	"github.com/softwarefactory-project/sf-operator/controllers/libs/utils"
+	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -25,6 +28,9 @@ const (
 	zuulDBConfigSecret = "zuul-db-connection"
 )
 
+//go:embed static/mariadb/fluentbit/fluent-bit.conf.tmpl
+var mariadbFluentBitForwarderConfig string
+
 type ZuulDBOpts struct {
 	Username string
 	Password string
@@ -34,6 +40,37 @@ type ZuulDBOpts struct {
 	Params   map[string]string
 }
 
+func createLogForwarderSidecar(r *SFController, annotations map[string]string) (apiv1.Volume, apiv1.Container) {
+
+	fbForwarderConfig := make(map[string]string)
+	fbForwarderConfig["fluent-bit.conf"], _ = utils.ParseString(
+		mariadbFluentBitForwarderConfig,
+		struct {
+			ExtraKeys              []logging.FluentBitLabel
+			FluentBitHTTPInputHost string
+			FluentBitHTTPInputPort string
+		}{[]logging.FluentBitLabel{}, r.cr.Spec.FluentBitLogForwarding.HTTPInputHost, strconv.Itoa(int(r.cr.Spec.FluentBitLogForwarding.HTTPInputPort))})
+	r.EnsureConfigMap("fluentbit-mariadb-cfg", fbForwarderConfig)
+
+	volume := base.MkVolumeCM("mariadb-log-forwarder-config",
+		"fluentbit-mariadb-cfg-config-map")
+
+	volumeMounts := []apiv1.VolumeMount{
+		{
+			Name:      "mariadb-logs",
+			MountPath: "/watch/",
+		},
+		{
+			Name:      "mariadb-log-forwarder-config",
+			MountPath: "/fluent-bit/etc/",
+		},
+	}
+	sidecar := logging.CreateFluentBitSideCarContainer("mariadb", []logging.FluentBitLabel{}, volumeMounts)
+	annotations["mariadb-fluent-bit.conf"] = utils.Checksum([]byte(fbForwarderConfig["fluent-bit.conf"]))
+	annotations["mariadb-fluent-bit-image"] = base.FluentBitImage
+	return volume, sidecar
+}
+
 func (r *SFController) CreateDBInitContainer(username string, password string, dbname string) apiv1.Container {
 	c := "CREATE DATABASE IF NOT EXISTS " + dbname + " CHARACTER SET utf8 COLLATE utf8_general_ci; "
 	g := "GRANT ALL PRIVILEGES ON " + dbname + ".* TO '" + username + "'@'%' IDENTIFIED BY '${USER_PASSWORD}' WITH GRANT OPTION; FLUSH PRIVILEGES;"
@@ -159,15 +196,36 @@ func (r *SFController) DeployMariadb() bool {
 		base.MkEmptyDirVolume("mariadb-run"),
 	}
 
-	r.GetOrCreate(&sts)
+	annotations := map[string]string{
+		"serial": "1",
+	}
+	if r.cr.Spec.FluentBitLogForwarding != nil {
+		fbVolume, fbSidecar := createLogForwarderSidecar(r, annotations)
+		sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, fbSidecar)
+		sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, fbVolume)
+	}
+	sts.Spec.Template.ObjectMeta.Annotations = annotations
+
+	current := appsv1.StatefulSet{}
+	if r.GetM(mariadbIdent, &current) {
+		if !utils.MapEquals(&current.Spec.Template.ObjectMeta.Annotations, &annotations) {
+			r.log.V(1).Info("mariaDB changed, rollout pods ...")
+			current.Spec = sts.DeepCopy().Spec
+			r.UpdateR(&current)
+			return false
+		}
+	} else {
+		current := sts
+		r.CreateR(&current)
+	}
 
 	servicePorts := []int32{mariadbPort}
 	srv := base.MkServicePod(mariadbIdent, r.ns, "mariadb-0", servicePorts, mariaDBPortName)
 	r.EnsureService(&srv)
 
 	var zuulDBSecret apiv1.Secret
 
-	if r.IsStatefulSetReady(&sts) {
+	if r.IsStatefulSetReady(&current) {
 		zuulDBSecret = apiv1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      zuulDBConfigSecret,
@@ -181,7 +239,7 @@ func (r *SFController) DeployMariadb() bool {
 		}
 	}
 
-	isReady := r.IsStatefulSetReady(&sts) && zuulDBSecret.Data != nil
+	isReady := r.IsStatefulSetReady(&current) && zuulDBSecret.Data != nil
 
 	conds.UpdateConditions(&r.cr.Status.Conditions, mariadbIdent, isReady)
 

controllers/static/mariadb/fluentbit/fluent-bit.conf.tmpl

@@ -0,0 +1,35 @@
+[SERVICE]
+    http_server On
+    http_port   2020
+    log_level   debug
+[INPUT]
+    name                tail
+    tag                 ${K8S_NAMESPACE}.${K8S_NODENAME}.${K8S_PODNAME}.mariadb
+    path                /watch/*.log
+    path_key            full_path
+    refresh_interval    5
+    read_from_head      True
+    db                  /watch/mariadb_fluentbit.db
+[FILTER]
+    name    modify
+    match   *
+    add     namespace ${K8S_NAMESPACE}
+    add     nodeName ${K8S_NODENAME}
+    add     podName ${K8S_PODNAME}
+    add     ip ${K8S_PODIP}
+    add     labels_run mariadb
+    add     component mariadb
+{{- range .ExtraKeys }}
+    add     {{ .Key }} ${K8S_{{ .Value -}}}
+{{- end }}
+[OUTPUT]
+    name    stdout
+    match   *
+    format  json_lines
+[OUTPUT]
+    name    http
+    match   *
+    uri     /${K8S_NAMESPACE}.${K8S_NODENAME}.${K8S_PODNAME}.mariadb
+    format  json
+    host    {{ .FluentBitHTTPInputHost }}
+    port    {{ .FluentBitHTTPInputPort }}

roles/health-check/test-log-forwarding/tasks/main.yaml

@@ -15,9 +15,13 @@
   until: "loki_buildinfo.status == 200 and '2.9.2' in loki_buildinfo.content"
 
 - name: Aggregate logs collected by loki
-  ansible.builtin.shell: "~/bin/logcli query '{application=\"zuul\"}' --quiet --limit 5"
-  register: zuul_logs
-  failed_when: zuul_logs.stdout|length < 1
+  ansible.builtin.shell: "~/bin/logcli query '{application=\"{{ item }}\"}' --quiet --limit 5"
+  register: xxx_logs
+  failed_when: xxx_logs.stdout|length < 1
+  loop:
+    - zuul
+    - nodepool
+    - mariadb
 
 - name: Ensure DIB logs are collected by loki
   ansible.builtin.shell: "~/bin/logcli query '{labels_run=\"dib\"}' --quiet --limit 5"