feat(dynamic-exporter): Add dynamic flow exporter support for hubble

Author: anubhabMajumdar

Makefile

@@ -53,6 +53,11 @@ CERT_FILES := tls.crt:tls-client-cert-file \
               tls.key:tls-client-key-file \
               ca.crt:tls-ca-cert-files
 
+##################
+# Dynamic Export #
+##################
+ENABLE_DYNAMIC_EXPORT ?= false
+
 # TAG is OS and platform agonstic, which can be used for binary version and image manifest tag,
 # while RETINA_PLATFORM_TAG is platform specific, which can be used for image built for specific platforms.
 RETINA_PLATFORM_TAG        ?= $(TAG)-$(subst /,-,$(PLATFORM))
@@ -518,6 +523,7 @@ helm-install-hubble:
 		--set agent.init.repository=$(IMAGE_REGISTRY)/$(RETINA_INIT_IMAGE) \
 		--set agent.init.tag=$(HELM_IMAGE_TAG) \
 		--set logLevel=info \
+		--set hubble.export.dynamic.enabled=$(ENABLE_DYNAMIC_EXPORT) \
 		--set hubble.tls.enabled=$(ENABLE_TLS) \
 		--set hubble.relay.tls.server.enabled=$(ENABLE_TLS) \
 		--set hubble.tls.auto.enabled=$(ENABLE_TLS) \
@@ -577,7 +583,7 @@ quick-deploy:
 .PHONY: quick-deploy-hubble
 quick-deploy-hubble:
 	$(MAKE) helm-uninstall || true
-	$(MAKE) helm-install-without-tls HELM_IMAGE_TAG=$(TAG)-linux-amd64
+	$(MAKE) helm-install-without-tls HELM_IMAGE_TAG=$(TAG)-linux-amd64 ENABLE_DYNAMIC_EXPORT=true
 
 
 .PHONY: simplify-dashboards

controller/Dockerfile

@@ -116,8 +116,8 @@ ENTRYPOINT ["./retina/initretina"]
 
 
 # agent final image
-# mcr.microsoft.com/cbl-mariner/distroless/minimal:2.0
-# mcr.microsoft.com/cbl-mariner/distroless/minimal@sha256:63a0a70ceaa1320bc6eb98b81106667d43e46b674731ea8d28e4de1b87e0747f
+# For debug: mcr.microsoft.com/cbl-mariner/distroless/debug:2.0
+# k exec -it ds/retina-agent -- busybox tail -f /var/run/retina/hubble/events.log
 FROM mariner-distroless AS agent
 COPY --from=tools /lib/ /lib
 COPY --from=tools /usr/lib/ /usr/lib

deploy/hubble/manifests/controller/helm/retina/templates/agent/daemonset.yaml

@@ -112,6 +112,11 @@ spec:
               mountPath: /var/lib/cilium/tls/hubble
               readOnly: true
           {{- end }}
+          {{- if .Values.hubble.export.dynamic.enabled }}
+            - name: hubble-flowlog-config
+              mountPath: /flowlog-config
+              readOnly: true
+          {{- end }}
         {{- end }}
       terminationGracePeriodSeconds: 90 # Allow for retina to cleanup plugin resources.
       volumes:
@@ -142,6 +147,12 @@ spec:
                   - key: ca.crt
                     path: client-ca.crt
       {{- end }}
+      {{- if .Values.hubble.export.dynamic.enabled }}
+      - name: hubble-flowlog-config
+        configMap:
+          name: {{ .Values.hubble.export.dynamic.config.configMapName }}
+          optional: true
+      {{- end }}
 {{- end }}
 ---
 {{- if .Values.os.windows}}

deploy/hubble/manifests/controller/helm/retina/templates/agent/flowlog-configmap.yaml

@@ -0,0 +1,12 @@
+{{- if and .Values.hubble.export.dynamic.enabled .Values.hubble.export.dynamic.config.createConfigMap }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.hubble.export.dynamic.config.configMapName }}
+  namespace: {{ .Release.Namespace }}
+data:
+  flowlogs.yaml: |
+    flowLogs:
+{{ .Values.hubble.export.dynamic.config.content | toYaml | indent 4 }}
+{{- end }}

deploy/hubble/manifests/controller/helm/retina/values.yaml

@@ -821,10 +821,10 @@ hubble:
     # --- Dynamic exporters configuration.
     # Dynamic exporters may be reconfigured without a need of agent restarts.
     dynamic:
-      enabled: false
+      enabled: true
       config:
         # ---- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents.
-        configMapName: cilium-flowlog-config
+        configMapName: retina-flowlog-config
         # ---- True if helm installer should create config map.
         # Switch to false if you want to self maintain the file content.
         createConfigMap: true
@@ -834,7 +834,7 @@ hubble:
             fieldMask: []
             includeFilters: []
             excludeFilters: []
-            filePath: "/var/run/cilium/hubble/events.log"
+            filePath: "/var/run/retina/hubble/events.log"
         #- name: "test002"
         #  filePath: "/var/log/network/flow-log/pa/test002.log"
         #  fieldMask: ["source.namespace", "source.pod_name", "destination.namespace", "destination.pod_name", "verdict"]

pkg/hubble/hubble_linux.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pkg/errors"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/cilium/cilium/pkg/hubble/exporter"
 	"github.com/sirupsen/logrus"
 	"go.uber.org/zap"
 )
@@ -114,6 +115,19 @@ func (rh *RetinaHubble) start(ctx context.Context) error {
 		}),
 	)
 
+	// Start the dynamic exporter if the config file path is provided.
+	rh.log.Info("Starting dynamic exporter", zap.String("config", option.Config.HubbleFlowlogsConfigFilePath))
+	if option.Config.HubbleFlowlogsConfigFilePath != "" {
+		dynamicHubbleExporter := exporter.NewDynamicExporter(
+			rh.log,
+			option.Config.HubbleFlowlogsConfigFilePath,
+			option.Config.HubbleExportFileMaxSizeMB,
+			option.Config.HubbleExportFileMaxBackups)
+		opt := observeroption.WithOnDecodedEvent(dynamicHubbleExporter)
+		observerOpts = append(observerOpts, opt)
+		rh.log.Info("Started dynamic exporter")
+	}
+
 	// TODO: Replace with our custom parser.
 	payloadParser := parser.New(rh.log, rh.ipc)