Add support for memberlist bind network configuration (#1059)

* Add support for memberlist bind network configuration Signed-off-by: Ruben Vargas <[email protected]> * remove instance_addr from ingestor ring Signed-off-by: Ruben Vargas <[email protected]> * fix tests Signed-off-by: Ruben Vargas <[email protected]> * fix test Signed-off-by: Ruben Vargas <[email protected]> * Add advertise_addr to memberlist config Signed-off-by: Ruben Vargas <[email protected]> --------- Signed-off-by: Ruben Vargas <[email protected]>
grafana · Oct 22, 2024 · 712891e · 712891e
1 parent a312f46
commit 712891e
Show file tree

Hide file tree

Showing 26 changed files with 998 additions and 317 deletions.
diff --git a/.chloggen/fix_hashring.yaml b/.chloggen/fix_hashring.yaml
@@ -0,0 +1,19 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: tempostack
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for memberlist bind network configuration
+
+# One or more tracking issues related to the change
+issues: [1060]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Adds support to configure the memberlist instance_addr field using the pod network IP range instead of the default private network range used. 
+  In managed Kubernetes/OpenShift cluster environments as well as in special on-prem setup the private IP range might not be available for using them. 
+  With this change set the TempoStack administrator can choose as a bind address the current pod network IP assigned by the cluster's pod network.
diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go
@@ -303,6 +303,18 @@ var AllStatusConditions = []ConditionStatus{ConditionReady, ConditionFailed, Con
 // ConditionReason defines possible reasons for each condition.
 type ConditionReason string
 
+// InstanceAddrType defines the type of pod network to use for advertising IPs to the ring.
+//
+// +kubebuilder:validation:Enum=default;podIP
+type InstanceAddrType string
+
+const (
+	// InstanceAddrDefault when using the first from any private network interfaces (RFC 1918 and RFC 6598).
+	InstanceAddrDefault InstanceAddrType = "default"
+	// InstanceAddrPodIP when using the public pod IP from the cluster's pod network.
+	InstanceAddrPodIP InstanceAddrType = "podIP"
+)
+
 const (
 	// ReasonReady defines a healthy tempo instance.
 	ReasonReady ConditionReason = "Ready"
@@ -423,6 +435,16 @@ type MemberListSpec struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch",displayName="Enable IPv6"
 	EnableIPv6 *bool `json:"enableIPv6,omitempty"`
+
+	// InstanceAddrType defines the type of address to use to advertise to the ring.
+	// Defaults to the first address from any private network interfaces of the current pod.
+	// Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+	// are not available.
+	//
+	// +optional
+	// +kubebuilder:validation:optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:select:default","urn:alm:descriptor:com.tectonic.ui:select:podIP"},displayName="Instance Address"
+	InstanceAddrType InstanceAddrType `json:"instanceAddrType,omitempty"`
 }
 
 // HashRingSpec defines the hash ring configuration.

diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml
@@ -584,6 +584,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml
@@ -79,6 +79,16 @@ spec:
                         description: EnableIPv6 enables IPv6 support for the memberlist
                           based hash ring.
                         type: boolean
+                      instanceAddrType:
+                        description: |-
+                          InstanceAddrType defines the type of address to use to advertise to the ring.
+                          Defaults to the first address from any private network interfaces of the current pod.
+                          Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+                          are not available.
+                        enum:
+                        - default
+                        - podIP
+                        type: string
                     type: object
                 type: object
               images:

diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml
@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0
-    createdAt: "2024-10-16T08:06:56Z"
+    createdAt: "2024-10-12T18:04:06Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -584,6 +584,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml
@@ -79,6 +79,16 @@ spec:
                         description: EnableIPv6 enables IPv6 support for the memberlist
                           based hash ring.
                         type: boolean
+                      instanceAddrType:
+                        description: |-
+                          InstanceAddrType defines the type of address to use to advertise to the ring.
+                          Defaults to the first address from any private network interfaces of the current pod.
+                          Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+                          are not available.
+                        enum:
+                        - default
+                        - podIP
+                        type: string
                     type: object
                 type: object
               images:

diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml
@@ -75,6 +75,16 @@ spec:
                         description: EnableIPv6 enables IPv6 support for the memberlist
                           based hash ring.
                         type: boolean
+                      instanceAddrType:
+                        description: |-
+                          InstanceAddrType defines the type of address to use to advertise to the ring.
+                          Defaults to the first address from any private network interfaces of the current pod.
+                          Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+                          are not available.
+                        enum:
+                        - default
+                        - podIP
+                        type: string
                     type: object
                 type: object
               images:

diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml
@@ -513,6 +513,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml
@@ -513,6 +513,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml
@@ -8,6 +8,7 @@ spec:                                    # TempoStackSpec defines the desired st
   hashRing:                              # HashRing defines the spec for the distributed hash ring configuration.
     memberlist:                          # MemberList configuration spec
       enableIPv6: false                  # EnableIPv6 enables IPv6 support for the memberlist based hash ring.
+      instanceAddrType: ""               # InstanceAddrType defines the type of address to use to advertise to the ring. Defaults to the first address from any private network interfaces of the current pod. Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598) are not available.
   images:                                # Images defines the image for each container.
     jaegerQuery: ""                      # JaegerQuery defines the tempo-query container image.
     oauthProxy: ""                       # OauthProxy defines the oauth proxy image used to protect the jaegerUI on single tenant.

diff --git a/internal/manifests/compactor/compactor.go b/internal/manifests/compactor/compactor.go
@@ -27,6 +27,11 @@ func BuildCompactor(params manifestutils.Params) ([]client.Object, error) {
 	}
 	gates := params.CtrlConfig.Gates
 	tempo := params.Tempo
+
+	if err := memberlist.ConfigureHashRingEnv(&d.Spec.Template.Spec, params.Tempo); err != nil {
+		return nil, err
+	}
+
 	if gates.HTTPEncryption || gates.GRPCEncryption {
 		caBundleName := naming.SigningCABundleName(tempo.Name)
 		if err := manifestutils.ConfigureServiceCA(&d.Spec.Template.Spec, caBundleName); err != nil {
@@ -90,6 +95,7 @@ func deployment(params manifestutils.Params) (*v1.Deployment, error) {
 								"-target=compactor",
 								"-config.file=/conf/tempo.yaml",
 								"-log.level=info",
+								"-config.expand-env=true",
 							},
 							Ports: []corev1.ContainerPort{
 								{