diff --git a/.chloggen/fix_hashring.yaml b/.chloggen/fix_hashring.yaml new file mode 100755 index 000000000..f1d8a8d05 --- /dev/null +++ b/.chloggen/fix_hashring.yaml @@ -0,0 +1,19 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add support for memberlist bind network configuration + +# One or more tracking issues related to the change +issues: [1060] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Adds support to configure the memberlist instance_addr field using the pod network IP range instead of the default private network range used. + In managed Kubernetes/OpenShift cluster environments as well as in special on-prem setup the private IP range might not be available for using them. + With this change set the TempoStack administrator can choose as a bind address the current pod network IP assigned by the cluster's pod network. diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index e868b207d..f71123172 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -303,6 +303,18 @@ var AllStatusConditions = []ConditionStatus{ConditionReady, ConditionFailed, Con // ConditionReason defines possible reasons for each condition. type ConditionReason string +// InstanceAddrType defines the type of pod network to use for advertising IPs to the ring. +// +// +kubebuilder:validation:Enum=default;podIP +type InstanceAddrType string + +const ( + // InstanceAddrDefault when using the first from any private network interfaces (RFC 1918 and RFC 6598). + InstanceAddrDefault InstanceAddrType = "default" + // InstanceAddrPodIP when using the public pod IP from the cluster's pod network. + InstanceAddrPodIP InstanceAddrType = "podIP" +) + const ( // ReasonReady defines a healthy tempo instance. ReasonReady ConditionReason = "Ready" @@ -423,6 +435,16 @@ type MemberListSpec struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch",displayName="Enable IPv6" EnableIPv6 *bool `json:"enableIPv6,omitempty"` + + // InstanceAddrType defines the type of address to use to advertise to the ring. + // Defaults to the first address from any private network interfaces of the current pod. + // Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598) + // are not available. + // + // +optional + // +kubebuilder:validation:optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:select:default","urn:alm:descriptor:com.tectonic.ui:select:podIP"},displayName="Instance Address" + InstanceAddrType InstanceAddrType `json:"instanceAddrType,omitempty"` } // HashRingSpec defines the hash ring configuration. diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index e52d8057c..103fd3d01 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -584,6 +584,15 @@ spec: path: hashRing.memberlist.enableIPv6 x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: InstanceAddrType defines the type of address to use to advertise + to the ring. Defaults to the first address from any private network interfaces + of the current pod. Alternatively the public pod IP can be used in case + private networks (RFC 1918 and RFC 6598) are not available. + displayName: Instance Address + path: hashRing.memberlist.instanceAddrType + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:default + - urn:alm:descriptor:com.tectonic.ui:select:podIP - description: Images defines the image for each container. displayName: Container Images path: images diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index 6ec9eaffb..ad90ab922 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -79,6 +79,16 @@ spec: description: EnableIPv6 enables IPv6 support for the memberlist based hash ring. type: boolean + instanceAddrType: + description: |- + InstanceAddrType defines the type of address to use to advertise to the ring. + Defaults to the first address from any private network interfaces of the current pod. + Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598) + are not available. + enum: + - default + - podIP + type: string type: object type: object images: diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 9764552e3..7a2ead8e5 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-16T08:06:56Z" + createdAt: "2024-10-12T18:04:06Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -584,6 +584,15 @@ spec: path: hashRing.memberlist.enableIPv6 x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: InstanceAddrType defines the type of address to use to advertise + to the ring. Defaults to the first address from any private network interfaces + of the current pod. Alternatively the public pod IP can be used in case + private networks (RFC 1918 and RFC 6598) are not available. + displayName: Instance Address + path: hashRing.memberlist.instanceAddrType + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:default + - urn:alm:descriptor:com.tectonic.ui:select:podIP - description: Images defines the image for each container. displayName: Container Images path: images diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index 6ec9eaffb..ad90ab922 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -79,6 +79,16 @@ spec: description: EnableIPv6 enables IPv6 support for the memberlist based hash ring. type: boolean + instanceAddrType: + description: |- + InstanceAddrType defines the type of address to use to advertise to the ring. + Defaults to the first address from any private network interfaces of the current pod. + Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598) + are not available. + enum: + - default + - podIP + type: string type: object type: object images: diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index 9b156e754..da0c38c5e 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -75,6 +75,16 @@ spec: description: EnableIPv6 enables IPv6 support for the memberlist based hash ring. type: boolean + instanceAddrType: + description: |- + InstanceAddrType defines the type of address to use to advertise to the ring. + Defaults to the first address from any private network interfaces of the current pod. + Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598) + are not available. + enum: + - default + - podIP + type: string type: object type: object images: diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml index 183b5d776..6500bcdf8 100644 --- a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml @@ -513,6 +513,15 @@ spec: path: hashRing.memberlist.enableIPv6 x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: InstanceAddrType defines the type of address to use to advertise + to the ring. Defaults to the first address from any private network interfaces + of the current pod. Alternatively the public pod IP can be used in case + private networks (RFC 1918 and RFC 6598) are not available. + displayName: Instance Address + path: hashRing.memberlist.instanceAddrType + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:default + - urn:alm:descriptor:com.tectonic.ui:select:podIP - description: Images defines the image for each container. displayName: Container Images path: images diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml index 55a0cd54a..4f3f7f264 100644 --- a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml @@ -513,6 +513,15 @@ spec: path: hashRing.memberlist.enableIPv6 x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: InstanceAddrType defines the type of address to use to advertise + to the ring. Defaults to the first address from any private network interfaces + of the current pod. Alternatively the public pod IP can be used in case + private networks (RFC 1918 and RFC 6598) are not available. + displayName: Instance Address + path: hashRing.memberlist.instanceAddrType + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:default + - urn:alm:descriptor:com.tectonic.ui:select:podIP - description: Images defines the image for each container. displayName: Container Images path: images diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index ac32bdb41..a7641ae03 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -8,6 +8,7 @@ spec: # TempoStackSpec defines the desired st hashRing: # HashRing defines the spec for the distributed hash ring configuration. memberlist: # MemberList configuration spec enableIPv6: false # EnableIPv6 enables IPv6 support for the memberlist based hash ring. + instanceAddrType: "" # InstanceAddrType defines the type of address to use to advertise to the ring. Defaults to the first address from any private network interfaces of the current pod. Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598) are not available. images: # Images defines the image for each container. jaegerQuery: "" # JaegerQuery defines the tempo-query container image. oauthProxy: "" # OauthProxy defines the oauth proxy image used to protect the jaegerUI on single tenant. diff --git a/internal/manifests/compactor/compactor.go b/internal/manifests/compactor/compactor.go index e88726861..942817a20 100644 --- a/internal/manifests/compactor/compactor.go +++ b/internal/manifests/compactor/compactor.go @@ -27,6 +27,11 @@ func BuildCompactor(params manifestutils.Params) ([]client.Object, error) { } gates := params.CtrlConfig.Gates tempo := params.Tempo + + if err := memberlist.ConfigureHashRingEnv(&d.Spec.Template.Spec, params.Tempo); err != nil { + return nil, err + } + if gates.HTTPEncryption || gates.GRPCEncryption { caBundleName := naming.SigningCABundleName(tempo.Name) if err := manifestutils.ConfigureServiceCA(&d.Spec.Template.Spec, caBundleName); err != nil { @@ -90,6 +95,7 @@ func deployment(params manifestutils.Params) (*v1.Deployment, error) { "-target=compactor", "-config.file=/conf/tempo.yaml", "-log.level=info", + "-config.expand-env=true", }, Ports: []corev1.ContainerPort{ { diff --git a/internal/manifests/compactor/compactor_test.go b/internal/manifests/compactor/compactor_test.go index bc1056192..deed86532 100644 --- a/internal/manifests/compactor/compactor_test.go +++ b/internal/manifests/compactor/compactor_test.go @@ -19,174 +19,217 @@ import ( ) func TestBuildCompactor(t *testing.T) { - objects, err := BuildCompactor(manifestutils.Params{Tempo: v1alpha1.TempoStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "project1", + + tests := []struct { + name string + instanceAddrType v1alpha1.InstanceAddrType + expectedContainerEnvVars []corev1.EnvVar + }{ + { + name: "default", + expectedContainerEnvVars: []corev1.EnvVar{}, }, - Spec: v1alpha1.TempoStackSpec{ - Images: configv1alpha1.ImagesSpec{ - Tempo: "docker.io/grafana/tempo:1.5.0", - }, - ServiceAccount: "tempo-test-serviceaccount", - Template: v1alpha1.TempoTemplateSpec{ - Compactor: v1alpha1.TempoComponentSpec{ - Replicas: ptr.To(int32(2)), - NodeSelector: map[string]string{"a": "b"}, - Tolerations: []corev1.Toleration{ - { - Key: "c", + { + name: "set InstanceAddrType to PodIP", + expectedContainerEnvVars: []corev1.EnvVar{ + { + Name: "HASH_RING_INSTANCE_ADDR", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "status.podIP", }, }, }, }, - Resources: v1alpha1.Resources{ - Total: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1000m"), - corev1.ResourceMemory: resource.MustParse("2Gi"), - }, - }, - }, + instanceAddrType: v1alpha1.InstanceAddrPodIP, }, - }}) - require.NoError(t, err) + } - labels := manifestutils.ComponentLabels("compactor", "test") - annotations := manifestutils.CommonAnnotations("") - assert.Equal(t, 2, len(objects)) + for _, ts := range tests { + t.Run(ts.name, func(t *testing.T) { - assert.Equal(t, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tempo-test-compactor", - Namespace: "project1", - Labels: labels, - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: manifestutils.HttpMemberlistPortName, - Protocol: corev1.ProtocolTCP, - Port: manifestutils.PortMemberlist, - TargetPort: intstr.FromString(manifestutils.HttpMemberlistPortName), + instanceAddrType := v1alpha1.InstanceAddrDefault + if ts.instanceAddrType != "" { + instanceAddrType = ts.instanceAddrType + } + + objects, err := BuildCompactor(manifestutils.Params{Tempo: v1alpha1.TempoStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "project1", }, - { - Name: manifestutils.HttpPortName, - Protocol: corev1.ProtocolTCP, - Port: manifestutils.PortHTTPServer, - TargetPort: intstr.FromString(manifestutils.HttpPortName), + Spec: v1alpha1.TempoStackSpec{ + Images: configv1alpha1.ImagesSpec{ + Tempo: "docker.io/grafana/tempo:1.5.0", + }, + ServiceAccount: "tempo-test-serviceaccount", + Template: v1alpha1.TempoTemplateSpec{ + Compactor: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(2)), + NodeSelector: map[string]string{"a": "b"}, + Tolerations: []corev1.Toleration{ + { + Key: "c", + }, + }, + }, + }, + Resources: v1alpha1.Resources{ + Total: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1000m"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + HashRing: v1alpha1.HashRingSpec{ + MemberList: v1alpha1.MemberListSpec{ + InstanceAddrType: instanceAddrType, + }, + }, }, - }, - Selector: labels, - }, - }, objects[1]) + }}) + require.NoError(t, err) - assert.Equal(t, &v1.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "tempo-test-compactor", - Namespace: "project1", - Labels: labels, - }, - Spec: v1.DeploymentSpec{ - Replicas: ptr.To(int32(2)), - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: corev1.PodTemplateSpec{ + labels := manifestutils.ComponentLabels("compactor", "test") + annotations := manifestutils.CommonAnnotations("") + assert.Equal(t, 2, len(objects)) + + assert.Equal(t, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Labels: k8slabels.Merge(labels, map[string]string{"tempo-gossip-member": "true"}), - Annotations: annotations, + Name: "tempo-test-compactor", + Namespace: "project1", + Labels: labels, }, - Spec: corev1.PodSpec{ - ServiceAccountName: "tempo-test-serviceaccount", - NodeSelector: map[string]string{"a": "b"}, - Tolerations: []corev1.Toleration{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ { - Key: "c", + Name: manifestutils.HttpMemberlistPortName, + Protocol: corev1.ProtocolTCP, + Port: manifestutils.PortMemberlist, + TargetPort: intstr.FromString(manifestutils.HttpMemberlistPortName), }, - }, - Containers: []corev1.Container{ { - Name: "tempo", - Image: "docker.io/grafana/tempo:1.5.0", - Env: []corev1.EnvVar{}, - Args: []string{ - "-target=compactor", - "-config.file=/conf/tempo.yaml", - "-log.level=info", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: manifestutils.ConfigVolumeName, - MountPath: "/conf", - ReadOnly: true, - }, + Name: manifestutils.HttpPortName, + Protocol: corev1.ProtocolTCP, + Port: manifestutils.PortHTTPServer, + TargetPort: intstr.FromString(manifestutils.HttpPortName), + }, + }, + Selector: labels, + }, + }, objects[1]) + + assert.Equal(t, &v1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "tempo-test-compactor", + Namespace: "project1", + Labels: labels, + }, + Spec: v1.DeploymentSpec{ + Replicas: ptr.To(int32(2)), + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: k8slabels.Merge(labels, map[string]string{"tempo-gossip-member": "true"}), + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "tempo-test-serviceaccount", + NodeSelector: map[string]string{"a": "b"}, + Tolerations: []corev1.Toleration{ { - Name: manifestutils.TmpStorageVolumeName, - MountPath: manifestutils.TmpTempoStoragePath, + Key: "c", }, }, - Ports: []corev1.ContainerPort{ + Containers: []corev1.Container{ { - Name: manifestutils.HttpPortName, - ContainerPort: manifestutils.PortHTTPServer, - Protocol: corev1.ProtocolTCP, - }, - { - Name: manifestutils.HttpMemberlistPortName, - ContainerPort: manifestutils.PortMemberlist, - Protocol: corev1.ProtocolTCP, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - Path: manifestutils.TempoReadinessPath, - Port: intstr.FromString(manifestutils.HttpPortName), + Name: "tempo", + Image: "docker.io/grafana/tempo:1.5.0", + Env: ts.expectedContainerEnvVars, + Args: []string{ + "-target=compactor", + "-config.file=/conf/tempo.yaml", + "-log.level=info", + "-config.expand-env=true", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: manifestutils.ConfigVolumeName, + MountPath: "/conf", + ReadOnly: true, + }, + { + Name: manifestutils.TmpStorageVolumeName, + MountPath: manifestutils.TmpTempoStoragePath, + }, + }, + Ports: []corev1.ContainerPort{ + { + Name: manifestutils.HttpPortName, + ContainerPort: manifestutils.PortHTTPServer, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.HttpMemberlistPortName, + ContainerPort: manifestutils.PortMemberlist, + Protocol: corev1.ProtocolTCP, + }, }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + Path: manifestutils.TempoReadinessPath, + Port: intstr.FromString(manifestutils.HttpPortName), + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(80, resource.BinarySI), + corev1.ResourceMemory: *resource.NewQuantity(193273536, resource.BinarySI), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(24, resource.BinarySI), + corev1.ResourceMemory: *resource.NewQuantity(57982064, resource.BinarySI), + }, + }, + SecurityContext: manifestutils.TempoContainerSecurityContext(), }, - InitialDelaySeconds: 15, - TimeoutSeconds: 1, }, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: *resource.NewMilliQuantity(80, resource.BinarySI), - corev1.ResourceMemory: *resource.NewQuantity(193273536, resource.BinarySI), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: *resource.NewMilliQuantity(24, resource.BinarySI), - corev1.ResourceMemory: *resource.NewQuantity(57982064, resource.BinarySI), + Volumes: []corev1.Volume{ + { + Name: manifestutils.ConfigVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tempo-test", + }, + }, + }, }, - }, - SecurityContext: manifestutils.TempoContainerSecurityContext(), - }, - }, - Volumes: []corev1.Volume{ - { - Name: manifestutils.ConfigVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tempo-test", + { + Name: manifestutils.TmpStorageVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, }, }, - { - Name: manifestutils.TmpStorageVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, }, }, - }, - }, - }, objects[0]) + }, objects[0]) + }) + } } func TestOverrideResources(t *testing.T) { diff --git a/internal/manifests/config/build.go b/internal/manifests/config/build.go index 9b17bc705..b5a5f58db 100644 --- a/internal/manifests/config/build.go +++ b/internal/manifests/config/build.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/tempo-operator/apis/tempo/v1alpha1" "github.com/grafana/tempo-operator/internal/manifests/manifestutils" + "github.com/grafana/tempo-operator/internal/manifests/memberlist" "github.com/grafana/tempo-operator/internal/manifests/naming" ) @@ -73,8 +74,9 @@ func buildConfiguration(params manifestutils.Params) ([]byte, error) { StorageParams: params.StorageParams, GlobalRetention: tempo.Spec.Retention.Global.Traces.Duration.String(), MemberList: memberlistOptions{ - JoinMembers: []string{naming.Name("gossip-ring", tempo.Name)}, - EnableIPv6: ptr.Deref(tempo.Spec.HashRing.MemberList.EnableIPv6, false), + JoinMembers: []string{naming.Name("gossip-ring", tempo.Name)}, + EnableIPv6: ptr.Deref(tempo.Spec.HashRing.MemberList.EnableIPv6, false), + InstanceAddr: gossipRingInstanceAddr(tempo.Spec.HashRing), }, QueryFrontendDiscovery: fmt.Sprintf("%s:%d", naming.Name("query-frontend-discovery", tempo.Name), manifestutils.PortGRPCServer), GlobalRateLimits: fromRateLimitSpecToRateLimitOptions(tempo.Spec.LimitSpec.Global), @@ -256,3 +258,17 @@ func renderTempoQueryTemplate(opts tempoQueryOptions) ([]byte, error) { return cfg, nil } + +func gossipRingInstanceAddr(spec v1alpha1.HashRingSpec) string { + var instanceAddr string + switch spec.MemberList.InstanceAddrType { + case v1alpha1.InstanceAddrPodIP: + instanceAddr = fmt.Sprintf("${%s}", memberlist.GossipInstanceAddrEnvVarName) + case v1alpha1.InstanceAddrDefault: + // Do nothing use tempo defaults + default: + // Do nothing use tempo defaults + } + + return instanceAddr +} diff --git a/internal/manifests/config/build_test.go b/internal/manifests/config/build_test.go index 23623d48d..9e8fc4394 100644 --- a/internal/manifests/config/build_test.go +++ b/internal/manifests/config/build_test.go @@ -2530,3 +2530,126 @@ query_frontend: require.NoError(t, err) require.YAMLEq(t, expCfg, string(cfg)) } + +func TestBuildConfigurationWithPodIPAddressType(t *testing.T) { + expCfg := ` +--- +compactor: + compaction: + block_retention: 48h0m0s + ring: + kvstore: + store: memberlist +distributor: + receivers: + jaeger: + protocols: + thrift_http: + endpoint: 0.0.0.0:14268 + thrift_binary: + endpoint: 0.0.0.0:6832 + thrift_compact: + endpoint: 0.0.0.0:6831 + grpc: + endpoint: 0.0.0.0:14250 + zipkin: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + ring: + kvstore: + store: memberlist + instance_addr: ${HASH_RING_INSTANCE_ADDR} +ingester: + lifecycler: + ring: + kvstore: + store: memberlist + replication_factor: 1 + address: ${HASH_RING_INSTANCE_ADDR} + tokens_file_path: /var/tempo/tokens.json + max_block_duration: 10m +memberlist: + abort_if_cluster_join_fails: false + join_members: + - tempo-test-gossip-ring + advertise_addr: ${HASH_RING_INSTANCE_ADDR} +multitenancy_enabled: false +querier: + max_concurrent_queries: 20 + search: + external_hedge_requests_at: 8s + external_hedge_requests_up_to: 2 + frontend_worker: + frontend_address: "tempo-test-query-frontend-discovery:9095" +server: + grpc_server_max_recv_msg_size: 4194304 + grpc_server_max_send_msg_size: 4194304 + http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s + log_format: logfmt +storage: + trace: + backend: s3 + blocklist_poll: 5m + cache: none + local: + path: /var/tempo/traces + s3: + bucket: tempo + endpoint: "minio:9000" + insecure: true + wal: + path: /var/tempo/wal +usage_report: + reporting_enabled: false +query_frontend: + search: + concurrent_jobs: 2000 + max_duration: 0s +` + cfg, err := buildConfiguration(manifestutils.Params{ + Tempo: v1alpha1.TempoStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Second * 30}, + Storage: v1alpha1.ObjectStorageSpec{ + Secret: v1alpha1.ObjectStorageSecretSpec{ + Type: v1alpha1.ObjectStorageSecretS3, + }, + }, + ReplicationFactor: 1, + Retention: v1alpha1.RetentionSpec{ + Global: v1alpha1.RetentionConfig{ + Traces: metav1.Duration{Duration: 48 * time.Hour}, + }, + }, + HashRing: v1alpha1.HashRingSpec{ + MemberList: v1alpha1.MemberListSpec{ + InstanceAddrType: v1alpha1.InstanceAddrPodIP, + }, + }, + }, + }, + StorageParams: manifestutils.StorageParams{ + S3: &manifestutils.S3{ + LongLived: &manifestutils.S3LongLived{ + Endpoint: "minio:9000", + Bucket: "tempo", + Insecure: true, + }, + }, + }, + TLSProfile: tlsprofile.TLSProfileOptions{ + MinTLSVersion: string(openshiftconfigv1.VersionTLS13), + }, + }) + require.NoError(t, err) + require.YAMLEq(t, expCfg, string(cfg)) +} diff --git a/internal/manifests/config/options.go b/internal/manifests/config/options.go index d34519764..b39a27074 100644 --- a/internal/manifests/config/options.go +++ b/internal/manifests/config/options.go @@ -74,8 +74,9 @@ type tlsOptions struct { } type memberlistOptions struct { - JoinMembers []string - EnableIPv6 bool + JoinMembers []string + EnableIPv6 bool + InstanceAddr string } type receiverTLSOptions struct { diff --git a/internal/manifests/config/tempo-config.yaml b/internal/manifests/config/tempo-config.yaml index cee5abb49..568a5f8b7 100644 --- a/internal/manifests/config/tempo-config.yaml +++ b/internal/manifests/config/tempo-config.yaml @@ -90,12 +90,18 @@ distributor: ring: kvstore: store: memberlist + {{- with .MemberList.InstanceAddr }} + instance_addr: {{ . }} + {{- end }} ingester: lifecycler: ring: kvstore: store: memberlist replication_factor: {{ .ReplicationFactor }} +{{- with .MemberList.InstanceAddr }} + address: {{ . }} +{{- end }} tokens_file_path: /var/tempo/tokens.json {{- if .MemberList.EnableIPv6 }} enable_inet6: true @@ -107,6 +113,9 @@ memberlist: {{- range .MemberList.JoinMembers }} - {{ . }} {{- end }} + {{- with .MemberList.InstanceAddr }} + advertise_addr: {{ . }} + {{- end }} multitenancy_enabled: {{ .Multitenancy }} {{- if or .GlobalRateLimits.IngestionBurstSizeBytes diff --git a/internal/manifests/distributor/distributor.go b/internal/manifests/distributor/distributor.go index 20ed1a101..d99206372 100644 --- a/internal/manifests/distributor/distributor.go +++ b/internal/manifests/distributor/distributor.go @@ -24,6 +24,11 @@ func BuildDistributor(params manifestutils.Params) ([]client.Object, error) { dep := deployment(params) var err error dep.Spec.Template, err = manifestutils.PatchTracingJaegerEnv(params.Tempo, dep.Spec.Template) + + if err := memberlist.ConfigureHashRingEnv(&dep.Spec.Template.Spec, params.Tempo); err != nil { + return nil, err + } + if err != nil { return nil, err } @@ -245,6 +250,7 @@ func deployment(params manifestutils.Params) *v1.Deployment { "-target=distributor", "-config.file=/conf/tempo.yaml", "-log.level=info", + "-config.expand-env=true", }, Ports: containerPorts, ReadinessProbe: manifestutils.TempoReadinessProbe(params.CtrlConfig.Gates.HTTPEncryption), diff --git a/internal/manifests/distributor/distributor_test.go b/internal/manifests/distributor/distributor_test.go index 3f8450591..91e287bbf 100644 --- a/internal/manifests/distributor/distributor_test.go +++ b/internal/manifests/distributor/distributor_test.go @@ -28,11 +28,13 @@ func TestBuildDistributor(t *testing.T) { expectedServiceAnnotations map[string]string expectCABundleConfigMap bool enableServingCertsService bool + instanceAddrType v1alpha1.InstanceAddrType expectedContainerPorts []corev1.ContainerPort expectedServicePorts []corev1.ServicePort expectedResources corev1.ResourceRequirements expectedVolumes []corev1.Volume expectedVolumeMounts []corev1.VolumeMount + expectedContainerEnvVars []corev1.EnvVar }{ { name: "Gateway disabled", @@ -174,6 +176,7 @@ func TestBuildDistributor(t *testing.T) { MountPath: manifestutils.TmpTempoStoragePath, }, }, + expectedContainerEnvVars: []corev1.EnvVar{}, }, { name: "Receiver TLS enable", @@ -348,6 +351,7 @@ func TestBuildDistributor(t *testing.T) { ReadOnly: true, }, }, + expectedContainerEnvVars: []corev1.EnvVar{}, }, { name: "Receiver TLS enable with ServingCertsService feature enabled", @@ -523,6 +527,7 @@ func TestBuildDistributor(t *testing.T) { ReadOnly: true, }, }, + expectedContainerEnvVars: []corev1.EnvVar{}, }, { name: "Receiver TLS enable with ServingCertsService feature enabled no custom certs", @@ -700,6 +705,7 @@ func TestBuildDistributor(t *testing.T) { ReadOnly: true, }, }, + expectedContainerEnvVars: []corev1.EnvVar{}, }, { name: "Gateway enable", @@ -786,11 +792,171 @@ func TestBuildDistributor(t *testing.T) { MountPath: manifestutils.TmpTempoStoragePath, }, }, + expectedContainerEnvVars: []corev1.EnvVar{}, + }, + { + name: "set InstanceAddrType to PodIP", + enableGateway: false, + expectedObjects: 2, + expectedContainerPorts: []corev1.ContainerPort{ + { + Name: manifestutils.PortOtlpHttpName, + ContainerPort: manifestutils.PortOtlpHttp, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.OtlpGrpcPortName, + ContainerPort: manifestutils.PortOtlpGrpcServer, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.HttpPortName, + ContainerPort: manifestutils.PortHTTPServer, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.HttpMemberlistPortName, + ContainerPort: manifestutils.PortMemberlist, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.PortJaegerThriftHTTPName, + ContainerPort: manifestutils.PortJaegerThriftHTTP, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.PortJaegerThriftCompactName, + ContainerPort: manifestutils.PortJaegerThriftCompact, + Protocol: corev1.ProtocolUDP, + }, + { + Name: manifestutils.PortJaegerThriftBinaryName, + ContainerPort: manifestutils.PortJaegerThriftBinary, + Protocol: corev1.ProtocolUDP, + }, + { + Name: manifestutils.PortJaegerGrpcName, + ContainerPort: manifestutils.PortJaegerGrpc, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.PortZipkinName, + ContainerPort: manifestutils.PortZipkin, + Protocol: corev1.ProtocolTCP, + }, + }, + expectedServicePorts: []corev1.ServicePort{ + { + Name: manifestutils.PortOtlpHttpName, + Port: manifestutils.PortOtlpHttp, + TargetPort: intstr.FromString(manifestutils.PortOtlpHttpName), + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.OtlpGrpcPortName, + Protocol: corev1.ProtocolTCP, + Port: manifestutils.PortOtlpGrpcServer, + TargetPort: intstr.FromString(manifestutils.OtlpGrpcPortName), + }, + { + Name: manifestutils.HttpPortName, + Protocol: corev1.ProtocolTCP, + Port: manifestutils.PortHTTPServer, + TargetPort: intstr.FromString(manifestutils.HttpPortName), + }, + { + Name: manifestutils.PortJaegerThriftHTTPName, + Port: manifestutils.PortJaegerThriftHTTP, + TargetPort: intstr.FromString(manifestutils.PortJaegerThriftHTTPName), + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.PortJaegerThriftCompactName, + Port: manifestutils.PortJaegerThriftCompact, + TargetPort: intstr.FromString(manifestutils.PortJaegerThriftCompactName), + Protocol: corev1.ProtocolUDP, + }, + { + Name: manifestutils.PortJaegerThriftBinaryName, + Port: manifestutils.PortJaegerThriftBinary, + TargetPort: intstr.FromString(manifestutils.PortJaegerThriftBinaryName), + Protocol: corev1.ProtocolUDP, + }, + { + Name: manifestutils.PortJaegerGrpcName, + Port: manifestutils.PortJaegerGrpc, + TargetPort: intstr.FromString(manifestutils.PortJaegerGrpcName), + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.PortZipkinName, + Port: manifestutils.PortZipkin, + TargetPort: intstr.FromString(manifestutils.PortZipkinName), + Protocol: corev1.ProtocolTCP, + }, + }, + expectedResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(270, resource.BinarySI), + corev1.ResourceMemory: *resource.NewQuantity(257698032, resource.BinarySI), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(81, resource.BinarySI), + corev1.ResourceMemory: *resource.NewQuantity(77309416, resource.BinarySI), + }, + }, + expectedVolumes: []corev1.Volume{ + { + Name: manifestutils.ConfigVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tempo-test", + }, + }, + }, + }, + { + Name: manifestutils.TmpStorageVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + expectedVolumeMounts: []corev1.VolumeMount{ + { + Name: manifestutils.ConfigVolumeName, + MountPath: "/conf", + ReadOnly: true, + }, + { + Name: manifestutils.TmpStorageVolumeName, + MountPath: manifestutils.TmpTempoStoragePath, + }, + }, + expectedContainerEnvVars: []corev1.EnvVar{ + { + Name: "HASH_RING_INSTANCE_ADDR", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "status.podIP", + }, + }, + }, + }, + instanceAddrType: v1alpha1.InstanceAddrPodIP, }, } for _, ts := range tests { t.Run(ts.name, func(t *testing.T) { + + instanceAddrType := v1alpha1.InstanceAddrDefault + if ts.instanceAddrType != "" { + instanceAddrType = ts.instanceAddrType + } + objects, err := BuildDistributor(manifestutils.Params{ Tempo: v1alpha1.TempoStack{ ObjectMeta: metav1.ObjectMeta{ @@ -827,6 +993,11 @@ func TestBuildDistributor(t *testing.T) { }, }, }, + HashRing: v1alpha1.HashRingSpec{ + MemberList: v1alpha1.MemberListSpec{ + InstanceAddrType: instanceAddrType, + }, + }, }, }, CtrlConfig: configv1alpha1.ProjectConfig{ @@ -879,8 +1050,9 @@ func TestBuildDistributor(t *testing.T) { "-target=distributor", "-config.file=/conf/tempo.yaml", "-log.level=info", + "-config.expand-env=true", }, - Env: []corev1.EnvVar{}, + Env: ts.expectedContainerEnvVars, VolumeMounts: ts.expectedVolumeMounts, Ports: ts.expectedContainerPorts, ReadinessProbe: &corev1.Probe{ diff --git a/internal/manifests/ingester/ingester.go b/internal/manifests/ingester/ingester.go index 51f9d0325..00adec471 100644 --- a/internal/manifests/ingester/ingester.go +++ b/internal/manifests/ingester/ingester.go @@ -27,6 +27,10 @@ func BuildIngester(params manifestutils.Params) ([]client.Object, error) { return nil, err } + if err := memberlist.ConfigureHashRingEnv(&ss.Spec.Template.Spec, params.Tempo); err != nil { + return nil, err + } + gates := params.CtrlConfig.Gates tempo := params.Tempo @@ -94,6 +98,7 @@ func statefulSet(params manifestutils.Params) (*v1.StatefulSet, error) { "-target=ingester", "-config.file=/conf/tempo.yaml", "-log.level=info", + "-config.expand-env=true", }, VolumeMounts: []corev1.VolumeMount{ { diff --git a/internal/manifests/ingester/ingester_test.go b/internal/manifests/ingester/ingester_test.go index 4ea65df1e..68dd34efc 100644 --- a/internal/manifests/ingester/ingester_test.go +++ b/internal/manifests/ingester/ingester_test.go @@ -18,219 +18,283 @@ import ( ) func TestBuildIngester(t *testing.T) { - storageClassName := "default" - filesystem := corev1.PersistentVolumeFilesystem - objects, err := BuildIngester(manifestutils.Params{Tempo: v1alpha1.TempoStack{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "project1", - }, - Spec: v1alpha1.TempoStackSpec{ - Images: configv1alpha1.ImagesSpec{ - Tempo: "docker.io/grafana/tempo:1.5.0", - }, - ServiceAccount: "tempo-test-serviceaccount", - Storage: v1alpha1.ObjectStorageSpec{ - Secret: v1alpha1.ObjectStorageSecretSpec{ - Name: "test-storage-secret", - Type: "s3", + + tests := []struct { + name string + instanceAddrType v1alpha1.InstanceAddrType + expectedContainerEnvVars []corev1.EnvVar + }{ + { + name: "default", + expectedContainerEnvVars: []corev1.EnvVar{ + { + Name: "S3_SECRET_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "access_key_secret", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-storage-secret", + }, + }, + }, + }, { + Name: "S3_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "access_key_id", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-storage-secret", + }, + }, + }, }, }, - StorageSize: resource.MustParse("10Gi"), - StorageClassName: &storageClassName, - Template: v1alpha1.TempoTemplateSpec{ - Ingester: v1alpha1.TempoComponentSpec{ - NodeSelector: map[string]string{"a": "b"}, - Tolerations: []corev1.Toleration{ - { - Key: "c", + }, + { + name: "set InstanceAddrType to PodIP", + instanceAddrType: v1alpha1.InstanceAddrPodIP, + expectedContainerEnvVars: []corev1.EnvVar{ + { + Name: "S3_SECRET_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "access_key_secret", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-storage-secret", + }, + }, + }, + }, { + Name: "S3_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "access_key_id", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-storage-secret", + }, }, }, }, - }, - Resources: v1alpha1.Resources{ - Total: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1000m"), - corev1.ResourceMemory: resource.MustParse("2Gi"), + { + Name: "HASH_RING_INSTANCE_ADDR", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "status.podIP", + }, }, }, }, }, - }}) - require.NoError(t, err) + } + for _, ts := range tests { + t.Run(ts.name, func(t *testing.T) { + storageClassName := "default" + filesystem := corev1.PersistentVolumeFilesystem - labels := manifestutils.ComponentLabels("ingester", "test") - annotations := manifestutils.CommonAnnotations("") - assert.Equal(t, 2, len(objects)) - assert.Equal(t, &v1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tempo-test-ingester", - Namespace: "project1", - Labels: labels, - }, - Spec: v1.StatefulSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - PodManagementPolicy: v1.ParallelPodManagement, - Template: corev1.PodTemplateSpec{ + instanceAddrType := v1alpha1.InstanceAddrDefault + if ts.instanceAddrType != "" { + instanceAddrType = ts.instanceAddrType + } + objects, err := BuildIngester(manifestutils.Params{Tempo: v1alpha1.TempoStack{ ObjectMeta: metav1.ObjectMeta{ - Labels: k8slabels.Merge(labels, map[string]string{"tempo-gossip-member": "true"}), - Annotations: annotations, + Name: "test", + Namespace: "project1", }, - Spec: corev1.PodSpec{ - ServiceAccountName: "tempo-test-serviceaccount", - NodeSelector: map[string]string{"a": "b"}, - Tolerations: []corev1.Toleration{ - { - Key: "c", + Spec: v1alpha1.TempoStackSpec{ + Images: configv1alpha1.ImagesSpec{ + Tempo: "docker.io/grafana/tempo:1.5.0", + }, + ServiceAccount: "tempo-test-serviceaccount", + Storage: v1alpha1.ObjectStorageSpec{ + Secret: v1alpha1.ObjectStorageSecretSpec{ + Name: "test-storage-secret", + Type: "s3", }, }, - Affinity: manifestutils.DefaultAffinity(labels), - Containers: []corev1.Container{ - { - Name: "tempo", - Image: "docker.io/grafana/tempo:1.5.0", - Args: []string{"-target=ingester", - "-config.file=/conf/tempo.yaml", - "-log.level=info", - "--storage.trace.s3.secret_key=$(S3_SECRET_KEY)", - "--storage.trace.s3.access_key=$(S3_ACCESS_KEY)", + StorageSize: resource.MustParse("10Gi"), + StorageClassName: &storageClassName, + Template: v1alpha1.TempoTemplateSpec{ + Ingester: v1alpha1.TempoComponentSpec{ + NodeSelector: map[string]string{"a": "b"}, + Tolerations: []corev1.Toleration{ + { + Key: "c", + }, }, - Env: []corev1.EnvVar{ + }, + }, + Resources: v1alpha1.Resources{ + Total: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1000m"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + HashRing: v1alpha1.HashRingSpec{ + MemberList: v1alpha1.MemberListSpec{ + InstanceAddrType: instanceAddrType, + }, + }, + }, + }}) + require.NoError(t, err) + + labels := manifestutils.ComponentLabels("ingester", "test") + annotations := manifestutils.CommonAnnotations("") + assert.Equal(t, 2, len(objects)) + + assert.Equal(t, &v1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tempo-test-ingester", + Namespace: "project1", + Labels: labels, + }, + Spec: v1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + PodManagementPolicy: v1.ParallelPodManagement, + Template: corev1.PodTemplateSpec{ + + ObjectMeta: metav1.ObjectMeta{ + Labels: k8slabels.Merge(labels, map[string]string{"tempo-gossip-member": "true"}), + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "tempo-test-serviceaccount", + NodeSelector: map[string]string{"a": "b"}, + Tolerations: []corev1.Toleration{ { - Name: "S3_SECRET_KEY", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "access_key_secret", - LocalObjectReference: corev1.LocalObjectReference{ - Name: "test-storage-secret", - }, + Key: "c", + }, + }, + Affinity: manifestutils.DefaultAffinity(labels), + Containers: []corev1.Container{ + { + Name: "tempo", + Image: "docker.io/grafana/tempo:1.5.0", + Args: []string{"-target=ingester", + "-config.file=/conf/tempo.yaml", + "-log.level=info", + "-config.expand-env=true", + "--storage.trace.s3.secret_key=$(S3_SECRET_KEY)", + "--storage.trace.s3.access_key=$(S3_ACCESS_KEY)", + }, + Env: ts.expectedContainerEnvVars, + VolumeMounts: []corev1.VolumeMount{ + { + Name: manifestutils.ConfigVolumeName, + MountPath: "/conf", + ReadOnly: true, + }, + { + Name: dataVolumeName, + MountPath: "/var/tempo", }, }, - }, { - Name: "S3_ACCESS_KEY", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "access_key_id", - LocalObjectReference: corev1.LocalObjectReference{ - Name: "test-storage-secret", + Ports: []corev1.ContainerPort{ + { + Name: manifestutils.HttpMemberlistPortName, + ContainerPort: manifestutils.PortMemberlist, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.HttpPortName, + ContainerPort: manifestutils.PortHTTPServer, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.GrpcPortName, + ContainerPort: manifestutils.PortGRPCServer, + Protocol: corev1.ProtocolTCP, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + Path: manifestutils.TempoReadinessPath, + Port: intstr.FromString(manifestutils.HttpPortName), }, }, + InitialDelaySeconds: 15, + TimeoutSeconds: 1, }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(380, resource.BinarySI), + corev1.ResourceMemory: *resource.NewQuantity(1073741824, resource.BinarySI), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(114, resource.BinarySI), + corev1.ResourceMemory: *resource.NewQuantity(322122560, resource.BinarySI), + }, + }, + SecurityContext: manifestutils.TempoContainerSecurityContext(), }, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: manifestutils.ConfigVolumeName, - MountPath: "/conf", - ReadOnly: true, - }, - { - Name: dataVolumeName, - MountPath: "/var/tempo", - }, - }, - Ports: []corev1.ContainerPort{ - { - Name: manifestutils.HttpMemberlistPortName, - ContainerPort: manifestutils.PortMemberlist, - Protocol: corev1.ProtocolTCP, - }, - { - Name: manifestutils.HttpPortName, - ContainerPort: manifestutils.PortHTTPServer, - Protocol: corev1.ProtocolTCP, - }, + Volumes: []corev1.Volume{ { - Name: manifestutils.GrpcPortName, - ContainerPort: manifestutils.PortGRPCServer, - Protocol: corev1.ProtocolTCP, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - Path: manifestutils.TempoReadinessPath, - Port: intstr.FromString(manifestutils.HttpPortName), + Name: manifestutils.ConfigVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tempo-test", + }, + }, }, }, - InitialDelaySeconds: 15, - TimeoutSeconds: 1, - }, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: *resource.NewMilliQuantity(380, resource.BinarySI), - corev1.ResourceMemory: *resource.NewQuantity(1073741824, resource.BinarySI), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: *resource.NewMilliQuantity(114, resource.BinarySI), - corev1.ResourceMemory: *resource.NewQuantity(322122560, resource.BinarySI), - }, }, - SecurityContext: manifestutils.TempoContainerSecurityContext(), }, }, - Volumes: []corev1.Volume{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ { - Name: manifestutils.ConfigVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tempo-test", + ObjectMeta: metav1.ObjectMeta{ + Name: dataVolumeName, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), }, }, + StorageClassName: &storageClassName, + VolumeMode: &filesystem, }, }, }, }, - }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: dataVolumeName, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("10Gi"), - }, + }, objects[0]) + assert.Equal(t, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tempo-test-ingester", + Namespace: "project1", + Labels: labels, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: manifestutils.HttpPortName, + Protocol: corev1.ProtocolTCP, + Port: manifestutils.PortHTTPServer, + TargetPort: intstr.FromString(manifestutils.HttpPortName), + }, + { + Name: manifestutils.GrpcPortName, + Protocol: corev1.ProtocolTCP, + Port: manifestutils.PortGRPCServer, + TargetPort: intstr.FromString(manifestutils.GrpcPortName), }, - StorageClassName: &storageClassName, - VolumeMode: &filesystem, }, + Selector: labels, }, - }, - }, - }, objects[0]) - assert.Equal(t, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tempo-test-ingester", - Namespace: "project1", - Labels: labels, - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: manifestutils.HttpPortName, - Protocol: corev1.ProtocolTCP, - Port: manifestutils.PortHTTPServer, - TargetPort: intstr.FromString(manifestutils.HttpPortName), - }, - { - Name: manifestutils.GrpcPortName, - Protocol: corev1.ProtocolTCP, - Port: manifestutils.PortGRPCServer, - TargetPort: intstr.FromString(manifestutils.GrpcPortName), - }, - }, - Selector: labels, - }, - }, objects[1]) + }, objects[1]) + }) + } } func TestOverrideResources(t *testing.T) { diff --git a/internal/manifests/memberlist/gossip.go b/internal/manifests/memberlist/gossip.go index f898359b3..e8f1a74ae 100644 --- a/internal/manifests/memberlist/gossip.go +++ b/internal/manifests/memberlist/gossip.go @@ -1,6 +1,7 @@ package memberlist import ( + "github.com/imdario/mergo" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8slabels "k8s.io/apimachinery/pkg/labels" @@ -13,6 +14,8 @@ import ( const ( componentName = "gossip-ring" + // GossipInstanceAddrEnvVarName is the name of the hash ring instance address env var. + GossipInstanceAddrEnvVarName = "HASH_RING_INSTANCE_ADDR" ) var ( @@ -46,3 +49,36 @@ func BuildGossip(tempo v1alpha1.TempoStack) *corev1.Service { }, } } + +// ConfigureHashRingEnv adds an environment variable with the podIP if instanceAddressType = podIP. +func ConfigureHashRingEnv(p *corev1.PodSpec, tempo v1alpha1.TempoStack) error { + + memberList := tempo.Spec.HashRing.MemberList + enableIPV6 := memberList.EnableIPv6 != nil && *memberList.EnableIPv6 + + if !enableIPV6 && memberList.InstanceAddrType != v1alpha1.InstanceAddrPodIP { + return nil + } + + src := corev1.Container{ + Env: []corev1.EnvVar{ + { + Name: GossipInstanceAddrEnvVarName, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "status.podIP", + }, + }, + }, + }, + } + + for i := range p.Containers { + if err := mergo.Merge(&p.Containers[i], src, mergo.WithAppendSlice); err != nil { + return err + } + } + + return nil +} diff --git a/internal/manifests/memberlist/gossip_test.go b/internal/manifests/memberlist/gossip_test.go index 703669efb..4e2ec2ed2 100644 --- a/internal/manifests/memberlist/gossip_test.go +++ b/internal/manifests/memberlist/gossip_test.go @@ -45,3 +45,83 @@ func TestBuildGossip(t *testing.T) { }, }, service) } + +func TestConfigureHashRingEnv(t *testing.T) { + + tests := []struct { + name string + instanceAddrType v1alpha1.InstanceAddrType + expectedContainerEnvVars []corev1.EnvVar + }{ + { + name: "with podIP", + instanceAddrType: v1alpha1.InstanceAddrPodIP, + expectedContainerEnvVars: []corev1.EnvVar{ + { + Name: "HASH_RING_INSTANCE_ADDR", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "status.podIP", + }, + }, + }, + }, + }, + + { + name: "with podIP", + instanceAddrType: v1alpha1.InstanceAddrDefault, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + podSpec := &corev1.PodSpec{ + ServiceAccountName: "tempo-test-serviceaccount", + Containers: []corev1.Container{ + { + Name: "tempo", + Image: "docker.io/grafana/tempo:1.5.0", + Args: []string{ + "-target=compactor", + "-config.file=/conf/tempo.yaml", + "-log.level=info", + "-config.expand-env=true", + }, + }, + }, + } + + tempoStackSpec := v1alpha1.TempoStack{ + Spec: v1alpha1.TempoStackSpec{ + HashRing: v1alpha1.HashRingSpec{ + MemberList: v1alpha1.MemberListSpec{ + InstanceAddrType: tc.instanceAddrType, + }, + }, + }, + } + + err := ConfigureHashRingEnv(podSpec, tempoStackSpec) + assert.NoError(t, err) + + assert.Equal(t, podSpec, &corev1.PodSpec{ + ServiceAccountName: "tempo-test-serviceaccount", + Containers: []corev1.Container{ + { + Name: "tempo", + Image: "docker.io/grafana/tempo:1.5.0", + Args: []string{ + "-target=compactor", + "-config.file=/conf/tempo.yaml", + "-log.level=info", + "-config.expand-env=true", + }, + Env: tc.expectedContainerEnvVars, + }, + }, + }) + }) + } +} diff --git a/internal/manifests/querier/querier.go b/internal/manifests/querier/querier.go index a3b36d1f1..26213384b 100644 --- a/internal/manifests/querier/querier.go +++ b/internal/manifests/querier/querier.go @@ -22,6 +22,10 @@ func BuildQuerier(params manifestutils.Params) ([]client.Object, error) { return nil, err } + if err := memberlist.ConfigureHashRingEnv(&d.Spec.Template.Spec, params.Tempo); err != nil { + return nil, err + } + d.Spec.Template, err = manifestutils.PatchTracingJaegerEnv(params.Tempo, d.Spec.Template) if err != nil { return nil, err @@ -94,6 +98,7 @@ func deployment(params manifestutils.Params) (*v1.Deployment, error) { "-target=querier", "-config.file=/conf/tempo.yaml", "-log.level=info", + "-config.expand-env=true", }, Ports: []corev1.ContainerPort{ { diff --git a/internal/manifests/querier/querier_test.go b/internal/manifests/querier/querier_test.go index 8bd738f7b..57dfcabdf 100644 --- a/internal/manifests/querier/querier_test.go +++ b/internal/manifests/querier/querier_test.go @@ -124,6 +124,7 @@ func TestBuildQuerier(t *testing.T) { "-target=querier", "-config.file=/conf/tempo.yaml", "-log.level=info", + "-config.expand-env=true", }, VolumeMounts: []corev1.VolumeMount{ { diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go index 97b8d0d42..d79b53ee7 100644 --- a/internal/manifests/queryfrontend/query_frontend.go +++ b/internal/manifests/queryfrontend/query_frontend.go @@ -47,6 +47,10 @@ func BuildQueryFrontend(params manifestutils.Params) ([]client.Object, error) { return nil, err } + if err := memberlist.ConfigureHashRingEnv(&d.Spec.Template.Spec, params.Tempo); err != nil { + return nil, err + } + d.Spec.Template, err = manifestutils.PatchTracingJaegerEnv(params.Tempo, d.Spec.Template) if err != nil { return nil, err @@ -197,6 +201,7 @@ func deployment(params manifestutils.Params) (*appsv1.Deployment, error) { "-config.file=/conf/tempo-query-frontend.yaml", "-mem-ballast-size-mbs=1024", "-log.level=info", + "-config.expand-env=true", }, Ports: []corev1.ContainerPort{ { diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go index 9f54ea5cb..7b1316df4 100644 --- a/internal/manifests/queryfrontend/query_frontend_test.go +++ b/internal/manifests/queryfrontend/query_frontend_test.go @@ -154,6 +154,7 @@ func getExpectedDeployment(withJaeger bool) *v1.Deployment { "-config.file=/conf/tempo-query-frontend.yaml", "-mem-ballast-size-mbs=1024", "-log.level=info", + "-config.expand-env=true", }, Ports: []corev1.ContainerPort{ {