fix: Make the otel trace endpoint configurable (#326)

Co-authored-by: David Jackson <[email protected]>
wandb · Feb 3, 2025 · 28b919d · 28b919d
1 parent 15c1c31
commit 28b919d
Show file tree

Hide file tree

Showing 9 changed files with 27 additions and 9 deletions.
diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.24.8
+version: 0.24.9
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 

diff --git a/charts/operator-wandb/charts/app/templates/deployment.yaml b/charts/operator-wandb/charts/app/templates/deployment.yaml
@@ -260,7 +260,7 @@ spec:
               value: {{ toJson .Values.global.banners | quote }}
             {{- if ne .Values.traceRatio 0.0 }}
             - name: GORILLA_TRACER
-              value: "otlp+grpc://{{ .Release.Name }}-otel-daemonset:4317?trace_ratio={{ .Values.traceRatio }}"
+              value: {{ include "wandb.otelTracesEndpoint" . | quote }}
             {{- end }}
             - name: OVERFLOW_BUCKET_ADDR
               value: {{ (include "wandb.bucket" . | fromYaml).url | quote }}

diff --git a/charts/operator-wandb/charts/executor/templates/deployment.yaml b/charts/operator-wandb/charts/executor/templates/deployment.yaml
@@ -83,7 +83,7 @@ spec:
               value: "{{ (include "wandb.bucket" . | fromYaml).url }}"
             {{- if ne .Values.traceRatio 0.0 }}
             - name: GORILLA_TRACER
-              value: "otlp+grpc://{{ .Release.Name }}-otel-daemonset:4317?trace_ratio={{ .Values.traceRatio }}"
+              value: {{ include "wandb.otelTracesEndpoint" . | quote }}
             {{- end }}
             - name: ONLY_SERVICE
               value: gorilla-executor

diff --git a/charts/operator-wandb/charts/filestream/templates/deployment.yaml b/charts/operator-wandb/charts/filestream/templates/deployment.yaml
@@ -107,7 +107,7 @@ spec:
             {{- end }}
             {{- if ne .Values.traceRatio 0.0 }}
             - name: GORILLA_TRACER
-              value: "otlp+grpc://{{ .Release.Name }}-otel-daemonset:4317?trace_ratio={{ .Values.traceRatio }}"
+              value: {{ include "wandb.otelTracesEndpoint" . | quote }}
             {{- end }}
             - name: ONLY_SERVICE
               value: gorilla-filestream

diff --git a/charts/operator-wandb/charts/flat-run-fields-updater/templates/deployment.yaml b/charts/operator-wandb/charts/flat-run-fields-updater/templates/deployment.yaml
@@ -111,7 +111,7 @@ spec:
             {{- end }}
             {{- if ne .Values.traceRatio 0.0 }}
             - name: GORILLA_TRACER
-              value: "otlp+grpc://{{ .Release.Name }}-otel-daemonset:4317?trace_ratio={{ .Values.traceRatio }}"
+              value: {{ include "wandb.otelTracesEndpoint" . | quote }}
             {{- end }}
             - name: ONLY_SERVICE
               value: gorilla-flat-run-fields-updater

diff --git a/charts/operator-wandb/charts/parquet/templates/deployment.yaml b/charts/operator-wandb/charts/parquet/templates/deployment.yaml
@@ -108,7 +108,7 @@ spec:
               value: "{{ (include "wandb.bucket" . | fromYaml).url }}"
             {{- if ne .Values.traceRatio 0.0 }}
             - name: GORILLA_TRACER
-              value: "otlp+grpc://{{ .Release.Name }}-otel-daemonset:4317?trace_ratio={{ .Values.traceRatio }}"
+              value: {{ include "wandb.otelTracesEndpoint" . | quote }}
             {{- end }}
             - name: ONLY_SERVICE
               value: gorilla-parquet

diff --git a/charts/operator-wandb/templates/_helpers.tpl b/charts/operator-wandb/templates/_helpers.tpl
@@ -60,3 +60,14 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Return the endpoint to send otel traces to (only works when called within a subchart or traceRatio will not be present)
+*/}}
+{{- define "wandb.otelTracesEndpoint" -}}
+{{- if .Values.global.otel.traces.host -}}
+otlp+{{ .Values.global.otel.traces.proto }}://{{ .Values.global.otel.traces.host }}:{{ .Values.global.otel.traces.port }}?trace_ratio={{ default 0.0 .Values.traceRatio }}
+{{- else -}}
+otlp+{{ .Values.global.otel.traces.proto }}://{{ .Release.Name }}-otel-daemonset:{{ .Values.global.otel.traces.port }}?trace_ratio={{ default 0.0 .Values.traceRatio }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/operator-wandb/templates/gorilla.yaml b/charts/operator-wandb/templates/gorilla.yaml
@@ -41,9 +41,6 @@ data:
   GORILLA_STATSD_HOST: "0.0.0.0"
     {{- end }}
   {{- end }}
-  {{- if ne .Values.app.traceRatio 0.0 }}
-  GORILLA_TRACER: "otlp+grpc://{{ .Release.Name }}-otel-daemonset:4317?trace_ratio={{ .Values.app.traceRatio }}"
-  {{- end }}
   WEAVE_SERVICE: "{{ .Release.Name }}-weave:9994"
   PARQUET_HOST: "http://{{ .Release.Name }}-parquet:8087"
   PARQUET_ENABLED: "true"

diff --git a/charts/operator-wandb/values.yaml b/charts/operator-wandb/values.yaml
@@ -71,6 +71,12 @@ global:
     secret: ""
     clientId: ""
 
+  otel:
+    traces:
+      host: ""       # defaults to the otel-daemonset service
+      port: 4317     # grpc default is 4317, http default is 4318
+      proto: "grpc"  # grpc, http
+
   clickhouse:
     install: false
     host: ""
@@ -500,6 +506,8 @@ glue:
       value: '{{ include "wandb.mysql" . | trim }}'
     GORILLA_RUN_STORE:
       value: '{{ include "wandb.mysql" . | trim }}'
+    GORILLA_TRACER:
+      value: '{{ include "wandb.otelTracesEndpoint" . | trim }}'
     GORILLA_FILE_STREAM_STORE_ADDRESS:
       value: '{{ include "wandb.fileStreamStoreProducer" . }}'
     GORILLA_HISTORY_STORE:
@@ -671,6 +679,8 @@ api:
       value: '{{ include "wandb.mysql" . | trim }}'
     GORILLA_RUN_STORE:
       value: '{{ include "wandb.mysql" . | trim }}'
+    GORILLA_TRACER:
+      value: '{{ include "wandb.otelTracesEndpoint" . | trim }}'
     GORILLA_FILE_STREAM_STORE_ADDRESS:
       value: '{{ include "wandb.fileStreamStoreProducer" . }}'
     GORILLA_HISTORY_STORE: