From 9b6c27346f774a098b799be57ea907953064aa8d Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 14:22:05 -0800
Subject: [PATCH 01/27] fix: Add tests for operator-wandb

---
 .github/workflows/lint.yaml                   | 43 ++++++++++++
 .github/workflows/test-operator-wandb.yaml    | 70 +++++++++++++++++++
 .github/workflows/test-operator.yaml          | 54 ++++++++++++++
 .../{lint-test.yaml => test-wandb.yaml}       | 11 ++-
 charts/operator-wandb/Chart.yaml              |  2 +-
 .../templates/tests/test-connection.yaml      | 23 ++++++
 test-configs/minio/default.yaml               | 51 ++++++++++++++
 test-configs/operator-wandb/default.yaml      | 21 ++++++
 8 files changed, 268 insertions(+), 7 deletions(-)
 create mode 100644 .github/workflows/lint.yaml
 create mode 100644 .github/workflows/test-operator-wandb.yaml
 create mode 100644 .github/workflows/test-operator.yaml
 rename .github/workflows/{lint-test.yaml => test-wandb.yaml} (91%)
 create mode 100644 charts/operator-wandb/templates/tests/test-connection.yaml
 create mode 100644 test-configs/minio/default.yaml
 create mode 100644 test-configs/operator-wandb/default.yaml

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
new file mode 100644
index 00000000..8b571b3c
--- /dev/null
+++ b/.github/workflows/lint.yaml
@@ -0,0 +1,43 @@
+name: Lint Charts
+
+on:
+  pull_request:
+    paths: charts
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    environment: Helm Charts
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v3.4
+        with:
+          version: v3.16.3
+
+      # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
+      # yamllint (https://github.com/adrienverge/yamllint) which require Python
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.6.1
+        with:
+          version: v3.11.0
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config ct.yaml)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Run chart-testing  (lint)
+        run: ct lint --config ct.yaml
\ No newline at end of file
diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
new file mode 100644
index 00000000..1b594090
--- /dev/null
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -0,0 +1,70 @@
+name: Test operator-wandb Chart
+
+on:
+  pull_request:
+    paths: charts/operator-wandb
+
+jobs:
+  test:
+    name: Test Chart
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
+        configuration: ["default"]
+    environment: Helm Charts
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v3.4
+        with:
+          version: v3.16.3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.6.1
+        with:
+          version: v3.11.0
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config ct.yaml)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Create kind cluster
+        uses: helm/kind-action@v1.12.0
+        with:
+          version: v0.26.0
+          cluster_name: chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}
+          node_image: kindest/node:${{ matrix.k8s-version }}
+        if: env.ACT || steps.list-changed.outputs.changed == 'true'
+
+      - name: Install Minio
+        run: kubectl --context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} apply -f test-configs/minio/default.yaml
+        if: env.ACT || steps.list-changed.outputs.changed == 'true'
+
+      - name: Wait for Minio to be Ready
+        run: kubectl --context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} -n minio wait --for=condition=Ready pod/minio --timeout=300s
+        if: env.ACT || steps.list-changed.outputs.changed == 'true'
+
+      - name: Run chart-testing (install)
+        env:
+          LICENSE: ${{ secrets.LICENSE }}
+        if: steps.list-changed.outputs.changed == 'true'
+        run: |
+          ct install \
+          --charts ./charts/operator-wandb \
+          --config ct.yaml \
+          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}' \
+          --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE'
\ No newline at end of file
diff --git a/.github/workflows/test-operator.yaml b/.github/workflows/test-operator.yaml
new file mode 100644
index 00000000..e89b4665
--- /dev/null
+++ b/.github/workflows/test-operator.yaml
@@ -0,0 +1,54 @@
+# TODO(dpanzella): Uncomment this file when the ready for automated tests of the operator chart
+
+#name: Test operator Chart
+#
+#on:
+#  pull_request:
+#    paths: charts/operator
+#
+#jobs:
+#  test:
+#    runs-on: ubuntu-latest
+#    environment: Helm Charts
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v2
+#        with:
+#          fetch-depth: 0
+#
+#      - name: Set up Helm
+#        uses: azure/setup-helm@v3.4
+#        with:
+#          version: v3.16.3
+#
+#      # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
+#      # yamllint (https://github.com/adrienverge/yamllint) which require Python
+#      - name: Set up Python
+#        uses: actions/setup-python@v4
+#        with:
+#          python-version: "3.10"
+#
+#      - name: Set up chart-testing
+#        uses: helm/chart-testing-action@v2.6.1
+#        with:
+#          version: v3.11.0
+#
+#      - name: Run chart-testing (list-changed)
+#        id: list-changed
+#        run: |
+#          changed=$(ct list-changed --config ct.yaml)
+#          if [[ -n "$changed" ]]; then
+#            echo "::set-output name=changed::true"
+#          fi
+#
+#      - name: Create kind cluster
+#        uses: helm/kind-action@v1.8.0
+#        with:
+#          version: v0.20.0
+#        if: env.ACT || steps.list-changed.outputs.changed == 'true'
+#
+#      - name: Run chart-testing (install)
+#        env:
+#          LICENSE: ${{ secrets.LICENSE }}
+#        if: steps.list-changed.outputs.changed == 'true'
+#        run: ct install --charts ./charts/operator --config ct.yaml --helm-extra-set-args --set=license=$LICENSE
diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/test-wandb.yaml
similarity index 91%
rename from .github/workflows/lint-test.yaml
rename to .github/workflows/test-wandb.yaml
index ff32a20b..fd02783b 100644
--- a/.github/workflows/lint-test.yaml
+++ b/.github/workflows/test-wandb.yaml
@@ -1,9 +1,11 @@
-name: Lint and Test Charts
+name: Test wandb Chart
 
-on: pull_request
+on:
+  pull_request:
+    paths: charts/wandb
 
 jobs:
-  lint-test:
+  test:
     runs-on: ubuntu-latest
     environment: Helm Charts
     steps:
@@ -37,9 +39,6 @@ jobs:
             echo "::set-output name=changed::true"
           fi
 
-      - name: Run chart-testing  (lint)
-        run: ct lint --config ct.yaml
-
       - name: Create kind cluster
         uses: helm/kind-action@v1.8.0
         with:
diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml
index bcdac359..c53990c9 100644
--- a/charts/operator-wandb/Chart.yaml
+++ b/charts/operator-wandb/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.24.8
+version: 0.24.9
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 
diff --git a/charts/operator-wandb/templates/tests/test-connection.yaml b/charts/operator-wandb/templates/tests/test-connection.yaml
new file mode 100644
index 00000000..8d1345cd
--- /dev/null
+++ b/charts/operator-wandb/templates/tests/test-connection.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "wandb.fullname" . }}-test-connection"
+  labels:
+    {{- include "wandb.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wandb-verify
+      image: python:3.10
+      env:
+        - name: WANDB_BASE_URL
+          value: "http://{{ .Release.Name }}-app:8080"
+        - name: WANDB_API_KEY
+          value: "test-api-key"
+      # wandb verify is terribly flaky, there's a PR up to fix it until then we try 3 times
+      command:
+        - sh
+        - -c
+        - "pip install wandb==0.17.8 && (wandb verify || (sleep 10 && wandb verify) || (sleep 10 && wandb verify))"
+  restartPolicy: Never
diff --git a/test-configs/minio/default.yaml b/test-configs/minio/default.yaml
new file mode 100644
index 00000000..2c8908e2
--- /dev/null
+++ b/test-configs/minio/default.yaml
@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: minio # Change this value if you want a different namespace name
+  labels:
+    name: minio # Change this value to match metadata.name
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: minio
+  name: minio
+  namespace: minio
+spec:
+  containers:
+    - name: minio
+      image: quay.io/minio/minio:latest
+      env:
+        - name: MINIO_ACCESS_KEY
+          value: "minio"
+        - name: MINIO_SECRET_KEY
+          value: "minio123"
+      command:
+        - /bin/bash
+        - -c
+      args:
+        - mkdir -p /data/bucket && minio server /data --console-address :9090
+      ports:
+        - containerPort: 9000
+          name: minio
+      volumeMounts:
+        - mountPath: /data
+          name: localvolume #
+  volumes:
+    - name: localvolume
+      hostPath:
+        path: /mnt/minio/data
+        type: DirectoryOrCreate
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: minio # Change this value to match the namespace metadata.name
+spec:
+  selector:
+    app: minio
+  ports:
+    - port: 9000
+      name: minio
\ No newline at end of file
diff --git a/test-configs/operator-wandb/default.yaml b/test-configs/operator-wandb/default.yaml
new file mode 100644
index 00000000..cac3dd7c
--- /dev/null
+++ b/test-configs/operator-wandb/default.yaml
@@ -0,0 +1,21 @@
+global:
+  bucket:
+    provider: "s3"
+    name: "minio.minio.svc.cluster.local:9000/bucket"
+    region: "us-east-1"
+    accessKey: "minio"
+    secretKey: "minio123"
+
+app:
+  extraEnv:
+    GLOBAL_ADMIN_API_KEY: "test-api-key"
+    GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
+
+ingress:
+  install: false
+  create: false
+
+mysql:
+  install: true
+redis:
+  install: true

From e3ff218df1827bf272f3da7888b2dc1b1381d1ad Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 14:26:05 -0800
Subject: [PATCH 02/27] fix: Paths for testing jobs

---
 .github/workflows/lint.yaml                | 2 +-
 .github/workflows/test-operator-wandb.yaml | 2 +-
 .github/workflows/test-operator.yaml       | 2 +-
 .github/workflows/test-wandb.yaml          | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index 8b571b3c..6b6b07a3 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -2,7 +2,7 @@ name: Lint Charts
 
 on:
   pull_request:
-    paths: charts
+    paths: charts/**
 
 jobs:
   lint-test:
diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 1b594090..b8e73780 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -2,7 +2,7 @@ name: Test operator-wandb Chart
 
 on:
   pull_request:
-    paths: charts/operator-wandb
+    paths: charts/operator-wandb/**
 
 jobs:
   test:
diff --git a/.github/workflows/test-operator.yaml b/.github/workflows/test-operator.yaml
index e89b4665..ee234ed0 100644
--- a/.github/workflows/test-operator.yaml
+++ b/.github/workflows/test-operator.yaml
@@ -4,7 +4,7 @@
 #
 #on:
 #  pull_request:
-#    paths: charts/operator
+#    paths: charts/operator/**
 #
 #jobs:
 #  test:
diff --git a/.github/workflows/test-wandb.yaml b/.github/workflows/test-wandb.yaml
index fd02783b..8cf9dbb2 100644
--- a/.github/workflows/test-wandb.yaml
+++ b/.github/workflows/test-wandb.yaml
@@ -2,7 +2,7 @@ name: Test wandb Chart
 
 on:
   pull_request:
-    paths: charts/wandb
+    paths: charts/wandb/**
 
 jobs:
   test:

From d32b209f9adf813c5146ea4913af5ad37baead2c Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 14:46:36 -0800
Subject: [PATCH 03/27] fix: test older mysql version

---
 charts/operator-wandb/charts/mysql/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/operator-wandb/charts/mysql/values.yaml b/charts/operator-wandb/charts/mysql/values.yaml
index b779d139..1fa4cd6b 100644
--- a/charts/operator-wandb/charts/mysql/values.yaml
+++ b/charts/operator-wandb/charts/mysql/values.yaml
@@ -9,7 +9,7 @@ fullnameOverride: ""
 
 image:
   repository: mysql
-  tag: latest
+  tag: 8
   pullPolicy: Always
 
 tolerations: []

From 43101056df816b640fc6b426bab4b5ef30e72834 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 14:55:53 -0800
Subject: [PATCH 04/27] fix: revert mysql version change, and try running tests
 on ubuntu 22.04

---
 .github/workflows/test-operator-wandb.yaml     | 2 +-
 charts/operator-wandb/charts/mysql/values.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index b8e73780..5c0f35f4 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -7,11 +7,11 @@ on:
 jobs:
   test:
     name: Test Chart
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
         configuration: ["default"]
+    runs-on: ubuntu-22.04
     environment: Helm Charts
     steps:
       - name: Checkout
diff --git a/charts/operator-wandb/charts/mysql/values.yaml b/charts/operator-wandb/charts/mysql/values.yaml
index 1fa4cd6b..b779d139 100644
--- a/charts/operator-wandb/charts/mysql/values.yaml
+++ b/charts/operator-wandb/charts/mysql/values.yaml
@@ -9,7 +9,7 @@ fullnameOverride: ""
 
 image:
   repository: mysql
-  tag: 8
+  tag: latest
   pullPolicy: Always
 
 tolerations: []

From b34822a29d8cfcdc95043f5eab4840e7395ffc7c Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 15:10:45 -0800
Subject: [PATCH 05/27] fix: increase the timeout for helm install in tests

---
 .github/workflows/test-operator-wandb.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 5c0f35f4..187513b2 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -66,5 +66,5 @@ jobs:
           ct install \
           --charts ./charts/operator-wandb \
           --config ct.yaml \
-          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}' \
+          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} --timeout 600s' \
           --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE'
\ No newline at end of file

From 4d02fef16f053500ccb7c6bbe9a655d107f96f34 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 15:35:52 -0800
Subject: [PATCH 06/27] fix: try adding a startup probe to mysql

---
 .../charts/mysql/templates/statefulset.yaml              | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
index 438e0931..a3dcd453 100644
--- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
+++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
@@ -70,6 +70,15 @@ spec:
               - sh
               - -c
               - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
+          startupProbe:
+            initialDelaySeconds: 20
+            periodSeconds: 5
+            failureThreshold: 120
+            exec:
+              command:
+              - sh
+              - -c
+              - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
           volumeMounts:
             - name: data
               mountPath: /var/lib/mysql

From b1a36519ae8b7e0e019fdbfdc5ffbb63c853c8d8 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 20:31:37 -0800
Subject: [PATCH 07/27] fix: test removing security settings

---
 charts/operator-wandb/charts/mysql/templates/statefulset.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
index a3dcd453..b3423867 100644
--- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
+++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
@@ -32,7 +32,7 @@ spec:
       containers:
         - name: mysql
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          {{- include "wandb.podSecurityContext" .Values.container.securityContext | nindent 10 }}
+         {{- /* include "wandb.podSecurityContext" .Values.container.securityContext | nindent 10 */ }}
           ports:
           - name: mysql
             containerPort: 3306

From 95a4e079d0d57e1293678a471af4cc38f5c68ed9 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 20:33:42 -0800
Subject: [PATCH 08/27] fix: test removing security settings

---
 charts/operator-wandb/charts/mysql/templates/statefulset.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
index b3423867..00fd6403 100644
--- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
+++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
@@ -32,7 +32,6 @@ spec:
       containers:
         - name: mysql
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-         {{- /* include "wandb.podSecurityContext" .Values.container.securityContext | nindent 10 */ }}
           ports:
           - name: mysql
             containerPort: 3306

From 330ff9db75e0287ad93d1a0c9860cfebadbd83f1 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 20:51:35 -0800
Subject: [PATCH 09/27] fix: try and older version of mysql

---
 .github/workflows/test-operator-wandb.yaml                    | 2 +-
 charts/operator-wandb/charts/mysql/templates/statefulset.yaml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 187513b2..5c0f35f4 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -66,5 +66,5 @@ jobs:
           ct install \
           --charts ./charts/operator-wandb \
           --config ct.yaml \
-          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} --timeout 600s' \
+          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}' \
           --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE'
\ No newline at end of file
diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
index 00fd6403..02ad81e9 100644
--- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
+++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
@@ -31,7 +31,7 @@ spec:
       {{- include "wandb.podSecurityContext" .Values.pod.securityContext | nindent 6 }}
       containers:
         - name: mysql
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: "{{ .Values.image.repository }}:8.2"
           ports:
           - name: mysql
             containerPort: 3306
@@ -72,7 +72,7 @@ spec:
           startupProbe:
             initialDelaySeconds: 20
             periodSeconds: 5
-            failureThreshold: 120
+            failureThreshold: 60
             exec:
               command:
               - sh

From 407728b7e944760a24cbc4ceafe3e23417e384ad Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 21:14:34 -0800
Subject: [PATCH 10/27] fix: revert some mysql changes and remove multiple k8s
 versions until I can get the tests to work in GA

---
 .github/workflows/test-operator-wandb.yaml                    | 2 +-
 charts/operator-wandb/charts/mysql/templates/statefulset.yaml | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 5c0f35f4..784ae02b 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -9,7 +9,7 @@ jobs:
     name: Test Chart
     strategy:
       matrix:
-        k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
+        k8s-version: ["v1.32.1"]
         configuration: ["default"]
     runs-on: ubuntu-22.04
     environment: Helm Charts
diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
index 02ad81e9..707fdef7 100644
--- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
+++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
@@ -31,7 +31,8 @@ spec:
       {{- include "wandb.podSecurityContext" .Values.pod.securityContext | nindent 6 }}
       containers:
         - name: mysql
-          image: "{{ .Values.image.repository }}:8.2"
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          {{- include "wandb.podSecurityContext" .Values.container.securityContext | nindent 10 }}
           ports:
           - name: mysql
             containerPort: 3306

From 19c5e9e2515be743c022a0994ab20baa09efdfae Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 21:44:57 -0800
Subject: [PATCH 11/27] fix: try updating the userid in podsecurity

---
 charts/operator-wandb/charts/mysql/values.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/operator-wandb/charts/mysql/values.yaml b/charts/operator-wandb/charts/mysql/values.yaml
index b779d139..6e9e61d2 100644
--- a/charts/operator-wandb/charts/mysql/values.yaml
+++ b/charts/operator-wandb/charts/mysql/values.yaml
@@ -29,9 +29,9 @@ pod:
   annotations: {}
   securityContext:
     runAsNonRoot: true
-    runAsUser: 1000
+    runAsUser: 999
     runAsGroup: 0
-    fsGroup: 1000
+    fsGroup: 999
     fsGroupChangePolicy: "OnRootMismatch"
     seccompProfile:
       type: ""

From 238b37f62dd97deb7f4668ed4e14a7146d56a35a Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 21:51:56 -0800
Subject: [PATCH 12/27] fix: try allowPrivilegeEscalation instead.

---
 charts/operator-wandb/charts/mysql/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/operator-wandb/charts/mysql/values.yaml b/charts/operator-wandb/charts/mysql/values.yaml
index 6e9e61d2..873227ae 100644
--- a/charts/operator-wandb/charts/mysql/values.yaml
+++ b/charts/operator-wandb/charts/mysql/values.yaml
@@ -41,7 +41,7 @@ container:
     capabilities:
       add: []
       drop: []
-    allowPrivilegeEscalation: false
+    allowPrivilegeEscalation: true
     readOnlyRootFilesystem: false
     privileged: false
 

From 5626cc2f7b1317e8942534ea075096d21a708ec8 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 21:59:44 -0800
Subject: [PATCH 13/27] fix: try tcp port as health check

---
 .../charts/mysql/templates/statefulset.yaml   | 21 ++++++-------------
 .../operator-wandb/charts/mysql/values.yaml   |  2 +-
 2 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
index 707fdef7..c2b2a1be 100644
--- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
+++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml
@@ -59,26 +59,17 @@ spec:
             {{- include "mysql.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
           livenessProbe:
-            exec:
-              command:
-              - sh
-              - -c
-              - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
+            tcpSocket:
+              port: 3306
           readinessProbe:
-            exec:
-              command:
-              - sh
-              - -c
-              - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
+            tcpSocket:
+              port: 3306
           startupProbe:
             initialDelaySeconds: 20
             periodSeconds: 5
             failureThreshold: 60
-            exec:
-              command:
-              - sh
-              - -c
-              - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
+            tcpSocket:
+              port: 3306
           volumeMounts:
             - name: data
               mountPath: /var/lib/mysql
diff --git a/charts/operator-wandb/charts/mysql/values.yaml b/charts/operator-wandb/charts/mysql/values.yaml
index 873227ae..6e9e61d2 100644
--- a/charts/operator-wandb/charts/mysql/values.yaml
+++ b/charts/operator-wandb/charts/mysql/values.yaml
@@ -41,7 +41,7 @@ container:
     capabilities:
       add: []
       drop: []
-    allowPrivilegeEscalation: true
+    allowPrivilegeEscalation: false
     readOnlyRootFilesystem: false
     privileged: false
 

From f0dcd572eab623be56f6e7e1f51510f1321602f6 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 22:11:03 -0800
Subject: [PATCH 14/27] fix: Add back previous k8s versions and add a config
 for newer features

---
 .github/workflows/test-operator-wandb.yaml    |  4 +--
 .../operator-wandb/separate_pods.yaml         | 26 +++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)
 create mode 100644 test-configs/operator-wandb/separate_pods.yaml

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 784ae02b..0082adad 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -9,8 +9,8 @@ jobs:
     name: Test Chart
     strategy:
       matrix:
-        k8s-version: ["v1.32.1"]
-        configuration: ["default"]
+        k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
+        configuration: ["default", "separate_pods"]
     runs-on: ubuntu-22.04
     environment: Helm Charts
     steps:
diff --git a/test-configs/operator-wandb/separate_pods.yaml b/test-configs/operator-wandb/separate_pods.yaml
new file mode 100644
index 00000000..2915a633
--- /dev/null
+++ b/test-configs/operator-wandb/separate_pods.yaml
@@ -0,0 +1,26 @@
+global:
+  bucket:
+    provider: "s3"
+    name: "minio.minio.svc.cluster.local:9000/bucket"
+    region: "us-east-1"
+    accessKey: "minio"
+    secretKey: "minio123"
+
+  beta:
+    glue:
+      enabled: true
+    api:
+      enabled: true
+app:
+  extraEnv:
+    GLOBAL_ADMIN_API_KEY: "test-api-key"
+    GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
+
+ingress:
+  install: false
+  create: false
+
+mysql:
+  install: true
+redis:
+  install: true

From a7fd2fcc5264a6d541f9df16dfe4d535b48b4651 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 22:13:41 -0800
Subject: [PATCH 15/27] fix: Fix invalid cluster name in GA

---
 .github/workflows/test-operator-wandb.yaml                      | 2 +-
 .../operator-wandb/{separate_pods.yaml => separate-pods.yaml}   | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename test-configs/operator-wandb/{separate_pods.yaml => separate-pods.yaml} (100%)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 0082adad..783df43f 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
-        configuration: ["default", "separate_pods"]
+        configuration: ["default", "separate-pods"]
     runs-on: ubuntu-22.04
     environment: Helm Charts
     steps:
diff --git a/test-configs/operator-wandb/separate_pods.yaml b/test-configs/operator-wandb/separate-pods.yaml
similarity index 100%
rename from test-configs/operator-wandb/separate_pods.yaml
rename to test-configs/operator-wandb/separate-pods.yaml

From 8bfe0ab3fd77268fc8bb2539e31f12baf18149a2 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 22:28:43 -0800
Subject: [PATCH 16/27] fix: try to use larger runner

---
 .github/workflows/test-operator-wandb.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 783df43f..1b2a3dcf 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -11,7 +11,7 @@ jobs:
       matrix:
         k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
         configuration: ["default", "separate-pods"]
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest-8-cores
     environment: Helm Charts
     steps:
       - name: Checkout

From 60526d8942cc33c4cb06832422e2f0e670781ba5 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 22:34:51 -0800
Subject: [PATCH 17/27] fix: reduce resource requests

---
 .github/workflows/test-operator-wandb.yaml    |  2 +-
 test-configs/operator-wandb/default.yaml      | 24 +++++++++++++
 .../operator-wandb/separate-pods.yaml         | 34 +++++++++++++++++++
 3 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 1b2a3dcf..0b93bbbd 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -11,7 +11,7 @@ jobs:
       matrix:
         k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
         configuration: ["default", "separate-pods"]
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-latest
     environment: Helm Charts
     steps:
       - name: Checkout
diff --git a/test-configs/operator-wandb/default.yaml b/test-configs/operator-wandb/default.yaml
index cac3dd7c..74fa4a57 100644
--- a/test-configs/operator-wandb/default.yaml
+++ b/test-configs/operator-wandb/default.yaml
@@ -10,6 +10,24 @@ app:
   extraEnv:
     GLOBAL_ADMIN_API_KEY: "test-api-key"
     GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
+  resources:
+    requests: {}
+    limits: {}
+
+parquet:
+  resources:
+    requests: {}
+    limits: {}
+
+weave:
+  resources:
+    requests: {}
+    limits: {}
+
+console:
+  resources:
+    requests: {}
+    limits: {}
 
 ingress:
   install: false
@@ -17,5 +35,11 @@ ingress:
 
 mysql:
   install: true
+  resources:
+    requests: {}
+    limits: {}
 redis:
   install: true
+  resources:
+    requests: {}
+    limits: {}
diff --git a/test-configs/operator-wandb/separate-pods.yaml b/test-configs/operator-wandb/separate-pods.yaml
index 2915a633..bd744a1f 100644
--- a/test-configs/operator-wandb/separate-pods.yaml
+++ b/test-configs/operator-wandb/separate-pods.yaml
@@ -15,6 +15,34 @@ app:
   extraEnv:
     GLOBAL_ADMIN_API_KEY: "test-api-key"
     GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
+  resources:
+    requests: {}
+    limits: {}
+
+api:
+  resources:
+    requests: {}
+    limits: {}
+
+glue:
+  resources:
+    requests: {}
+    limits: {}
+
+parquet:
+  resources:
+    requests: {}
+    limits: {}
+
+weave:
+  resources:
+    requests: {}
+    limits: {}
+
+console:
+  resources:
+    requests: {}
+    limits: {}
 
 ingress:
   install: false
@@ -22,5 +50,11 @@ ingress:
 
 mysql:
   install: true
+  resources:
+    requests: {}
+    limits: {}
 redis:
   install: true
+  resources:
+    requests: {}
+    limits: {}

From 1467724d88b53476f04a92447fcb01378e22f327 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 22:46:55 -0800
Subject: [PATCH 18/27] fix: reformat requests

---
 test-configs/operator-wandb/default.yaml      | 24 ++++---------
 .../operator-wandb/separate-pods.yaml         | 36 +++++++------------
 2 files changed, 18 insertions(+), 42 deletions(-)

diff --git a/test-configs/operator-wandb/default.yaml b/test-configs/operator-wandb/default.yaml
index 74fa4a57..5aa6d681 100644
--- a/test-configs/operator-wandb/default.yaml
+++ b/test-configs/operator-wandb/default.yaml
@@ -10,24 +10,16 @@ app:
   extraEnv:
     GLOBAL_ADMIN_API_KEY: "test-api-key"
     GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 
 parquet:
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 
 weave:
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 
 console:
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 
 ingress:
   install: false
@@ -35,11 +27,7 @@ ingress:
 
 mysql:
   install: true
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 redis:
   install: true
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
diff --git a/test-configs/operator-wandb/separate-pods.yaml b/test-configs/operator-wandb/separate-pods.yaml
index bd744a1f..f802517f 100644
--- a/test-configs/operator-wandb/separate-pods.yaml
+++ b/test-configs/operator-wandb/separate-pods.yaml
@@ -15,34 +15,26 @@ app:
   extraEnv:
     GLOBAL_ADMIN_API_KEY: "test-api-key"
     GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 
 api:
-  resources:
-    requests: {}
-    limits: {}
+  containers:
+    api:
+      resources: {}
 
 glue:
-  resources:
-    requests: {}
-    limits: {}
+  containers:
+    glue:
+      resources: {}
 
 parquet:
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 
 weave:
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 
 console:
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 
 ingress:
   install: false
@@ -50,11 +42,7 @@ ingress:
 
 mysql:
   install: true
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}
 redis:
   install: true
-  resources:
-    requests: {}
-    limits: {}
+  resources: {}

From c4ba5bedce53979e3ee5466fee9beb4ba53bab76 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 22:51:01 -0800
Subject: [PATCH 19/27] fix: one more try

---
 test-configs/operator-wandb/default.yaml       | 12 ++++++------
 test-configs/operator-wandb/separate-pods.yaml | 16 ++++++++--------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/test-configs/operator-wandb/default.yaml b/test-configs/operator-wandb/default.yaml
index 5aa6d681..dad21c2c 100644
--- a/test-configs/operator-wandb/default.yaml
+++ b/test-configs/operator-wandb/default.yaml
@@ -10,16 +10,16 @@ app:
   extraEnv:
     GLOBAL_ADMIN_API_KEY: "test-api-key"
     GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
-  resources: {}
+  resources: null
 
 parquet:
-  resources: {}
+  resources: null
 
 weave:
-  resources: {}
+  resources: null
 
 console:
-  resources: {}
+  resources: null
 
 ingress:
   install: false
@@ -27,7 +27,7 @@ ingress:
 
 mysql:
   install: true
-  resources: {}
+  resources: null
 redis:
   install: true
-  resources: {}
+  resources: null
diff --git a/test-configs/operator-wandb/separate-pods.yaml b/test-configs/operator-wandb/separate-pods.yaml
index f802517f..c7267484 100644
--- a/test-configs/operator-wandb/separate-pods.yaml
+++ b/test-configs/operator-wandb/separate-pods.yaml
@@ -15,26 +15,26 @@ app:
   extraEnv:
     GLOBAL_ADMIN_API_KEY: "test-api-key"
     GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
-  resources: {}
+  resources: null
 
 api:
   containers:
     api:
-      resources: {}
+      resources: null
 
 glue:
   containers:
     glue:
-      resources: {}
+      resources: null
 
 parquet:
-  resources: {}
+  resources: null
 
 weave:
-  resources: {}
+  resources: null
 
 console:
-  resources: {}
+  resources: null
 
 ingress:
   install: false
@@ -42,7 +42,7 @@ ingress:
 
 mysql:
   install: true
-  resources: {}
+  resources: null
 redis:
   install: true
-  resources: {}
+  resources: null

From 1e69ca8c0cede5b29047e721d838b4f62960d83b Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Sun, 2 Feb 2025 22:57:42 -0800
Subject: [PATCH 20/27] fix: explicitly request less resources

---
 test-configs/operator-wandb/default.yaml      | 30 +++++++++++---
 .../operator-wandb/separate-pods.yaml         | 40 +++++++++++++++----
 2 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/test-configs/operator-wandb/default.yaml b/test-configs/operator-wandb/default.yaml
index dad21c2c..858bf0cd 100644
--- a/test-configs/operator-wandb/default.yaml
+++ b/test-configs/operator-wandb/default.yaml
@@ -10,16 +10,28 @@ app:
   extraEnv:
     GLOBAL_ADMIN_API_KEY: "test-api-key"
     GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+    memory: "128Mi"
 
 parquet:
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
 
 weave:
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
 
 console:
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
 
 ingress:
   install: false
@@ -27,7 +39,13 @@ ingress:
 
 mysql:
   install: true
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
 redis:
   install: true
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
diff --git a/test-configs/operator-wandb/separate-pods.yaml b/test-configs/operator-wandb/separate-pods.yaml
index c7267484..b50cc949 100644
--- a/test-configs/operator-wandb/separate-pods.yaml
+++ b/test-configs/operator-wandb/separate-pods.yaml
@@ -15,26 +15,44 @@ app:
   extraEnv:
     GLOBAL_ADMIN_API_KEY: "test-api-key"
     GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
 
 api:
   containers:
     api:
-      resources: null
+      resources:
+        requests:
+          cpu: "100m"
+          memory: "128Mi"
 
 glue:
   containers:
     glue:
-      resources: null
+      resources:
+        requests:
+          cpu: "100m"
+          memory: "128Mi"
 
 parquet:
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
 
 weave:
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
 
 console:
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
 
 ingress:
   install: false
@@ -42,7 +60,13 @@ ingress:
 
 mysql:
   install: true
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
 redis:
   install: true
-  resources: null
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"

From cf08efa459f77cba4453835306be32f4c17be530 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Tue, 4 Feb 2025 18:09:41 -0800
Subject: [PATCH 21/27] fix: WIP on fixing redis and adding more test scenarios

---
 .github/workflows/test-operator-wandb.yaml    |  4 +-
 charts/operator-wandb/Chart.lock              | 10 +-
 .../bufstream/config/bufstream.yaml.tpl       |  4 +-
 .../bufstream/templates/deployment.yaml       |  4 +-
 .../charts/stackdriver/values.yaml            |  2 +-
 charts/operator-wandb/templates/_redis.tpl    |  2 +-
 charts/operator-wandb/templates/redis.yaml    |  6 +-
 charts/operator-wandb/values.yaml             | 14 +--
 test-configs/operator-wandb/default.yaml      |  2 +
 .../operator-wandb/runs-v2-bufstream.yaml     | 99 +++++++++++++++++++
 10 files changed, 124 insertions(+), 23 deletions(-)
 create mode 100644 test-configs/operator-wandb/runs-v2-bufstream.yaml

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 0b93bbbd..c197895c 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -9,8 +9,8 @@ jobs:
     name: Test Chart
     strategy:
       matrix:
-        k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
-        configuration: ["default", "separate-pods"]
+        k8s-version: ["v1.32.1"]
+        configuration: ["runs-v2-bufstream"]
     runs-on: ubuntu-latest
     environment: Helm Charts
     steps:
diff --git a/charts/operator-wandb/Chart.lock b/charts/operator-wandb/Chart.lock
index 4b9beee2..20030472 100644
--- a/charts/operator-wandb/Chart.lock
+++ b/charts/operator-wandb/Chart.lock
@@ -4,7 +4,7 @@ dependencies:
   version: 0.1.0
 - name: wandb-base
   repository: file://../wandb-base
-  version: 0.4.0
+  version: 0.4.1
 - name: console
   repository: file://charts/console
   version: 0.1.0
@@ -58,9 +58,9 @@ dependencies:
   version: 0.1.0
 - name: wandb-base
   repository: file://../wandb-base
-  version: 0.4.0
+  version: 0.4.1
 - name: wandb-base
   repository: file://../wandb-base
-  version: 0.4.0
-digest: sha256:d9e104084992aaca36330a56d4a5b825fd20f0bdf995070bf93f46a883375169
-generated: "2025-01-29T12:38:09.214239-06:00"
+  version: 0.4.1
+digest: sha256:d6b5ef46b80814179a954ed674dc248c65cdc8192ead7a11f1df3861da29feed
+generated: "2025-02-04T09:35:45.742766-08:00"
diff --git a/charts/operator-wandb/charts/bufstream/config/bufstream.yaml.tpl b/charts/operator-wandb/charts/bufstream/config/bufstream.yaml.tpl
index 7d05b936..da2eeb21 100644
--- a/charts/operator-wandb/charts/bufstream/config/bufstream.yaml.tpl
+++ b/charts/operator-wandb/charts/bufstream/config/bufstream.yaml.tpl
@@ -24,7 +24,7 @@ storage:
   {{- end }}
   {{- if .Values.storage.s3.accessKeyId }}
   access_key_id:
-    string: {{ .Values.storage.s3.accessKeyId | quote }}
+    string: {{ tpl .Values.storage.s3.accessKeyId . | quote }}
   secret_access_key:
     path: /config/secrets/storage/secret_access_key
   {{- end }}
@@ -40,7 +40,7 @@ storage:
 etcd:
   addresses:
 {{- range .Values.metadata.etcd.addresses }}
-  - host: {{ .host | quote }}
+  - host: {{ tpl .host $ | quote }}
     port: {{ .port }}
 {{- end }}
 {{- with .Values.metadata.etcd.tls }}
diff --git a/charts/operator-wandb/charts/bufstream/templates/deployment.yaml b/charts/operator-wandb/charts/bufstream/templates/deployment.yaml
index a2f24e0b..21d7bd05 100644
--- a/charts/operator-wandb/charts/bufstream/templates/deployment.yaml
+++ b/charts/operator-wandb/charts/bufstream/templates/deployment.yaml
@@ -250,9 +250,9 @@ spec:
         {{- if and (eq .Values.storage.use "s3") (.Values.storage.s3.accessKeyId) }}
         - name: storage
           secret:
-            secretName: {{ .Values.storage.s3.secretName }}
+            secretName: {{ tpl .Values.storage.s3.secretName . }}
             items:
-              - key: secret_access_key
+              - key: {{ tpl .Values.storage.s3.secretKeyName . }}
                 path: secret_access_key
         {{- end }}
         {{- if and (eq .Values.storage.use "gcs") (.Values.storage.gcs.secretName) }}
diff --git a/charts/operator-wandb/charts/stackdriver/values.yaml b/charts/operator-wandb/charts/stackdriver/values.yaml
index 51ed7b65..d7463a98 100644
--- a/charts/operator-wandb/charts/stackdriver/values.yaml
+++ b/charts/operator-wandb/charts/stackdriver/values.yaml
@@ -50,7 +50,7 @@ stackdriver:
     # Filter objects: project, group.id, resource.type, resource.labels.[KEY], metric.type, metric.labels.[KEY]
     # https://cloud.google.com/monitoring/api/v3/filters
     filters:
-      []
+      - cloudsql.googleapis.com/database:metadata.user_labels."customer-ns"="dpanzella-test-gcp"
       # - 'pubsub.googleapis.com/subscription:resource.labels.subscription_id=monitoring.regex.full_match("us-west4.*my-team.*")'
     # The frequency to request
     interval: "5m"
diff --git a/charts/operator-wandb/templates/_redis.tpl b/charts/operator-wandb/templates/_redis.tpl
index 9c29f6e2..412b117f 100644
--- a/charts/operator-wandb/templates/_redis.tpl
+++ b/charts/operator-wandb/templates/_redis.tpl
@@ -5,7 +5,7 @@ Return name of secret where redis information is stored
 {{- if .Values.global.redis.secret.secretName -}}
   {{ .Values.global.redis.secret.secretName }}
 {{- else -}}
-  {{- print .Release.Name "-redis-secret" -}}
+  {{- print .Release.Name "-redis" -}}
 {{- end -}}
 {{- end -}}
 
diff --git a/charts/operator-wandb/templates/redis.yaml b/charts/operator-wandb/templates/redis.yaml
index 7330fa1e..f3f3adc5 100644
--- a/charts/operator-wandb/templates/redis.yaml
+++ b/charts/operator-wandb/templates/redis.yaml
@@ -1,16 +1,14 @@
-{{- if not .Values.redis.install }}
-{{- $secretName := (include "wandb.redis.passwordSecret" .)  }}
+{{- if not .Values.global.redis.secret.secretName }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: "{{ $secretName }}"
+  name: "{{ include "wandb.redis.passwordSecret" . }}"
   labels:
     {{- include "wandb.commonLabels" . | nindent 4 }}
 data:
   REDIS_PASSWORD: {{ include "wandb.redis.password" . | b64enc }}
   REDIS_CA_CERT: {{ include "wandb.redis.caCert" . | b64enc }}
 {{- end }}
-
 ---
 apiVersion: v1
 kind: ConfigMap
diff --git a/charts/operator-wandb/values.yaml b/charts/operator-wandb/values.yaml
index 310a1622..a546ab20 100644
--- a/charts/operator-wandb/values.yaml
+++ b/charts/operator-wandb/values.yaml
@@ -122,7 +122,7 @@ global:
   redis:
     host: ""
     port: 6379
-    password: ""
+    password: "redis-password"
     parameters: {}
     caCert: ""
     secret:
@@ -323,7 +323,9 @@ redis:
   nameOverride: "redis"
   architecture: standalone
   auth:
-    enabled: false
+    enabled: true
+    existingSecret: '{{ include "wandb.redis.passwordSecret" . }}'
+    existingSecretPasswordKey: '{{ .Values.global.redis.secret.secretKey }}'
   metrics:
     enabled: false
     service:
@@ -526,7 +528,7 @@ glue:
     "{{ .Release.Name }}-mysql": "secretRef"
     "{{ .Release.Name }}-bucket-configmap": "configMapRef"
     "{{ .Release.Name }}-mysql-configmap": "configMapRef"
-    "{{ .Release.Name }}-redis-secret": "secretRef"
+    '{{ include "wandb.redis.passwordSecret" . }}': "secretRef"
     "{{ .Release.Name }}-redis-configmap": "configMapRef"
     "{{ .Release.Name }}-global-secret": "secretRef"
     "{{ .Release.Name }}-gorilla-secret": "secretRef"
@@ -622,7 +624,7 @@ glue:
         optional: true
     - name: redis-ca
       secret:
-        secretName: "{{ .Release.Name }}-redis-secret"
+        secretName: '{{ include "wandb.redis.passwordSecret" . }}'
         items:
           - key: REDIS_CA_CERT
             path: redis_ca.pem
@@ -717,7 +719,7 @@ api:
     "{{ .Release.Name }}-mysql": "secretRef"
     "{{ .Release.Name }}-bucket-configmap": "configMapRef"
     "{{ .Release.Name }}-mysql-configmap": "configMapRef"
-    "{{ .Release.Name }}-redis-secret": "secretRef"
+    '{{ include "wandb.redis.passwordSecret" . }}': "secretRef"
     "{{ .Release.Name }}-redis-configmap": "configMapRef"
     "{{ .Release.Name }}-kafka-configmap": "configMapRef"
     "{{ .Release.Name }}-global-secret": "secretRef"
@@ -819,7 +821,7 @@ api:
         optional: true
     - name: redis-ca
       secret:
-        secretName: "{{ .Release.Name }}-redis-secret"
+        secretName: '{{ include "wandb.redis.passwordSecret" . }}'
         items:
           - key: REDIS_CA_CERT
             path: redis_ca.pem
diff --git a/test-configs/operator-wandb/default.yaml b/test-configs/operator-wandb/default.yaml
index 858bf0cd..832857b1 100644
--- a/test-configs/operator-wandb/default.yaml
+++ b/test-configs/operator-wandb/default.yaml
@@ -45,6 +45,8 @@ mysql:
       memory: "128Mi"
 redis:
   install: true
+  auth:
+    enabled: true
   resources:
     requests:
       cpu: "100m"
diff --git a/test-configs/operator-wandb/runs-v2-bufstream.yaml b/test-configs/operator-wandb/runs-v2-bufstream.yaml
new file mode 100644
index 00000000..1c0a2a34
--- /dev/null
+++ b/test-configs/operator-wandb/runs-v2-bufstream.yaml
@@ -0,0 +1,99 @@
+global:
+  bucket:
+    provider: "s3"
+    name: "minio.minio.svc.cluster.local:9000/bucket"
+    region: "us-east-1"
+    accessKey: "minio"
+    secretKey: "minio123"
+
+  beta:
+    glue:
+      enabled: true
+    api:
+      enabled: true
+    bufstream:
+      enabled: true
+app:
+  extraEnv:
+    GLOBAL_ADMIN_API_KEY: "test-api-key"
+    GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
+
+api:
+  containers:
+    api:
+      resources:
+        requests:
+          cpu: "100m"
+          memory: "128Mi"
+
+bufstream:
+  install: true
+  storage:
+    use: s3
+    s3:
+      accessKeyId: "minio"
+      secretName: '{{ (include "wandb.bucket" . | fromYaml).secretName }}'
+      secretKeyName: '{{ (include "wandb.bucket" . | fromYaml).secretKeyName }}'
+      bucket: bucket
+      region: "us-east-1"
+      forcePathStyle: true
+      endpoint: http://minio.minio.svc.cluster.local:9000
+  metadata:
+    use: etcd
+    etcd:
+      addresses:
+        - host: "{{ .Release.Name }}-etcd.{{ .Release.Namespace }}.svc.cluster.local"
+          port: 2379
+  bufstream:
+    deployment:
+      resources:
+        requests:
+          cpu: "100m"
+          memory: "128Mi"
+
+glue:
+  containers:
+    glue:
+      resources:
+        requests:
+          cpu: "100m"
+          memory: "128Mi"
+
+parquet:
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
+
+weave:
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
+
+console:
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
+
+ingress:
+  install: false
+  create: false
+
+mysql:
+  install: true
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
+redis:
+  install: true
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"

From de08d371ff50d567fe52b5401f9fb21d0db49661 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Tue, 4 Feb 2025 21:01:36 -0800
Subject: [PATCH 22/27] fix: re-add k8s versions and 3 configs, bufstream is
 still not working

---
 .github/workflows/test-operator-wandb.yaml         | 4 ++--
 test-configs/operator-wandb/runs-v2-bufstream.yaml | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index c197895c..2666dbc0 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -9,8 +9,8 @@ jobs:
     name: Test Chart
     strategy:
       matrix:
-        k8s-version: ["v1.32.1"]
-        configuration: ["runs-v2-bufstream"]
+        k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
+        configuration: ["default", "separate-pods""runs-v2-bufstream"]
     runs-on: ubuntu-latest
     environment: Helm Charts
     steps:
diff --git a/test-configs/operator-wandb/runs-v2-bufstream.yaml b/test-configs/operator-wandb/runs-v2-bufstream.yaml
index 1c0a2a34..85fd9bc4 100644
--- a/test-configs/operator-wandb/runs-v2-bufstream.yaml
+++ b/test-configs/operator-wandb/runs-v2-bufstream.yaml
@@ -17,6 +17,13 @@ app:
   extraEnv:
     GLOBAL_ADMIN_API_KEY: "test-api-key"
     GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true"
+    GORILLA_RUN_STORE_ONPREM_MIGRATE_CREATE_RUN_TABLES: 'true'
+    GORILLA_RUN_STORE_ONPREM_MIGRATE_CREATE_RUN_STORE: 'true'
+    GORILLA_RUN_STORE_ONPREM_MIGRATE_SHADOW_RUN_UPDATES: 'true'
+    GORILLA_RUN_STORE_ONPREM_MIGRATE_DISABLE_READS: 'false'
+    GORILLA_RUN_STORE_ONPREM_MIGRATE_FLAT_RUNS_MIGRATOR: 'true'
+    GORILLA_STATSD_HOST: datadog.datadog
+    GORILLA_STATSD_PORT: 8125
   resources:
     requests:
       cpu: "100m"
@@ -32,6 +39,7 @@ api:
 
 bufstream:
   install: true
+  zone: "kind"
   storage:
     use: s3
     s3:

From 15dad984770af2fd0168a0573e59d1221d9b6348 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Tue, 4 Feb 2025 21:10:34 -0800
Subject: [PATCH 23/27] fix: fix formatting in workflow

---
 .github/workflows/lint.yaml                | 2 +-
 .github/workflows/test-operator-wandb.yaml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index 6b6b07a3..5207c037 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -40,4 +40,4 @@ jobs:
           fi
 
       - name: Run chart-testing  (lint)
-        run: ct lint --config ct.yaml
\ No newline at end of file
+        run: ct lint --config ct.yaml
diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 2666dbc0..1ff6de95 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
-        configuration: ["default", "separate-pods""runs-v2-bufstream"]
+        configuration: ["default", "separate-pods", "runs-v2-bufstream"]
     runs-on: ubuntu-latest
     environment: Helm Charts
     steps:
@@ -67,4 +67,4 @@ jobs:
           --charts ./charts/operator-wandb \
           --config ct.yaml \
           --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}' \
-          --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE'
\ No newline at end of file
+          --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE'

From 6b35bb48bb1120335322704f519b3be1162299b0 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Tue, 4 Feb 2025 21:20:27 -0800
Subject: [PATCH 24/27] fix: I bet actually enabling frfu is going to help
 runsv2 work

---
 test-configs/operator-wandb/runs-v2-bufstream.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test-configs/operator-wandb/runs-v2-bufstream.yaml b/test-configs/operator-wandb/runs-v2-bufstream.yaml
index 85fd9bc4..959e08cb 100644
--- a/test-configs/operator-wandb/runs-v2-bufstream.yaml
+++ b/test-configs/operator-wandb/runs-v2-bufstream.yaml
@@ -63,6 +63,13 @@ bufstream:
           cpu: "100m"
           memory: "128Mi"
 
+flat-run-fields-updater:
+  install: true
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "128Mi"
+
 glue:
   containers:
     glue:

From 4bd5fe62c1a49574284c608dea1d639a4b7a5d8d Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Tue, 4 Feb 2025 21:54:57 -0800
Subject: [PATCH 25/27] fix: See if it's just taking to long to stabilize

---
 .github/workflows/test-operator-wandb.yaml                 | 3 ++-
 charts/operator-wandb/templates/tests/test-connection.yaml | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 1ff6de95..9bc2ff53 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -9,6 +9,7 @@ jobs:
     name: Test Chart
     strategy:
       matrix:
+        fail-fast: false
         k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
         configuration: ["default", "separate-pods", "runs-v2-bufstream"]
     runs-on: ubuntu-latest
@@ -66,5 +67,5 @@ jobs:
           ct install \
           --charts ./charts/operator-wandb \
           --config ct.yaml \
-          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}' \
+          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} --wait 600' \
           --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE'
diff --git a/charts/operator-wandb/templates/tests/test-connection.yaml b/charts/operator-wandb/templates/tests/test-connection.yaml
index 8d1345cd..a7010b65 100644
--- a/charts/operator-wandb/templates/tests/test-connection.yaml
+++ b/charts/operator-wandb/templates/tests/test-connection.yaml
@@ -19,5 +19,5 @@ spec:
       command:
         - sh
         - -c
-        - "pip install wandb==0.17.8 && (wandb verify || (sleep 10 && wandb verify) || (sleep 10 && wandb verify))"
+        - "pip install wandb==0.17.8 && ((sleep 30 && wandb verify) || (sleep 30 && wandb verify) || (sleep 30 && wandb verify))"
   restartPolicy: Never

From 7da499acaec6c2931586d9b47f99f30a81012db3 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Tue, 4 Feb 2025 21:56:08 -0800
Subject: [PATCH 26/27] fix: put the config in the correct place

---
 .github/workflows/test-operator-wandb.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index 9bc2ff53..f2be8a1b 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -8,8 +8,8 @@ jobs:
   test:
     name: Test Chart
     strategy:
+      fail-fast: false
       matrix:
-        fail-fast: false
         k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"]
         configuration: ["default", "separate-pods", "runs-v2-bufstream"]
     runs-on: ubuntu-latest

From 154a7d4f9f317dee28f74155d59499e17a835883 Mon Sep 17 00:00:00 2001
From: Daniel Panzella <daniel.panzella@wandb.com>
Date: Tue, 4 Feb 2025 21:58:31 -0800
Subject: [PATCH 27/27] fix: Helps to use the correct syntax

---
 .github/workflows/test-operator-wandb.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml
index f2be8a1b..a75754cf 100644
--- a/.github/workflows/test-operator-wandb.yaml
+++ b/.github/workflows/test-operator-wandb.yaml
@@ -67,5 +67,5 @@ jobs:
           ct install \
           --charts ./charts/operator-wandb \
           --config ct.yaml \
-          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} --wait 600' \
+          --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} --timeout 600s' \
           --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE'