From 9b6c27346f774a098b799be57ea907953064aa8d Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 14:22:05 -0800 Subject: [PATCH 01/10] fix: Add tests for operator-wandb --- .github/workflows/lint.yaml | 43 ++++++++++++ .github/workflows/test-operator-wandb.yaml | 70 +++++++++++++++++++ .github/workflows/test-operator.yaml | 54 ++++++++++++++ .../{lint-test.yaml => test-wandb.yaml} | 11 ++- charts/operator-wandb/Chart.yaml | 2 +- .../templates/tests/test-connection.yaml | 23 ++++++ test-configs/minio/default.yaml | 51 ++++++++++++++ test-configs/operator-wandb/default.yaml | 21 ++++++ 8 files changed, 268 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/lint.yaml create mode 100644 .github/workflows/test-operator-wandb.yaml create mode 100644 .github/workflows/test-operator.yaml rename .github/workflows/{lint-test.yaml => test-wandb.yaml} (91%) create mode 100644 charts/operator-wandb/templates/tests/test-connection.yaml create mode 100644 test-configs/minio/default.yaml create mode 100644 test-configs/operator-wandb/default.yaml diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..8b571b3c --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,43 @@ +name: Lint Charts + +on: + pull_request: + paths: charts + +jobs: + lint-test: + runs-on: ubuntu-latest + environment: Helm Charts + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3.4 + with: + version: v3.16.3 + + # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and + # yamllint (https://github.com/adrienverge/yamllint) which require Python + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.6.1 + with: + version: v3.11.0 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --config ct.yaml) + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + + - name: Run chart-testing (lint) + run: ct lint --config ct.yaml \ No newline at end of file diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml new file mode 100644 index 00000000..1b594090 --- /dev/null +++ b/.github/workflows/test-operator-wandb.yaml @@ -0,0 +1,70 @@ +name: Test operator-wandb Chart + +on: + pull_request: + paths: charts/operator-wandb + +jobs: + test: + name: Test Chart + runs-on: ubuntu-latest + strategy: + matrix: + k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"] + configuration: ["default"] + environment: Helm Charts + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3.4 + with: + version: v3.16.3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.6.1 + with: + version: v3.11.0 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --config ct.yaml) + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + + - name: Create kind cluster + uses: helm/kind-action@v1.12.0 + with: + version: v0.26.0 + cluster_name: chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} + node_image: kindest/node:${{ matrix.k8s-version }} + if: env.ACT || steps.list-changed.outputs.changed == 'true' + + - name: Install Minio + run: kubectl --context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} apply -f test-configs/minio/default.yaml + if: env.ACT || steps.list-changed.outputs.changed == 'true' + + - name: Wait for Minio to be Ready + run: kubectl --context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} -n minio wait --for=condition=Ready pod/minio --timeout=300s + if: env.ACT || steps.list-changed.outputs.changed == 'true' + + - name: Run chart-testing (install) + env: + LICENSE: ${{ secrets.LICENSE }} + if: steps.list-changed.outputs.changed == 'true' + run: | + ct install \ + --charts ./charts/operator-wandb \ + --config ct.yaml \ + --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}' \ + --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE' \ No newline at end of file diff --git a/.github/workflows/test-operator.yaml b/.github/workflows/test-operator.yaml new file mode 100644 index 00000000..e89b4665 --- /dev/null +++ b/.github/workflows/test-operator.yaml @@ -0,0 +1,54 @@ +# TODO(dpanzella): Uncomment this file when the ready for automated tests of the operator chart + +#name: Test operator Chart +# +#on: +# pull_request: +# paths: charts/operator +# +#jobs: +# test: +# runs-on: ubuntu-latest +# environment: Helm Charts +# steps: +# - name: Checkout +# uses: actions/checkout@v2 +# with: +# fetch-depth: 0 +# +# - name: Set up Helm +# uses: azure/setup-helm@v3.4 +# with: +# version: v3.16.3 +# +# # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and +# # yamllint (https://github.com/adrienverge/yamllint) which require Python +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: "3.10" +# +# - name: Set up chart-testing +# uses: helm/chart-testing-action@v2.6.1 +# with: +# version: v3.11.0 +# +# - name: Run chart-testing (list-changed) +# id: list-changed +# run: | +# changed=$(ct list-changed --config ct.yaml) +# if [[ -n "$changed" ]]; then +# echo "::set-output name=changed::true" +# fi +# +# - name: Create kind cluster +# uses: helm/kind-action@v1.8.0 +# with: +# version: v0.20.0 +# if: env.ACT || steps.list-changed.outputs.changed == 'true' +# +# - name: Run chart-testing (install) +# env: +# LICENSE: ${{ secrets.LICENSE }} +# if: steps.list-changed.outputs.changed == 'true' +# run: ct install --charts ./charts/operator --config ct.yaml --helm-extra-set-args --set=license=$LICENSE diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/test-wandb.yaml similarity index 91% rename from .github/workflows/lint-test.yaml rename to .github/workflows/test-wandb.yaml index ff32a20b..fd02783b 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/test-wandb.yaml @@ -1,9 +1,11 @@ -name: Lint and Test Charts +name: Test wandb Chart -on: pull_request +on: + pull_request: + paths: charts/wandb jobs: - lint-test: + test: runs-on: ubuntu-latest environment: Helm Charts steps: @@ -37,9 +39,6 @@ jobs: echo "::set-output name=changed::true" fi - - name: Run chart-testing (lint) - run: ct lint --config ct.yaml - - name: Create kind cluster uses: helm/kind-action@v1.8.0 with: diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml index bcdac359..c53990c9 100644 --- a/charts/operator-wandb/Chart.yaml +++ b/charts/operator-wandb/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: operator-wandb description: A Helm chart for deploying W&B to Kubernetes type: application -version: 0.24.8 +version: 0.24.9 appVersion: 1.0.0 icon: https://wandb.ai/logo.svg diff --git a/charts/operator-wandb/templates/tests/test-connection.yaml b/charts/operator-wandb/templates/tests/test-connection.yaml new file mode 100644 index 00000000..8d1345cd --- /dev/null +++ b/charts/operator-wandb/templates/tests/test-connection.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "wandb.fullname" . }}-test-connection" + labels: + {{- include "wandb.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wandb-verify + image: python:3.10 + env: + - name: WANDB_BASE_URL + value: "http://{{ .Release.Name }}-app:8080" + - name: WANDB_API_KEY + value: "test-api-key" + # wandb verify is terribly flaky, there's a PR up to fix it until then we try 3 times + command: + - sh + - -c + - "pip install wandb==0.17.8 && (wandb verify || (sleep 10 && wandb verify) || (sleep 10 && wandb verify))" + restartPolicy: Never diff --git a/test-configs/minio/default.yaml b/test-configs/minio/default.yaml new file mode 100644 index 00000000..2c8908e2 --- /dev/null +++ b/test-configs/minio/default.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: minio # Change this value if you want a different namespace name + labels: + name: minio # Change this value to match metadata.name +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + app: minio + name: minio + namespace: minio +spec: + containers: + - name: minio + image: quay.io/minio/minio:latest + env: + - name: MINIO_ACCESS_KEY + value: "minio" + - name: MINIO_SECRET_KEY + value: "minio123" + command: + - /bin/bash + - -c + args: + - mkdir -p /data/bucket && minio server /data --console-address :9090 + ports: + - containerPort: 9000 + name: minio + volumeMounts: + - mountPath: /data + name: localvolume # + volumes: + - name: localvolume + hostPath: + path: /mnt/minio/data + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: minio # Change this value to match the namespace metadata.name +spec: + selector: + app: minio + ports: + - port: 9000 + name: minio \ No newline at end of file diff --git a/test-configs/operator-wandb/default.yaml b/test-configs/operator-wandb/default.yaml new file mode 100644 index 00000000..cac3dd7c --- /dev/null +++ b/test-configs/operator-wandb/default.yaml @@ -0,0 +1,21 @@ +global: + bucket: + provider: "s3" + name: "minio.minio.svc.cluster.local:9000/bucket" + region: "us-east-1" + accessKey: "minio" + secretKey: "minio123" + +app: + extraEnv: + GLOBAL_ADMIN_API_KEY: "test-api-key" + GORILLA_INSECURE_ALLOW_API_KEY_ADMIN_ACCESS: "true" + +ingress: + install: false + create: false + +mysql: + install: true +redis: + install: true From e3ff218df1827bf272f3da7888b2dc1b1381d1ad Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 14:26:05 -0800 Subject: [PATCH 02/10] fix: Paths for testing jobs --- .github/workflows/lint.yaml | 2 +- .github/workflows/test-operator-wandb.yaml | 2 +- .github/workflows/test-operator.yaml | 2 +- .github/workflows/test-wandb.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 8b571b3c..6b6b07a3 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -2,7 +2,7 @@ name: Lint Charts on: pull_request: - paths: charts + paths: charts/** jobs: lint-test: diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml index 1b594090..b8e73780 100644 --- a/.github/workflows/test-operator-wandb.yaml +++ b/.github/workflows/test-operator-wandb.yaml @@ -2,7 +2,7 @@ name: Test operator-wandb Chart on: pull_request: - paths: charts/operator-wandb + paths: charts/operator-wandb/** jobs: test: diff --git a/.github/workflows/test-operator.yaml b/.github/workflows/test-operator.yaml index e89b4665..ee234ed0 100644 --- a/.github/workflows/test-operator.yaml +++ b/.github/workflows/test-operator.yaml @@ -4,7 +4,7 @@ # #on: # pull_request: -# paths: charts/operator +# paths: charts/operator/** # #jobs: # test: diff --git a/.github/workflows/test-wandb.yaml b/.github/workflows/test-wandb.yaml index fd02783b..8cf9dbb2 100644 --- a/.github/workflows/test-wandb.yaml +++ b/.github/workflows/test-wandb.yaml @@ -2,7 +2,7 @@ name: Test wandb Chart on: pull_request: - paths: charts/wandb + paths: charts/wandb/** jobs: test: From d32b209f9adf813c5146ea4913af5ad37baead2c Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 14:46:36 -0800 Subject: [PATCH 03/10] fix: test older mysql version --- charts/operator-wandb/charts/mysql/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/operator-wandb/charts/mysql/values.yaml b/charts/operator-wandb/charts/mysql/values.yaml index b779d139..1fa4cd6b 100644 --- a/charts/operator-wandb/charts/mysql/values.yaml +++ b/charts/operator-wandb/charts/mysql/values.yaml @@ -9,7 +9,7 @@ fullnameOverride: "" image: repository: mysql - tag: latest + tag: 8 pullPolicy: Always tolerations: [] From 43101056df816b640fc6b426bab4b5ef30e72834 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 14:55:53 -0800 Subject: [PATCH 04/10] fix: revert mysql version change, and try running tests on ubuntu 22.04 --- .github/workflows/test-operator-wandb.yaml | 2 +- charts/operator-wandb/charts/mysql/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml index b8e73780..5c0f35f4 100644 --- a/.github/workflows/test-operator-wandb.yaml +++ b/.github/workflows/test-operator-wandb.yaml @@ -7,11 +7,11 @@ on: jobs: test: name: Test Chart - runs-on: ubuntu-latest strategy: matrix: k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"] configuration: ["default"] + runs-on: ubuntu-22.04 environment: Helm Charts steps: - name: Checkout diff --git a/charts/operator-wandb/charts/mysql/values.yaml b/charts/operator-wandb/charts/mysql/values.yaml index 1fa4cd6b..b779d139 100644 --- a/charts/operator-wandb/charts/mysql/values.yaml +++ b/charts/operator-wandb/charts/mysql/values.yaml @@ -9,7 +9,7 @@ fullnameOverride: "" image: repository: mysql - tag: 8 + tag: latest pullPolicy: Always tolerations: [] From b34822a29d8cfcdc95043f5eab4840e7395ffc7c Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 15:10:45 -0800 Subject: [PATCH 05/10] fix: increase the timeout for helm install in tests --- .github/workflows/test-operator-wandb.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml index 5c0f35f4..187513b2 100644 --- a/.github/workflows/test-operator-wandb.yaml +++ b/.github/workflows/test-operator-wandb.yaml @@ -66,5 +66,5 @@ jobs: ct install \ --charts ./charts/operator-wandb \ --config ct.yaml \ - --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}' \ + --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} --timeout 600s' \ --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE' \ No newline at end of file From 4d02fef16f053500ccb7c6bbe9a655d107f96f34 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 15:35:52 -0800 Subject: [PATCH 06/10] fix: try adding a startup probe to mysql --- .../charts/mysql/templates/statefulset.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml index 438e0931..a3dcd453 100644 --- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml +++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml @@ -70,6 +70,15 @@ spec: - sh - -c - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}" + startupProbe: + initialDelaySeconds: 20 + periodSeconds: 5 + failureThreshold: 120 + exec: + command: + - sh + - -c + - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}" volumeMounts: - name: data mountPath: /var/lib/mysql From b1a36519ae8b7e0e019fdbfdc5ffbb63c853c8d8 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 20:31:37 -0800 Subject: [PATCH 07/10] fix: test removing security settings --- charts/operator-wandb/charts/mysql/templates/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml index a3dcd453..b3423867 100644 --- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml +++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml @@ -32,7 +32,7 @@ spec: containers: - name: mysql image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - {{- include "wandb.podSecurityContext" .Values.container.securityContext | nindent 10 }} + {{- /* include "wandb.podSecurityContext" .Values.container.securityContext | nindent 10 */ }} ports: - name: mysql containerPort: 3306 From 95a4e079d0d57e1293678a471af4cc38f5c68ed9 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 20:33:42 -0800 Subject: [PATCH 08/10] fix: test removing security settings --- charts/operator-wandb/charts/mysql/templates/statefulset.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml index b3423867..00fd6403 100644 --- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml +++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml @@ -32,7 +32,6 @@ spec: containers: - name: mysql image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - {{- /* include "wandb.podSecurityContext" .Values.container.securityContext | nindent 10 */ }} ports: - name: mysql containerPort: 3306 From 330ff9db75e0287ad93d1a0c9860cfebadbd83f1 Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 20:51:35 -0800 Subject: [PATCH 09/10] fix: try and older version of mysql --- .github/workflows/test-operator-wandb.yaml | 2 +- charts/operator-wandb/charts/mysql/templates/statefulset.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml index 187513b2..5c0f35f4 100644 --- a/.github/workflows/test-operator-wandb.yaml +++ b/.github/workflows/test-operator-wandb.yaml @@ -66,5 +66,5 @@ jobs: ct install \ --charts ./charts/operator-wandb \ --config ct.yaml \ - --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }} --timeout 600s' \ + --helm-extra-args '--kube-context kind-chart-testing-${{ matrix.k8s-version }}-${{ matrix.configuration }}' \ --helm-extra-set-args '--values test-configs/operator-wandb/${{ matrix.configuration }}.yaml --set=license=$LICENSE' \ No newline at end of file diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml index 00fd6403..02ad81e9 100644 --- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml +++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml @@ -31,7 +31,7 @@ spec: {{- include "wandb.podSecurityContext" .Values.pod.securityContext | nindent 6 }} containers: - name: mysql - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}:8.2" ports: - name: mysql containerPort: 3306 @@ -72,7 +72,7 @@ spec: startupProbe: initialDelaySeconds: 20 periodSeconds: 5 - failureThreshold: 120 + failureThreshold: 60 exec: command: - sh From 407728b7e944760a24cbc4ceafe3e23417e384ad Mon Sep 17 00:00:00 2001 From: Daniel Panzella Date: Sun, 2 Feb 2025 21:14:34 -0800 Subject: [PATCH 10/10] fix: revert some mysql changes and remove multiple k8s versions until I can get the tests to work in GA --- .github/workflows/test-operator-wandb.yaml | 2 +- charts/operator-wandb/charts/mysql/templates/statefulset.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-operator-wandb.yaml b/.github/workflows/test-operator-wandb.yaml index 5c0f35f4..784ae02b 100644 --- a/.github/workflows/test-operator-wandb.yaml +++ b/.github/workflows/test-operator-wandb.yaml @@ -9,7 +9,7 @@ jobs: name: Test Chart strategy: matrix: - k8s-version: ["v1.32.1", "v1.31.4", "v1.30.8"] + k8s-version: ["v1.32.1"] configuration: ["default"] runs-on: ubuntu-22.04 environment: Helm Charts diff --git a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml index 02ad81e9..707fdef7 100644 --- a/charts/operator-wandb/charts/mysql/templates/statefulset.yaml +++ b/charts/operator-wandb/charts/mysql/templates/statefulset.yaml @@ -31,7 +31,8 @@ spec: {{- include "wandb.podSecurityContext" .Values.pod.securityContext | nindent 6 }} containers: - name: mysql - image: "{{ .Values.image.repository }}:8.2" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + {{- include "wandb.podSecurityContext" .Values.container.securityContext | nindent 10 }} ports: - name: mysql containerPort: 3306