Synchronize kubeflow pipelines manifests 2.4.0 (#2953)

* Update kubeflow/pipelines manifests from 2.4.0 Signed-off-by: juliusvonkohout <[email protected]> * update the kfp version Signed-off-by: juliusvonkohout <[email protected]> * update metacontroller nly to 2.6.1 and use the kubeflow-edit clusterrole instead of cluster-admin. Signed-off-by: juliusvonkohout <[email protected]> * add checks for metacontroller Signed-off-by: juliusvonkohout <[email protected]> * revert to cluster-admin for now and add more tests.git status Signed-off-by: juliusvonkohout <[email protected]> * more tests Signed-off-by: juliusvonkohout <[email protected]> * more tests Signed-off-by: juliusvonkohout <[email protected]> * more tests Signed-off-by: juliusvonkohout <[email protected]> * more tests Signed-off-by: juliusvonkohout <[email protected]> * more tests Signed-off-by: juliusvonkohout <[email protected]> * more tests Signed-off-by: juliusvonkohout <[email protected]> * try kubeflow-admin instead of cluster-admin Signed-off-by: juliusvonkohout <[email protected]> * explicit clusterrole Signed-off-by: juliusvonkohout <[email protected]> --------- Signed-off-by: juliusvonkohout <[email protected]>
kubeflow · Jan 31, 2025 · 0259e7b · 0259e7b
1 parent 2f8fea0
commit 0259e7b
Show file tree

Hide file tree

Showing 39 changed files with 418 additions and 53 deletions.
diff --git a/.github/workflows/pipeline_swfs_test.yaml b/.github/workflows/pipeline_swfs_test.yaml
@@ -48,7 +48,32 @@ jobs:
       run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f -
 
     - name: Create KF Profile
-      run: kustomize build common/user-namespace/base | kubectl apply -f -
+      run: |
+        kustomize build common/user-namespace/base | kubectl apply -f -
+        sleep 30 # for the metacontroller to create the secret
+
+        METACONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("metacontroller")) | .metadata.name')
+        if [[ -z "$METACONTROLLER_POD" ]]; then
+          echo "Error: metacontroller pod not found in kubeflow namespace."
+          exit 1
+        fi
+        kubectl logs -n kubeflow "$METACONTROLLER_POD"
+
+        PIPELINES_PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("kubeflow-pipelines-profile-controller")) | .metadata.name')
+        if [[ -z "$PIPELINES_PROFILE_CONTROLLER_POD" ]]; then
+          echo "Error: kubeflow-pipelines-profile-controller pod not found in kubeflow namespace."
+          exit 1
+        fi
+        kubectl logs -n kubeflow "$PIPELINES_PROFILE_CONTROLLER_POD"
+
+        KF_PROFILE=kubeflow-user-example-com
+        kubectl -n $KF_PROFILE get pods,configmaps,secrets
+
+        if ! kubectl get secret mlpipeline-minio-artifact -n $KF_PROFILE > /dev/null 2>&1; then
+          echo "Error: Secret mlpipeline-minio-artifact not found in namespace $KF_PROFILE"
+          exit 1
+        fi
+        kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json | jq -r '.data | keys[] as $k | "\($k): \(. | .[$k] | @base64d)"' | tr '\n' ' '
 
     - name: Install seaweedfs
       run: |

diff --git a/.github/workflows/pipeline_test.yaml b/.github/workflows/pipeline_test.yaml
@@ -47,7 +47,32 @@ jobs:
       run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f -
 
     - name: Create KF Profile
-      run: kustomize build common/user-namespace/base | kubectl apply -f -
+      run: |
+        kustomize build common/user-namespace/base | kubectl apply -f -
+        sleep 30 # for the metacontroller to create the secret
+
+        METACONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("metacontroller")) | .metadata.name')
+        if [[ -z "$METACONTROLLER_POD" ]]; then
+          echo "Error: metacontroller pod not found in kubeflow namespace."
+          exit 1
+        fi
+        kubectl logs -n kubeflow "$METACONTROLLER_POD"
+
+        PIPELINES_PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("kubeflow-pipelines-profile-controller")) | .metadata.name')
+        if [[ -z "$PIPELINES_PROFILE_CONTROLLER_POD" ]]; then
+          echo "Error: kubeflow-pipelines-profile-controller pod not found in kubeflow namespace."
+          exit 1
+        fi
+        kubectl logs -n kubeflow "$PIPELINES_PROFILE_CONTROLLER_POD"
+
+        KF_PROFILE=kubeflow-user-example-com
+        kubectl -n $KF_PROFILE get pods,configmaps,secrets
+
+        if ! kubectl get secret mlpipeline-minio-artifact -n $KF_PROFILE > /dev/null 2>&1; then
+          echo "Error: Secret mlpipeline-minio-artifact not found in namespace $KF_PROFILE"
+          exit 1
+        fi
+        kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json | jq -r '.data | keys[] as $k | "\($k): \(. | .[$k] | @base64d)"' | tr '\n' ' '
 
     - name: port forward
       run: |

diff --git a/README.md b/README.md
@@ -56,7 +56,7 @@ This repo periodically syncs all official Kubeflow components from their respect
 | Katib | apps/katib/upstream | [v0.17.0](https://github.com/kubeflow/katib/tree/v0.17.0/manifests/v1beta1) |
 | KServe | contrib/kserve/kserve | [v0.14.1](https://github.com/kserve/kserve/releases/tag/v0.14.1/install/v0.14.1) |
 | KServe Models Web App | contrib/kserve/models-web-app | [0.13.0](https://github.com/kserve/models-web-app/tree/0.13.0/config) |
-| Kubeflow Pipelines | apps/pipeline/upstream | [2.3.0](https://github.com/kubeflow/pipelines/tree/2.3.0/manifests/kustomize) |
+| Kubeflow Pipelines | apps/pipeline/upstream | [2.4.0](https://github.com/kubeflow/pipelines/tree/2.4.0/manifests/kustomize) |
 | Kubeflow Model Registry | apps/model-registry/upstream | [v0.2.12](https://github.com/kubeflow/model-registry/tree/v0.2.12/manifests/kustomize) |
 
 The following is also a matrix with versions from common components that are

diff --git a/apps/pipeline/upstream/base/cache-deployer/cache-deployer-deployment.yaml b/apps/pipeline/upstream/base/cache-deployer/cache-deployer-deployment.yaml
@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
       - name: main
-        image: gcr.io/ml-pipeline/cache-deployer:dummy
+        image: ghcr.io/kubeflow/kfp-cache-deployer:dummy
         imagePullPolicy: Always
         env:
         - name: NAMESPACE_TO_WATCH

diff --git a/apps/pipeline/upstream/base/cache-deployer/kustomization.yaml b/apps/pipeline/upstream/base/cache-deployer/kustomization.yaml
@@ -7,5 +7,5 @@ resources:
 commonLabels:
   app: cache-deployer
 images:
-  - name: gcr.io/ml-pipeline/cache-deployer
-    newTag: 2.3.0
+  - name: ghcr.io/kubeflow/kfp-cache-deployer
+    newTag: 2.4.0
diff --git a/apps/pipeline/upstream/base/cache/cache-deployment.yaml b/apps/pipeline/upstream/base/cache/cache-deployment.yaml
@@ -16,7 +16,17 @@ spec:
     spec:
       containers:
       - name: server
-        image: gcr.io/ml-pipeline/cache-server:dummy
+        securityContext:
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          runAsNonRoot: true
+          runAsUser: 1000
+          runAsGroup: 0
+          capabilities:
+            drop:
+            - ALL
+        image: ghcr.io/kubeflow/kfp-cache-server:dummy
         env:
           - name: DEFAULT_CACHE_STALENESS
             valueFrom:

diff --git a/apps/pipeline/upstream/base/cache/kustomization.yaml b/apps/pipeline/upstream/base/cache/kustomization.yaml
@@ -9,5 +9,5 @@ resources:
 commonLabels:
   app: cache-server
 images:
-  - name: gcr.io/ml-pipeline/cache-server
-    newTag: 2.3.0
+  - name: ghcr.io/kubeflow/kfp-cache-server
+    newTag: 2.4.0
diff --git a/apps/pipeline/upstream/base/installs/generic/pipeline-install-config.yaml b/apps/pipeline/upstream/base/installs/generic/pipeline-install-config.yaml
@@ -11,7 +11,7 @@ data:
     until the changes take effect. A quick way to restart all deployments in a
     namespace: `kubectl rollout restart deployment -n <your-namespace>`.
   appName: pipeline
-  appVersion: 2.3.0
+  appVersion: 2.4.0
   dbHost: mysql # relic to be removed after release
   dbPort: "3306" # relic to be removed after release
   dbType: mysql
@@ -63,8 +63,8 @@ data:
   ## cacheImage is the image that the mutating webhook will use to patch
   ## cached steps with. Will be used to echo a message announcing that 
   ## the cached step result will be used. If not set it will default to 
-  ## 'gcr.io/google-containers/busybox'
-  cacheImage: "gcr.io/google-containers/busybox"
+  ## 'registry.k8s.io/busybox'
+  cacheImage: "registry.k8s.io/busybox"
   ## cacheNodeRestrictions the dummy container runing if output is cached
   ## will run with the same affinity and node selector as the default pipeline
   ## step. This is defaulted to 'false' to allow the pod to be scheduled on 

diff --git a/apps/pipeline/upstream/base/installs/generic/postgres/pipeline-install-config.yaml b/apps/pipeline/upstream/base/installs/generic/postgres/pipeline-install-config.yaml
@@ -63,8 +63,8 @@ data:
   ## cacheImage is the image that the mutating webhook will use to patch
   ## cached steps with. Will be used to echo a message announcing that 
   ## the cached step result will be used. If not set it will default to 
-  ## 'gcr.io/google-containers/busybox'
-  cacheImage: "gcr.io/google-containers/busybox"
+  ## 'registry.k8s.io/busybox'
+  cacheImage: "registry.k8s.io/busybox"
   ## cacheNodeRestrictions the dummy container runing if output is cached
   ## will run with the same affinity and node selector as the default pipeline
   ## step. This is defaulted to 'false' to allow the pod to be scheduled on 

diff --git a/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/deployment.yaml b/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/deployment.yaml
@@ -11,7 +11,7 @@ spec:
     spec:
       containers:
       - name: profile-controller
-        image: python:3.7
+        image: python:3.9
         command: ["python", "/hooks/sync.py"]
         envFrom:
         - configMapRef:

diff --git a/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py b/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py
@@ -36,9 +36,9 @@ def get_settings_from_env(controller_port=None,
     Settings are pulled from the all-caps version of the setting name.  The
     following defaults are used if those environment variables are not set
     to enable backwards compatibility with previous versions of this script:
-        visualization_server_image: gcr.io/ml-pipeline/visualization-server
+        visualization_server_image: ghcr.io/kubeflow/kfp-visualization-server
         visualization_server_tag: value of KFP_VERSION environment variable
-        frontend_image: gcr.io/ml-pipeline/frontend
+        frontend_image: ghcr.io/kubeflow/kfp-frontend
         frontend_tag: value of KFP_VERSION environment variable
         disable_istio_sidecar: Required (no default)
         minio_access_key: Required (no default)
@@ -51,11 +51,11 @@ def get_settings_from_env(controller_port=None,
 
     settings["visualization_server_image"] = \
         visualization_server_image or \
-        os.environ.get("VISUALIZATION_SERVER_IMAGE", "gcr.io/ml-pipeline/visualization-server")
+        os.environ.get("VISUALIZATION_SERVER_IMAGE", "ghcr.io/kubeflow/kfp-visualization-server")
 
     settings["frontend_image"] = \
         frontend_image or \
-        os.environ.get("FRONTEND_IMAGE", "gcr.io/ml-pipeline/frontend")
+        os.environ.get("FRONTEND_IMAGE", "ghcr.io/kubeflow/kfp-frontend")
 
     # Look for specific tags for each image first, falling back to
     # previously used KFP_VERSION environment variable for backwards

diff --git a/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py b/apps/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/test_sync.py
@@ -49,8 +49,8 @@
 DATA_MISSING_PIPELINE_ENABLED = {"parent": {}, "children": {}}
 
 # Default values when environments are not explicit
-DEFAULT_FRONTEND_IMAGE = "gcr.io/ml-pipeline/frontend"
-DEFAULT_VISUALIZATION_IMAGE = "gcr.io/ml-pipeline/visualization-server"
+DEFAULT_FRONTEND_IMAGE = "ghcr.io/kubeflow/kfp-frontend"
+DEFAULT_VISUALIZATION_IMAGE = "ghcr.io/kubeflow/kfp-visualization-server"
 
 # Variables used for environment variable sets
 VISUALIZATION_SERVER_IMAGE = "vis-image"

diff --git a/apps/pipeline/upstream/base/metadata/base/kustomization.yaml b/apps/pipeline/upstream/base/metadata/base/kustomization.yaml
@@ -9,5 +9,5 @@ resources:
   - metadata-envoy-service.yaml
   - metadata-grpc-sa.yaml
 images:
-  - name: gcr.io/ml-pipeline/metadata-envoy
-    newTag: 2.3.0
+  - name: ghcr.io/kubeflow/kfp-metadata-envoy
+    newTag: 2.4.0
diff --git a/apps/pipeline/upstream/base/metadata/base/metadata-envoy-deployment.yaml b/apps/pipeline/upstream/base/metadata/base/metadata-envoy-deployment.yaml
@@ -18,9 +18,19 @@ spec:
     spec:
       containers:
       - name: container
-        image: gcr.io/ml-pipeline/metadata-envoy:dummy
+        image: ghcr.io/kubeflow/kfp-metadata-envoy:dummy
         ports:
         - name: md-envoy
           containerPort: 9090
         - name: envoy-admin
           containerPort: 9901
+        securityContext:
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          runAsNonRoot: true
+          runAsUser: 1000
+          runAsGroup: 0
+          capabilities:
+            drop:
+            - ALL
diff --git a/apps/pipeline/upstream/base/metadata/base/metadata-grpc-deployment.yaml b/apps/pipeline/upstream/base/metadata/base/metadata-grpc-deployment.yaml
@@ -23,6 +23,16 @@ spec:
         # * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml
         # * test/tag_for_hosted.sh
         image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0
+        securityContext:
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          runAsNonRoot: true
+          runAsUser: 1000
+          runAsGroup: 0
+          capabilities:
+            drop:
+            - ALL
         env:
         - name: DBCONFIG_USER
           valueFrom:

diff --git a/apps/pipeline/upstream/base/pipeline/kustomization.yaml b/apps/pipeline/upstream/base/pipeline/kustomization.yaml
@@ -35,15 +35,15 @@ resources:
   - viewer-sa.yaml
   - kfp-launcher-configmap.yaml
 images:
-  - name: gcr.io/ml-pipeline/api-server
-    newTag: 2.3.0
-  - name: gcr.io/ml-pipeline/persistenceagent
-    newTag: 2.3.0
-  - name: gcr.io/ml-pipeline/scheduledworkflow
-    newTag: 2.3.0
-  - name: gcr.io/ml-pipeline/frontend
-    newTag: 2.3.0
-  - name: gcr.io/ml-pipeline/viewer-crd-controller
-    newTag: 2.3.0
-  - name: gcr.io/ml-pipeline/visualization-server
-    newTag: 2.3.0
+  - name: ghcr.io/kubeflow/kfp-api-server
+    newTag: 2.4.0
+  - name: ghcr.io/kubeflow/kfp-persistence-agent
+    newTag: 2.4.0
+  - name: ghcr.io/kubeflow/kfp-scheduled-workflow-controller
+    newTag: 2.4.0
+  - name: ghcr.io/kubeflow/kfp-frontend
+    newTag: 2.4.0
+  - name: ghcr.io/kubeflow/kfp-viewer-crd-controller
+    newTag: 2.4.0
+  - name: ghcr.io/kubeflow/kfp-visualization-server
+    newTag: 2.4.0
diff --git a/apps/pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml b/apps/pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml
@@ -6,5 +6,5 @@ resources:
   - metadata-writer-rolebinding.yaml
   - metadata-writer-sa.yaml
 images:
-  - name: gcr.io/ml-pipeline/metadata-writer
-    newTag: 2.3.0
+  - name: ghcr.io/kubeflow/kfp-metadata-writer
+    newTag: 2.4.0
diff --git a/apps/pipeline/upstream/base/pipeline/metadata-writer/metadata-writer-deployment.yaml b/apps/pipeline/upstream/base/pipeline/metadata-writer/metadata-writer-deployment.yaml
@@ -16,10 +16,20 @@ spec:
     spec:
       containers:
       - name: main
-        image: gcr.io/ml-pipeline/metadata-writer:dummy
+        image: ghcr.io/kubeflow/kfp-metadata-writer:dummy
         env:
         - name: NAMESPACE_TO_WATCH
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        securityContext:
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          runAsNonRoot: true
+          runAsUser: 1000
+          runAsGroup: 0
+          capabilities:
+            drop:
+            - ALL
       serviceAccountName: kubeflow-pipelines-metadata-writer
diff --git a/apps/pipeline/upstream/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/apps/pipeline/upstream/base/pipeline/ml-pipeline-apiserver-deployment.yaml
@@ -109,7 +109,7 @@ spec:
             secretKeyRef:
               name: mlpipeline-minio-artifact
               key: secretkey
-        image: gcr.io/ml-pipeline/api-server:dummy
+        image: ghcr.io/kubeflow/kfp-api-server:dummy
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-api-server
         ports:
@@ -156,6 +156,16 @@ spec:
           failureThreshold: 12
           periodSeconds: 5
           timeoutSeconds: 2
+        securityContext:
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          runAsNonRoot: true
+          runAsUser: 1000
+          runAsGroup: 0
+          capabilities:
+            drop:
+            - ALL
         resources:
           requests:
             cpu: 250m

diff --git a/apps/pipeline/upstream/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml b/apps/pipeline/upstream/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml
@@ -27,7 +27,7 @@ spec:
             value: "2"
           - name: LOG_LEVEL
             value: "info"
-        image: gcr.io/ml-pipeline/persistenceagent:dummy
+        image: ghcr.io/kubeflow/kfp-persistence-agent:dummy
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-persistenceagent
         resources:
@@ -37,6 +37,16 @@ spec:
         volumeMounts:
           - mountPath: /var/run/secrets/kubeflow/tokens
             name: persistenceagent-sa-token
+        securityContext:
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          runAsNonRoot: true
+          runAsUser: 1000
+          runAsGroup: 0
+          capabilities:
+            drop:
+            - ALL
       serviceAccountName: ml-pipeline-persistenceagent
       volumes:
         - name: persistenceagent-sa-token

diff --git a/apps/pipeline/upstream/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml b/apps/pipeline/upstream/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml
@@ -16,7 +16,7 @@ spec:
         cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
     spec:
       containers:
-      - image: gcr.io/ml-pipeline/scheduledworkflow:dummy
+      - image: ghcr.io/kubeflow/kfp-scheduled-workflow-controller:dummy
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-scheduledworkflow
         env:
@@ -31,4 +31,14 @@ spec:
               configMapKeyRef:
                 name: pipeline-install-config
                 key: cronScheduleTimezone
+        securityContext:
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          runAsNonRoot: true
+          runAsUser: 1000
+          runAsGroup: 0
+          capabilities:
+            drop:
+            - ALL
       serviceAccountName: ml-pipeline-scheduledworkflow