Refactor: Extract pipeline_test.py into a shared file for reuse across workflows #85
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy and test Kubeflow Pipelines manifests with seaweedfs and m2m auth in KinD | |
| on: | |
| pull_request: | |
| paths: | |
| - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh | |
| - .github/workflows/pipeline_swfs_test.yaml | |
| - apps/pipeline/upstream/** | |
| - tests/gh-actions/install_istio.sh | |
| - tests/gh-actions/install_cert_manager.sh | |
| - tests/gh-actions/install_oauth2-proxy.sh | |
| - common/cert-manager/** | |
| - common/oauth2-proxy/** | |
| - common/istio*/** | |
| - contrib/seaweedfs/** | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install KinD, Create KinD cluster and Install kustomize | |
| run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh | |
| - name: Install kubectl | |
| run: ./tests/gh-actions/install_kubectl.sh | |
| - name: Install Istio | |
| run: ./tests/gh-actions/install_istio.sh | |
| - name: Install oauth2-proxy | |
| run: ./tests/gh-actions/install_oauth2-proxy.sh | |
| - name: Install cert-manager | |
| run: ./tests/gh-actions/install_cert_manager.sh | |
| - name: Create kubeflow namespace | |
| run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - | |
| - name: Install KF Pipelines | |
| run: ./tests/gh-actions/install_pipelines.sh | |
| - name: Install KF Multi Tenancy | |
| run: ./tests/gh-actions/install_multi_tenancy.sh | |
| - name: Install kubeflow-istio-resources | |
| run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f - | |
| - name: Create KF Profile | |
| run: | | |
| kustomize build common/user-namespace/base | kubectl apply -f - | |
| sleep 30 # for the metacontroller to create the secret | |
| METACONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("metacontroller")) | .metadata.name') | |
| if [[ -z "$METACONTROLLER_POD" ]]; then | |
| echo "Error: metacontroller pod not found in kubeflow namespace." | |
| exit 1 | |
| fi | |
| kubectl logs -n kubeflow "$METACONTROLLER_POD" | |
| PIPELINES_PROFILE_CONTROLLER_POD=$(kubectl get pods -n kubeflow -o json | jq -r '.items[] | select(.metadata.name | startswith("kubeflow-pipelines-profile-controller")) | .metadata.name') | |
| if [[ -z "$PIPELINES_PROFILE_CONTROLLER_POD" ]]; then | |
| echo "Error: kubeflow-pipelines-profile-controller pod not found in kubeflow namespace." | |
| exit 1 | |
| fi | |
| kubectl logs -n kubeflow "$PIPELINES_PROFILE_CONTROLLER_POD" | |
| KF_PROFILE=kubeflow-user-example-com | |
| kubectl -n $KF_PROFILE get pods,configmaps,secrets | |
| if ! kubectl get secret mlpipeline-minio-artifact -n $KF_PROFILE > /dev/null 2>&1; then | |
| echo "Error: Secret mlpipeline-minio-artifact not found in namespace $KF_PROFILE" | |
| exit 1 | |
| fi | |
| kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json | jq -r '.data | keys[] as $k | "\($k): \(. | .[$k] | @base64d)"' | tr '\n' ' ' | |
| - name: Install seaweedfs | |
| run: | | |
| kustomize build contrib/seaweedfs/istio | kubectl apply -f - | |
| kubectl -n kubeflow wait --for=condition=available --timeout=600s deploy/seaweedfs | |
| kubectl -n kubeflow exec deploy/seaweedfs -c seaweedfs -- sh -c "echo \"s3.configure -user minio -access_key minio -secret_key minio123 -actions Read,Write,List -apply\" | /usr/bin/weed shell" | |
| - name: port forward | |
| run: | | |
| ingress_gateway_service=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}') | |
| nohup kubectl port-forward --namespace istio-system svc/${ingress_gateway_service} 8080:80 & | |
| while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready | |
| - name: List and deploy test pipeline with authorized ServiceAccount Token | |
| run: | | |
| KF_PROFILE=kubeflow-user-example-com | |
| TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)" | |
| python3 tests/gh-actions/pipeline_test.py run_pipeline "${TOKEN}" "${KF_PROFILE}" |