From d3e70c134727c94a8570f87d80df808fd44f63fc Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 28 Jan 2025 00:57:35 +0100 Subject: [PATCH] include postupgrade cluster state checking in install2 tests (#1749) * include postupgrade cluster state checking in install2 tests * f * ff * run install2 upgrade tests * update openebs install2 image metadata * create 'TestAlwaysFail' as a placeholder in the 'failing tests' matrix --- .github/workflows/ci.yaml | 14 ++-- e2e/install2_test.go | 91 ++++-------------------- e2e/scripts/check-postupgrade-state2.sh | 90 +++++++++++++++++++++++ pkg/addons2/openebs/static/metadata.yaml | 12 ++-- 4 files changed, 117 insertions(+), 90 deletions(-) create mode 100755 e2e/scripts/check-postupgrade-state2.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ac59e99a3..b38bfaea0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -903,10 +903,10 @@ jobs: fail-fast: false matrix: test: - - TestSingleNodeInstall2UbuntuJammy - - TestSingleNodeInstall2AlmaLinux8 - - TestSingleNodeInstall2Debian11 - - TestSingleNodeInstall2Debian12 + - TestSingleNodeInstall2UpgradeUbuntuJammy + - TestSingleNodeInstall2UpgradeAlmaLinux8 + - TestSingleNodeInstall2UpgradeDebian11 + - TestSingleNodeInstall2UpgradeDebian12 steps: - name: Checkout uses: actions/checkout@v4 @@ -980,10 +980,7 @@ jobs: fail-fast: false matrix: test: - - TestSingleNodeInstall2UpgradeUbuntuJammy - - TestSingleNodeInstall2UpgradeAlmaLinux8 - - TestSingleNodeInstall2UpgradeDebian11 - - TestSingleNodeInstall2UpgradeDebian12 + - TestAlwaysFail steps: - name: Checkout uses: actions/checkout@v4 @@ -1097,6 +1094,7 @@ jobs: matrix: test: - TestSingleNodeAirgapInstall2Upgrade + - TestAlwaysFail steps: - name: Checkout uses: actions/checkout@v4 diff --git a/e2e/install2_test.go b/e2e/install2_test.go index a3dc28ad5..6efd89471 100644 --- a/e2e/install2_test.go +++ b/e2e/install2_test.go @@ -58,7 +58,11 @@ func singleNodeInstallUpgradeTest(t *testing.T, tc cluster.Cluster, additionalAr t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } - // TODO: check postupgrade installation state + t.Logf("%s: checking postuprgrade state for an install2 cluster", time.Now().Format(time.RFC3339)) + line = []string{"check-postupgrade-state2.sh", fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), k8sVersion()} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) + } t.Logf("%s: resetting admin console password", time.Now().Format(time.RFC3339)) newPassword := "newpass" @@ -71,81 +75,6 @@ func singleNodeInstallUpgradeTest(t *testing.T, tc cluster.Cluster, additionalAr require.NoError(t, err, "unable to login with the new password") } -func TestSingleNodeInstall2UbuntuJammy(t *testing.T) { - t.Parallel() - - RequireEnvVars(t, []string{"SHORT_SHA"}) - - tc := docker.NewCluster(&docker.ClusterInput{ - T: t, - Nodes: 1, - Distro: "ubuntu-jammy", - LicensePath: "license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", - }) - defer tc.Cleanup() - singleNodeInstallTest(t, tc, nil) - t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) -} - -func TestSingleNodeInstall2AlmaLinux8(t *testing.T) { - t.Parallel() - - RequireEnvVars(t, []string{"SHORT_SHA"}) - - tc := docker.NewCluster(&docker.ClusterInput{ - T: t, - Nodes: 1, - Distro: "almalinux-8", - LicensePath: "license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", - }) - defer tc.Cleanup() - - t.Logf("%s: installing tar", time.Now().Format(time.RFC3339)) - line := []string{"yum-install-tar.sh"} - if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to check postupgrade state: %v: %s: %s", err, stdout, stderr) - } - - singleNodeInstallTest(t, tc, nil) - t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) -} - -func TestSingleNodeInstall2Debian11(t *testing.T) { - t.Parallel() - - RequireEnvVars(t, []string{"SHORT_SHA"}) - - tc := docker.NewCluster(&docker.ClusterInput{ - T: t, - Nodes: 1, - Distro: "debian-bullseye", - LicensePath: "license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", - }) - defer tc.Cleanup() - singleNodeInstallTest(t, tc, nil) - t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) -} - -func TestSingleNodeInstall2Debian12(t *testing.T) { - t.Parallel() - - RequireEnvVars(t, []string{"SHORT_SHA"}) - - tc := docker.NewCluster(&docker.ClusterInput{ - T: t, - Nodes: 1, - Distro: "debian-bookworm", - LicensePath: "license.yaml", - ECBinaryPath: "../output/bin/embedded-cluster", - }) - defer tc.Cleanup() - singleNodeInstallTest(t, tc, nil) - t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) -} - func TestSingleNodeInstall2UpgradeUbuntuJammy(t *testing.T) { t.Parallel() @@ -221,6 +150,16 @@ func TestSingleNodeInstall2UpgradeDebian12(t *testing.T) { t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } +// this test is used as a a placeholder for tests that run as part of the 'not yet implemented, thus expected to fail' list +// github actions requires that the list of tests is not empty, so this test is always included in case there are no other tests +// that are not yet implemented. +func TestAlwaysFail(t *testing.T) { + t.Parallel() + + t.Logf("This test always fails") + t.Fail() +} + func TestSingleNodeAirgapInstall2(t *testing.T) { t.Parallel() diff --git a/e2e/scripts/check-postupgrade-state2.sh b/e2e/scripts/check-postupgrade-state2.sh new file mode 100755 index 000000000..d82309ad7 --- /dev/null +++ b/e2e/scripts/check-postupgrade-state2.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -euox pipefail + +DIR=/usr/local/bin +. $DIR/common.sh + +function check_nginx_version { + if ! kubectl describe pod -n ingress-nginx | grep -q "4.12.0-beta.0"; then + return 1 + fi + return 0 +} + +main() { + local version="$1" + local k8s_version="$2" + + echo "TODO: check installation configmap state" + + echo "pods" + kubectl get pods -A + + # the airgap version of this command does not require the kots CLI, which is why it is used here + if ! ensure_app_deployed_airgap "$version"; then + echo "failed to find that version $version was deployed" + exit 1 + fi + + # ensure that memcached exists + if ! kubectl get ns memcached; then + echo "no memcached ns found" + kubectl get ns + exit 1 + fi + + # ensure that memcached pods exist + if ! kubectl get pods -n memcached | grep -q Running ; then + echo "no pods found for memcached deployment" + kubectl get pods -n memcached + exit 1 + fi + + # ensure that new app pods exist + # wait for new app pods to be running + if ! retry 5 eval "kubectl get pods -n $APP_NAMESPACE -l app=second | grep -q Running" ; then + echo "no pods found for second app version" + kubectl get pods -n "$APP_NAMESPACE" + exit 1 + fi + + # ensure that nginx-ingress has been updated + if ! retry 5 check_nginx_version ; then + echo "4.12.0-beta.0 not found in ingress-nginx pod" + kubectl describe pod -n ingress-nginx + exit 1 + fi + + # TODO: validate that labels are added after upgrading from an older version + echo "ensure that the admin console branding is available" + kubectl get cm -n kotsadm kotsadm-application-metadata + + echo "ensure that the kotsadm deployment exists" + kubectl get deployment -n kotsadm kotsadm + + echo "ensure the kotsadm statefulset does not exist" + if kubectl get statefulset -n kotsadm kotsadm; then + echo "kotsadm statefulset found" + kubectl get statefulset -n kotsadm kotsadm + exit 1 + fi + + echo "ensure the kotsadm-minio statefulset does not exist" + if kubectl get statefulset -n kotsadm kotsadm-minio; then + echo "kotsadm-minio statefulset found" + kubectl get statefulset -n kotsadm kotsadm-minio + exit 1 + fi + + echo "ensure that all nodes are running k8s $k8s_version" + if ! ensure_nodes_match_kube_version "$k8s_version"; then + echo "not all nodes are running k8s $k8s_version" + exit 1 + fi + + validate_data_dirs + + validate_no_pods_in_crashloop +} + +main "$@" diff --git a/pkg/addons2/openebs/static/metadata.yaml b/pkg/addons2/openebs/static/metadata.yaml index 3216bd398..5c17c3a6a 100644 --- a/pkg/addons2/openebs/static/metadata.yaml +++ b/pkg/addons2/openebs/static/metadata.yaml @@ -11,15 +11,15 @@ images: kubectl: repo: proxy.replicated.com/anonymous/replicated/ec-kubectl tag: - amd64: 1.32.1-r0-amd64@sha256:041b86cd6ce6776169261978a5d2cfecdb9cc3b826cc6ba011b498ea613c53cd - arm64: 1.32.1-r0-arm64@sha256:3d268ced4b5d1afdde2b9d9119bcbea2fa2775b8dae1a015729f66f5a0edfbdb + amd64: 1.32.1-r2-amd64@sha256:9b4d77b99dbea2f8026e06c1a59b22091b33c939c90b984f85cf35b2e28e84f0 + arm64: 1.32.1-r2-arm64@sha256:d9956fdb4acced920a2c42583f84a46d6bf4a61c9dee8176165afec504cf1464 openebs-linux-utils: repo: proxy.replicated.com/anonymous/replicated/ec-openebs-linux-utils tag: - amd64: 4.1.1-amd64@sha256:5f835d38f548cfabf0254ddd583a5de83ff60b78e5885dd66edfa0c03216dabd - arm64: 4.1.1-arm64@sha256:cf5014c390ea71ffbbbb4dde802b135567e823139738a62f99c422767e844be6 + amd64: 4.1.1-amd64@sha256:bf1b31734e44281d8dbe9dcf183c0982a729f21b30f9ad03460fe3371dc89a2e + arm64: 4.1.1-arm64@sha256:33cbd23af395263bd4dfeb8fa942d52747769651822133f903f2a2d0b951d9a1 openebs-provisioner-localpv: repo: proxy.replicated.com/anonymous/replicated/ec-openebs-provisioner-localpv tag: - amd64: 4.1.2-r2-amd64@sha256:a3b96568cca25384e89100b2a7b75c02176cb1a684f238d5202d30b8007c6d78 - arm64: 4.1.2-r2-arm64@sha256:931b37f95a4c069d0d7993d529458ab2c456d0e3f5eba77517871803f7614dd9 + amd64: 4.1.4-r0-amd64@sha256:c1f70b9d595199a00629f2a93162c0f482ba235e1fa8a08bb745f6d9ec82e1c7 + arm64: 4.1.4-r0-arm64@sha256:324d8115e7b2bbd891a3ffd79e813320ebcf16cba15bc4bdc19fe5bfc52fe628