diff --git a/test/e2e/pipeline/infra/cluster-api/cni.sh b/test/e2e/pipeline/infra/cluster-api/cni.sh index fab1acff3d..e10754aa03 100644 --- a/test/e2e/pipeline/infra/cluster-api/cni.sh +++ b/test/e2e/pipeline/infra/cluster-api/cni.sh @@ -7,6 +7,9 @@ WORKDIR=$(dirname "$FILEPATH") # shellcheck source=./pre-requirements.sh source "$WORKDIR/pre-requirements.sh" +# shellcheck source=../../utils.sh +source "$WORKDIR/../../utils.sh" + DOCKER_PROXY="${DOCKER_PROXY:-docker.io}" function install_calico() { @@ -56,11 +59,17 @@ EOF function wait_calico() { local kubeconfig=$1 - sleep 5 - "${KUBECTL}" wait --for condition=Ready=true -n calico-system pod --all --kubeconfig "$kubeconfig" --timeout=-1s - sleep 10 + if ! waitandretry 5s 12 "${KUBECTL} wait --for condition=Ready=true -n calico-system pod --all --kubeconfig $kubeconfig --timeout=-1s" + then + echo "Failed to wait for calico pods to be ready" + exit 1 + fi # set felix to use different port for VXLAN - "${KUBECTL}" patch felixconfiguration default --type='merge' -p '{"spec":{"vxlanPort": 6789}}' --kubeconfig "$kubeconfig" + if ! waitandretry 5s 12 "${KUBECTL} patch felixconfiguration default --type='merge' -p '{\"spec\":{\"vxlanPort\": 6789}}' --kubeconfig $kubeconfig"; + then + echo "Failed to patch felixconfiguration" + exit 1 + fi } function install_cilium() { @@ -112,6 +121,9 @@ function install_flannel() { function wait_flannel() { local kubeconfig=$1 - sleep 15 - "${KUBECTL}" wait --for condition=Ready=true -n kube-flannel pod --all --timeout=-1s --kubeconfig "$kubeconfig" + if ! waitandretry 5s 12 "${KUBECTL} wait --for condition=Ready=true -n kube-flannel pod --all --timeout=-1s --kubeconfig $kubeconfig"; + then + echo "Failed to wait for flannel pods to be ready" + exit 1 + fi } diff --git a/test/e2e/pipeline/utils.sh b/test/e2e/pipeline/utils.sh new file mode 100755 index 0000000000..35eaade00a --- /dev/null +++ b/test/e2e/pipeline/utils.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +#shellcheck disable=SC1091 + +# Define the retry function +waitandretry() { + local waittime="$1" + local retries="$2" + local command="$3" + local options="$-" # Get the current "set" options + + sleep "${waittime}" + + echo "Running command: ${command} (retries left: ${retries})" + + # Disable set -e + if [[ $options == *e* ]]; then + set +e + fi + + # Run the command, and save the exit code + $command + local exit_code=$? + + # restore initial options + if [[ $options == *e* ]]; then + set -e + fi + + # If the exit code is non-zero (i.e. command failed), and we have not + # reached the maximum number of retries, run the command again + if [[ $exit_code -ne 0 && $retries -gt 0 ]]; then + waitandretry "$waittime" $((retries - 1)) "$command" + else + # Return the exit code from the command + return $exit_code + fi +}