OpenShift installer retry mechanism1 (#893)

redhat-performance · Sep 19, 2024 · 37df5b9 · 37df5b9
1 parent e84dedc
commit 37df5b9
Showing 1 changed file with 42 additions and 13 deletions.
diff --git a/.github/workflows/Weekly_Perf_Env_Installer_CI.yml b/.github/workflows/Weekly_Perf_Env_Installer_CI.yml
@@ -1,7 +1,7 @@
 
 # https://crontab.guru/
 # Deploy OCP SNO Performance environment using Assisted Installer - run on Thursday
-name: Deploy Assisted Installer Perf Env Weekly CI
+name: Deploy Assisted installer Perf Env Weekly CI
 
 on:
   schedule:
@@ -73,19 +73,48 @@ jobs:
           CONTAINER_PRIVATE_KEY_PATH: ${{ secrets.CONTAINER_PRIVATE_KEY_PATH }}
           RUNNER_PATH: ${{ secrets.RUNNER_PATH }}
         run: |
+          # Retry function
+          retry() {
+            local max_attempts=3
+            local count=0
+            until [ $count -ge $max_attempts ]
+            do
+              podman run --rm \
+                -e OCP_ENV_FLAVOR="PERF" \
+                -e IBM_API_KEY="$IBM_API_KEY" \
+                -e GITHUB_TOKEN="$GITHUB_TOKEN" \
+                -e INSTALL_OCP_VERSION="$INSTALL_OCP_VERSION" \
+                -e INSTALL_STEP="${{ matrix.step }}" \
+                -e WORKER_IDS="$WORKER_IDS" \
+                -e PROVISION_IP="$PROVISION_IP" \
+                -e CONTAINER_PRIVATE_KEY_PATH="$CONTAINER_PRIVATE_KEY_PATH" \
+                -e PROVISION_USER="$PROVISION_USER" \
+                -e PROVISION_PORT="$PROVISION_PORT" \
+                -e PROVISION_KUBEADMIN_PASSWORD_PATH="$PROVISION_KUBEADMIN_PASSWORD_PATH" \
+                -e PROVISION_KUBECONFIG_PATH="$PROVISION_KUBECONFIG_PATH" \
+                -e PROVISION_INSTALLER_PATH="$PROVISION_INSTALLER_PATH" \
+                -e PROVISION_INSTALLER_CMD="$PROVISION_INSTALLER_CMD" \
+                -e PROVISION_INSTALLER_LOG="$PROVISION_INSTALLER_LOG" \
+                -e GITHUB_REPOSITORY_SHORT="$GITHUB_REPOSITORY_SHORT" \
+                -e PROVISION_TIMEOUT="3600" \
+                -e log_level="INFO" \
+                -v "$PROVISION_PRIVATE_KEY_PATH":"$CONTAINER_PRIVATE_KEY_PATH" \
+                -v "/$RUNNER_PATH/.ssh/config":"/$PROVISION_USER/.ssh/config" \
+                --privileged "${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}"
+
+              if [ $? -eq 0 ]; then
+                return 0
+              fi
+
+              count=$((count + 1))
+              echo "Retrying ($count/$max_attempts)..."
+              sleep 10
+            done
+            return 1
+          }
+
           build=$(pip freeze | grep benchmark-runner | sed 's/==/=/g')
           build_version="$(cut -d'=' -f2 <<<"$build")"
           echo '>>>>>>>>>>>>>>>>>>>>>>>>>> OCP start step: ${{ matrix.step }} >>>>>>>>>>>>>>>>>>>>>>>>>>>>'
-          retries=3
-          count=0
-          until podman run --rm -e OCP_ENV_FLAVOR="PERF" -e IBM_API_KEY="$IBM_API_KEY" -e GITHUB_TOKEN="$GITHUB_TOKEN" -e INSTALL_OCP_VERSION="$INSTALL_OCP_VERSION" -e INSTALL_STEP="${{ matrix.step }}" -e WORKER_IDS="$WORKER_IDS" -e PROVISION_IP="$PROVISION_IP" -e CONTAINER_PRIVATE_KEY_PATH="$CONTAINER_PRIVATE_KEY_PATH" -e CONTAINER_PRIVATE_KEY_PATH="$CONTAINER_PRIVATE_KEY_PATH" -e PROVISION_USER="$PROVISION_USER" -e PROVISION_PORT="$PROVISION_PORT" -e PROVISION_KUBEADMIN_PASSWORD_PATH="$PROVISION_KUBEADMIN_PASSWORD_PATH" -e PROVISION_KUBECONFIG_PATH="$PROVISION_KUBECONFIG_PATH" -e PROVISION_INSTALLER_PATH="$PROVISION_INSTALLER_PATH" -e PROVISION_INSTALLER_CMD="$PROVISION_INSTALLER_CMD" -e PROVISION_INSTALLER_LOG="$PROVISION_INSTALLER_LOG" -e GITHUB_REPOSITORY_SHORT="$GITHUB_REPOSITORY_SHORT" --privileged "${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}"; do
-            count=$((count + 1))
-            if [ $count -lt $retries ]; then
-              echo "Retry $count/$retries failed, retrying in 10 seconds..."
-              sleep 10
-            else
-              echo "Step failed after $retries retries."
-              exit 1
-            fi
-          done
+          retry
           echo '>>>>>>>>>>>>>>>>>>>>>>>>>> OCP end step: ${{ matrix.step }}   >>>>>>>>>>>>>>>>>>>>>>>>>>>>'