RANCHER-1970: Implement pods/deployments status check for helm deploy…

…ment stages (#751) * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: revert temp changes * RANCHER-1970: revert temp changes * RANCHER-1970: rename method * RANCHER-1970: rename method * Update logic for input parameters * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: add checkAllDeploymentsRunning status method * RANCHER-1970: revert temp changes * RANCHER-1970: revert temp changes * RANCHER-1970: rename method * RANCHER-1970: rename method * Update logic for input parameters * RANCHER-1970: replace pod method to new one --------- Co-authored-by: Oleksandr Haimanov <[email protected]>
folio-org · Dec 25, 2024 · aae6641 · aae6641
1 parent fe1e601
commit aae6641
Show file tree

Hide file tree

Showing 5 changed files with 88 additions and 5 deletions.
diff --git a/pipelines/folioRancher/folioDevTools/tenantManagement/createConsortia/Jenkinsfile b/pipelines/folioRancher/folioDevTools/tenantManagement/createConsortia/Jenkinsfile
@@ -70,9 +70,8 @@ ansiColor('xterm') {
       stage('[Helm] Deploy backend') {
         folioHelm.withKubeConfig(namespace.getClusterName()) {
           folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getBackendModules())
-          folioHelm.checkAllPodsRunning(namespace.getNamespaceName())
+          folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getBackendModules())
         }
-        sleep time: 3, unit: 'MINUTES'
       }
 
       stage('[Rest] Create tenants') {

diff --git a/...lines/folioRancher/folioNamespaceTools/manageNamespace/createDailySnapshotECS/Jenkinsfile b/...lines/folioRancher/folioNamespaceTools/manageNamespace/createDailySnapshotECS/Jenkinsfile
@@ -185,6 +185,7 @@ ansiColor('xterm') {
       stage('Update') {
         folioHelm.withKubeConfig(namespace.getClusterName()) {
           folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getBackendModules())
+          folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getBackendModules())
 
           folioEdge.renderEphemeralProperties(namespace)
           namespace.getModules().getEdgeModules().each { module ->
@@ -193,7 +194,7 @@ ansiColor('xterm') {
           }
 
           folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getEdgeModules())
-          folioHelm.checkAllPodsRunning(namespace.getNamespaceName())
+          folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getEdgeModules())
         }
       }
 

diff --git a/vars/folioDeployFlow.groovy b/vars/folioDeployFlow.groovy
@@ -41,9 +41,9 @@ void backend(RancherNamespace namespace, Closure preStages = { -> }, Closure pos
   stage('[Helm] Deploy backend') {
     folioHelm.withKubeConfig(namespace.getClusterName()) {
       folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getBackendModules())
-//      folioHelm.checkAllPodsRunning(namespace.getNamespaceName())
+      folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getBackendModules())
     }
-    pauseBetweenStages(10)
+    //pauseBetweenStages(10)
   }
 
   postStages()
@@ -76,6 +76,7 @@ void edge(RancherNamespace namespace, boolean skipEdgeUsersCreation = false, Clo
         kubectl.createConfigMap("${module.name}-ephemeral-properties", namespace.getNamespaceName(), "./${module.name}-ephemeral-properties")
       }
       folioHelm.deployFolioModulesParallel(namespace, edgeModules)
+      folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), edgeModules)
     }
     pauseBetweenStages()
   }

diff --git a/vars/folioHelm.groovy b/vars/folioHelm.groovy
@@ -153,6 +153,87 @@ void checkAllPodsRunning(String ns) {
   }
 }
 
+void checkDeploymentsRunning(String ns, FolioModule deploymentModule) {
+  checkDeploymentsRunning(ns, [deploymentModule])
+}
+
+void checkDeploymentsRunning(String ns, List<FolioModule> deploymentsList) {
+  println('Starting deployment monitoring...')
+
+  boolean allDeploymentsUpdated = false
+  int timer = 0
+  int maxTime = 10 * 60 // 10 minutes in seconds
+
+  try {
+    while (!allDeploymentsUpdated) {
+      def jsonOutput
+      try {
+        // Execute the kubectl command
+        jsonOutput = sh(
+          script: "kubectl get deployments -n ${ns} -o json",
+          returnStdout: true
+        ).trim()
+      } catch (Exception e) {
+        error("Failed to execute kubectl command: ${e.message}")
+      }
+
+      def deploymentsJson
+      try {
+        // Parse the JSON output
+        deploymentsJson = readJSON text: jsonOutput
+      } catch (Exception e) {
+        error("Failed to parse JSON output: ${e.message}")
+      }
+
+      // Check if there are any deployments in the namespace
+      if (!deploymentsJson.items || deploymentsJson.items.isEmpty()) {
+        error("No deployments found in namespace '${ns}'. Please check the namespace or deployment configuration.")
+      }
+
+      def unfinishedDeployments = []
+
+      // Check each deployment from the list
+      deploymentsList.each { folioModule ->
+        def deployment = deploymentsJson.items.find { it.metadata.name == folioModule.name }
+        if (deployment) {
+          def status = deployment.status
+          def specReplicas = deployment.spec.replicas
+          if (status.updatedReplicas != specReplicas ||
+              status.readyReplicas != specReplicas ||
+              status.unavailableReplicas > 0 ||
+              status.conditions.any { it.type == "Available" && it.status == "False" }) {
+            unfinishedDeployments << folioModule.name
+          }
+        } else {
+          println("Warning: Deployment '${folioModule.name}' not found in namespace '${ns}'")
+        }
+      }
+
+      if (unfinishedDeployments) {
+        println("Unfinished deployments: ${unfinishedDeployments}")
+        println("Rechecking in 30 seconds...")
+        sleep(time: 30, unit: 'SECONDS')
+        timer += 30
+      } else {
+        println("All deployments are successfully updated!")
+        allDeploymentsUpdated = true
+      }
+
+      // Check the timer
+      if (timer >= maxTime) {
+        error("Timeout: Some deployments are still not updated after 10 minutes.")
+      }
+    }
+  } catch (Exception e) {
+    // Handle general errors
+    println("Error occurred during deployment monitoring: ${e.message}")
+    throw e // Rethrow the error to mark the Jenkins build as failed
+  }
+}
+
+
+
+
 static String valuesPathOption(String path) {
   return path.trim() ? "-f ${path}" : ''
 }

diff --git a/vars/folioNamespaceCreate.groovy b/vars/folioNamespaceCreate.groovy
@@ -136,6 +136,7 @@ void call(CreateNamespaceParameters args) {
       folioHelm.withKubeConfig(namespace.getClusterName()) {
         folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getBackendModules())
 //        folioHelm.checkAllPodsRunning(namespace.getNamespaceName())
+        folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getBackendModules())
       }
     }