diff --git a/pipelines/folioRancher/folioDevTools/tenantManagement/createConsortia/Jenkinsfile b/pipelines/folioRancher/folioDevTools/tenantManagement/createConsortia/Jenkinsfile index 371a78d293..8e09d9924b 100644 --- a/pipelines/folioRancher/folioDevTools/tenantManagement/createConsortia/Jenkinsfile +++ b/pipelines/folioRancher/folioDevTools/tenantManagement/createConsortia/Jenkinsfile @@ -70,9 +70,8 @@ ansiColor('xterm') { stage('[Helm] Deploy backend') { folioHelm.withKubeConfig(namespace.getClusterName()) { folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getBackendModules()) - folioHelm.checkAllPodsRunning(namespace.getNamespaceName()) + folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getBackendModules()) } - sleep time: 3, unit: 'MINUTES' } stage('[Rest] Create tenants') { diff --git a/pipelines/folioRancher/folioNamespaceTools/manageNamespace/createDailySnapshotECS/Jenkinsfile b/pipelines/folioRancher/folioNamespaceTools/manageNamespace/createDailySnapshotECS/Jenkinsfile index ca6a35f63b..bc26db40d1 100644 --- a/pipelines/folioRancher/folioNamespaceTools/manageNamespace/createDailySnapshotECS/Jenkinsfile +++ b/pipelines/folioRancher/folioNamespaceTools/manageNamespace/createDailySnapshotECS/Jenkinsfile @@ -185,6 +185,7 @@ ansiColor('xterm') { stage('Update') { folioHelm.withKubeConfig(namespace.getClusterName()) { folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getBackendModules()) + folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getBackendModules()) folioEdge.renderEphemeralProperties(namespace) namespace.getModules().getEdgeModules().each { module -> @@ -193,7 +194,7 @@ ansiColor('xterm') { } folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getEdgeModules()) - folioHelm.checkAllPodsRunning(namespace.getNamespaceName()) + folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getEdgeModules()) } } diff --git a/vars/folioDeployFlow.groovy b/vars/folioDeployFlow.groovy index 204441abb9..f9dfd6fbdb 100644 --- a/vars/folioDeployFlow.groovy +++ b/vars/folioDeployFlow.groovy @@ -41,9 +41,9 @@ void backend(RancherNamespace namespace, Closure preStages = { -> }, Closure pos stage('[Helm] Deploy backend') { folioHelm.withKubeConfig(namespace.getClusterName()) { folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getBackendModules()) -// folioHelm.checkAllPodsRunning(namespace.getNamespaceName()) + folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getBackendModules()) } - pauseBetweenStages(10) + //pauseBetweenStages(10) } postStages() @@ -76,6 +76,7 @@ void edge(RancherNamespace namespace, boolean skipEdgeUsersCreation = false, Clo kubectl.createConfigMap("${module.name}-ephemeral-properties", namespace.getNamespaceName(), "./${module.name}-ephemeral-properties") } folioHelm.deployFolioModulesParallel(namespace, edgeModules) + folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), edgeModules) } pauseBetweenStages() } diff --git a/vars/folioHelm.groovy b/vars/folioHelm.groovy index 1a42cfc70d..d605e75299 100644 --- a/vars/folioHelm.groovy +++ b/vars/folioHelm.groovy @@ -153,6 +153,87 @@ void checkAllPodsRunning(String ns) { } } +void checkDeploymentsRunning(String ns, FolioModule deploymentModule) { + checkDeploymentsRunning(ns, [deploymentModule]) +} + +void checkDeploymentsRunning(String ns, List deploymentsList) { + println('Starting deployment monitoring...') + + boolean allDeploymentsUpdated = false + int timer = 0 + int maxTime = 10 * 60 // 10 minutes in seconds + + try { + while (!allDeploymentsUpdated) { + def jsonOutput + try { + // Execute the kubectl command + jsonOutput = sh( + script: "kubectl get deployments -n ${ns} -o json", + returnStdout: true + ).trim() + } catch (Exception e) { + error("Failed to execute kubectl command: ${e.message}") + } + + def deploymentsJson + try { + // Parse the JSON output + deploymentsJson = readJSON text: jsonOutput + } catch (Exception e) { + error("Failed to parse JSON output: ${e.message}") + } + + // Check if there are any deployments in the namespace + if (!deploymentsJson.items || deploymentsJson.items.isEmpty()) { + error("No deployments found in namespace '${ns}'. Please check the namespace or deployment configuration.") + } + + def unfinishedDeployments = [] + + // Check each deployment from the list + deploymentsList.each { folioModule -> + def deployment = deploymentsJson.items.find { it.metadata.name == folioModule.name } + if (deployment) { + def status = deployment.status + def specReplicas = deployment.spec.replicas + if (status.updatedReplicas != specReplicas || + status.readyReplicas != specReplicas || + status.unavailableReplicas > 0 || + status.conditions.any { it.type == "Available" && it.status == "False" }) { + unfinishedDeployments << folioModule.name + } + } else { + println("Warning: Deployment '${folioModule.name}' not found in namespace '${ns}'") + } + } + + if (unfinishedDeployments) { + println("Unfinished deployments: ${unfinishedDeployments}") + println("Rechecking in 30 seconds...") + sleep(time: 30, unit: 'SECONDS') + timer += 30 + } else { + println("All deployments are successfully updated!") + allDeploymentsUpdated = true + } + + // Check the timer + if (timer >= maxTime) { + error("Timeout: Some deployments are still not updated after 10 minutes.") + } + } + } catch (Exception e) { + // Handle general errors + println("Error occurred during deployment monitoring: ${e.message}") + throw e // Rethrow the error to mark the Jenkins build as failed + } +} + + + + static String valuesPathOption(String path) { return path.trim() ? "-f ${path}" : '' } diff --git a/vars/folioNamespaceCreate.groovy b/vars/folioNamespaceCreate.groovy index 3909c598f0..141c6cb3eb 100644 --- a/vars/folioNamespaceCreate.groovy +++ b/vars/folioNamespaceCreate.groovy @@ -136,6 +136,7 @@ void call(CreateNamespaceParameters args) { folioHelm.withKubeConfig(namespace.getClusterName()) { folioHelm.deployFolioModulesParallel(namespace, namespace.getModules().getBackendModules()) // folioHelm.checkAllPodsRunning(namespace.getNamespaceName()) + folioHelm.checkDeploymentsRunning(namespace.getNamespaceName(), namespace.getModules().getBackendModules()) } }