Wait until the nodes are ready

Author: ebattat

benchmark_runner/common/oc/oc.py

@@ -457,32 +457,51 @@ def get_worker_nodes(self):
         """
         return self.run(fr""" {self.__cli} get nodes -l node-role.kubernetes.io/worker= -o jsonpath="{{range .items[*]}}{{.metadata.name}}{{'\n'}}{{end}}" """)
 
-    @staticmethod
     @typechecked
-    def check_node_status(nodes_list: list):
+    def wait_for_nodes_ready(self, wait_time: int = None, timeout: int = int(environment_variables.environment_variables_dict['timeout'])):
         """
-        This method check node status
-        @param nodes_list:
-        @return: True when all nodes in ready status
+        This method waits until all nodes are in 'Ready' status.
+        @param wait_time: wait time between each loop
+        @param timeout: Maximum wait time in seconds (default set in environment variables)
+        @return: True if all nodes are in 'Ready' status within the timeout period
+        @raises: NodeNotReady if one or more nodes are not ready within the timeout
+        """
+        wait_time = wait_time or OC.SHORT_TIMEOUT
+        all_nodes_ready = None
+        current_wait_time = 0
+        while timeout <= 0 or current_wait_time < timeout:
+            all_nodes_ready = self.check_all_nodes_status()
+            if all_nodes_ready is True:
+                return True
+            logger.info(f"Waiting for node '{all_nodes_ready[0]}' to reach 'Ready' status, current status '{all_nodes_ready[1]}'.")
+            time.sleep(wait_time)
+            current_wait_time += wait_time
+        logger.info(f"oc get nodes:\n{self.run('oc get nodes')}")
+        raise NodeNotReady(node_name=all_nodes_ready[0], node_status=all_nodes_ready[1])
+
+    @typechecked
+    def check_all_nodes_status(self):
+        """
+        This method checks the status of all nodes in the list.
+        @return: True if all nodes are in 'Ready' status; otherwise, return (node_name, node_status)
         """
         # Check if any node is not in 'Ready' status
-        for node in nodes_list:
+        for node in self.get_node_status():
             node_name, node_status = node.split()
             if node_status != 'Ready':
-                raise NodeNotReady(node_name, node_status)
+                return node_name, node_status
 
         # If no nodes are found in a non-ready state
         return True
 
-    def verify_nodes_ready(self):
+    def get_node_status(self) -> list:
         """
-        This method verifies that all nodes are in 'Ready' status.
-        If any node is not ready, it raises an error with the node name and its status.
-        @return: True is all in 'Ready' status
+        This method returns node status list
+        @return:
         """
         # Get the node name and status for all nodes
         nodes_list = self.run(f"{self.__cli} get nodes --no-headers | awk '{{print $1, $2}}'").splitlines()
-        return self.check_node_status(nodes_list)
+        return nodes_list
 
     def delete_available_released_pv(self):
         """

benchmark_runner/main/main.py

@@ -175,7 +175,7 @@ def upgrade_ocp_bare_metal(step: str):
     elif step == 'verify_bare_metal_upgrade_complete':
         if bare_metal_operations.is_cluster_upgraded(oc, cnv_version=cnv_version, odf_version=odf_version, lso_version=lso_version):
             bare_metal_operations.verify_cluster_is_up(oc)
-            oc.verify_nodes_ready()
+            oc.wait_for_nodes_ready()
         else:
             error_message = f'OCP {upgrade_ocp_version} upgrade failed'
             logger.error(error_message)
@@ -200,7 +200,7 @@ def install_resources():
     logger.info(f'Start Bare-Metal OpenShift resources installation')
     oc = bare_metal_operations.oc_login()
     bare_metal_operations.verify_cluster_is_up(oc)
-    oc.verify_nodes_ready()
+    oc.wait_for_nodes_ready()
     bare_metal_operations.install_ocp_resources(resources=resources)
     bare_metal_operations.disconnect_from_provisioner()
     logger.info(f'End Bare-Metal OpenShift resources installation')

tests/integration/benchmark_runner/common/oc/test_oc.py

@@ -234,11 +234,11 @@ def test_collect_prometheus():
         assert tarfile.is_tarfile(tarball)
 
 
-def test_verify_nodes_ready():
+def test_wait_for_nodes_ready():
     """
-    This method test nodes are ready
+    This method waits till nodes are ready
     @return:
     """
     oc = OC(kubeadmin_password=test_environment_variable['kubeadmin_password'])
     oc.login()
-    assert oc.verify_nodes_ready()
+    assert oc.wait_for_nodes_ready()

tests/unittest/benchmark_runner/common/oc/test_oc.py

@@ -1,6 +1,8 @@
 
 import mock
 import pytest
+from unittest.mock import patch
+
 from benchmark_runner.common.oc.oc import OC
 from benchmark_runner.common.oc.oc_exceptions import YAMLNotExist, LoginFailed, NodeNotReady
 
@@ -51,13 +53,15 @@ def test_short_uuid():
 
 def test_check_node_status_ready():
     oc = OC()
-    result = oc.check_node_status(nodes_list=['node-0 Ready', 'node-1 Ready', 'node-2 Ready'])
-    assert result
+    with patch.object(OC, 'get_node_status', return_value=['node-0 Ready', 'node-1 Ready', 'node-2 Ready']):
+        result = oc.wait_for_nodes_ready()
+        assert result
 
 
 def test_check_node_status_not_ready():
     oc = OC()
     with pytest.raises(NodeNotReady) as exc_info:
-        oc.check_node_status(nodes_list=['node-0 Ready', 'node-1 NotReady', 'node-2 Ready'])
+        with patch.object(OC, 'get_node_status', return_value=['node-0 Ready', 'node-1 NotReady', 'node-2 Ready']):
+            oc.wait_for_nodes_ready(wait_time=3, timeout=10)
     # Check that the exception message is as expected
     assert str(exc_info.value) == "Node node-1 is not ready. Current status: NotReady"