Wait until the nodes are ready (#924)

Author: ebattat

benchmark_runner/common/oc/oc.py

@@ -457,32 +457,61 @@ def get_worker_nodes(self):
         """
         return self.run(fr""" {self.__cli} get nodes -l node-role.kubernetes.io/worker= -o jsonpath="{{range .items[*]}}{{.metadata.name}}{{'\n'}}{{end}}" """)
 
-    @staticmethod
     @typechecked
-    def check_node_status(nodes_list: list):
+    def wait_for_node_ready(self, node: str = None, wait_time: int = None, timeout: int = int(environment_variables.environment_variables_dict['timeout'])):
         """
-        This method check node status
-        @param nodes_list:
-        @return: True when all nodes in ready status
+        This method waits until all nodes are in 'Ready' status or specific node
+        @param node: wait for specific node to be ready, when None check all nodes
+        @param wait_time: wait time between each loop
+        @param timeout: Maximum wait time in seconds, negative value means no timeout (default set in environment variables)
+        @return: True if all nodes are in 'Ready' status within the timeout period
+        @raises: NodeNotReady if one or more nodes are not ready within the timeout
         """
-        # Check if any node is not in 'Ready' status
-        for node in nodes_list:
-            node_name, node_status = node.split()
+        wait_time = wait_time or OC.SHORT_TIMEOUT
+        nodes_status = None
+        current_wait_time = 0
+        while timeout <= 0 or current_wait_time < timeout:
+            nodes_status = self.check_node_status(node=node)
+            if nodes_status is True:
+                return True
+            logger.info(f"Waiting for '{nodes_status}' to reach 'Ready' status")
+            time.sleep(wait_time)
+            current_wait_time += wait_time
+        logger.info(f"oc get nodes:\n{self.run('oc get nodes')}")
+        raise NodeNotReady(nodes_status=nodes_status)
+
+    @typechecked
+    def check_node_status(self, node: str = None):
+        """
+        This method checks the status of all nodes or a specific node.
+        @param node: The name of a specific node to check for "Ready" status; if None, check all nodes.
+        @return: True if all nodes are in 'Ready' status, or a dictionary of nodes that are not in 'Ready' status.
+        """
+        not_ready_nodes = {}
+
+        for node_state in self.get_node_status():
+            node_name, node_status = node_state.split()
+
+            # If a specific node is given, only check that node
+            if node and node != node_name:
+                continue
+
             if node_status != 'Ready':
-                raise NodeNotReady(node_name, node_status)
+                not_ready_nodes[node_name] = node_status
+                # If checking a specific node and it's not ready, no need to check further
+                if node:
+                    break
 
-        # If no nodes are found in a non-ready state
-        return True
+        return True if not not_ready_nodes else not_ready_nodes
 
-    def verify_nodes_ready(self):
+    def get_node_status(self) -> list:
         """
-        This method verifies that all nodes are in 'Ready' status.
-        If any node is not ready, it raises an error with the node name and its status.
-        @return: True is all in 'Ready' status
+        This method returns node status list
+        @return:
         """
         # Get the node name and status for all nodes
         nodes_list = self.run(f"{self.__cli} get nodes --no-headers | awk '{{print $1, $2}}'").splitlines()
-        return self.check_node_status(nodes_list)
+        return nodes_list
 
     def delete_available_released_pv(self):
         """

benchmark_runner/common/oc/oc_exceptions.py

@@ -167,7 +167,7 @@ def __init__(self):
 
 
 class NodeNotReady(OCError):
-    """This exception returns node not ready timeout error"""
-    def __init__(self, node_name, node_status):
-        self.message = f"Node {node_name} is not ready. Current status: {node_status}"
+    """This exception indicates a node not ready due to a timeout error"""
+    def __init__(self, nodes_status: dict):
+        self.message = f"Node not ready: {nodes_status}"
         super(NodeNotReady, self).__init__(self.message)

benchmark_runner/main/main.py

@@ -175,7 +175,7 @@ def upgrade_ocp_bare_metal(step: str):
     elif step == 'verify_bare_metal_upgrade_complete':
         if bare_metal_operations.is_cluster_upgraded(oc, cnv_version=cnv_version, odf_version=odf_version, lso_version=lso_version):
             bare_metal_operations.verify_cluster_is_up(oc)
-            oc.verify_nodes_ready()
+            oc.wait_for_node_ready()
         else:
             error_message = f'OCP {upgrade_ocp_version} upgrade failed'
             logger.error(error_message)
@@ -200,7 +200,7 @@ def install_resources():
     logger.info(f'Start Bare-Metal OpenShift resources installation')
     oc = bare_metal_operations.oc_login()
     bare_metal_operations.verify_cluster_is_up(oc)
-    oc.verify_nodes_ready()
+    oc.wait_for_node_ready()
     bare_metal_operations.install_ocp_resources(resources=resources)
     bare_metal_operations.disconnect_from_provisioner()
     logger.info(f'End Bare-Metal OpenShift resources installation')

tests/integration/benchmark_runner/common/oc/test_oc.py

@@ -234,11 +234,11 @@ def test_collect_prometheus():
         assert tarfile.is_tarfile(tarball)
 
 
-def test_verify_nodes_ready():
+def test_wait_for_nodes_ready():
     """
-    This method test nodes are ready
+    This method waits till nodes are ready
     @return:
     """
     oc = OC(kubeadmin_password=test_environment_variable['kubeadmin_password'])
     oc.login()
-    assert oc.verify_nodes_ready()
+    assert oc.wait_for_node_ready()

tests/unittest/benchmark_runner/common/oc/test_oc.py

@@ -1,6 +1,8 @@
 
 import mock
 import pytest
+from unittest.mock import patch
+
 from benchmark_runner.common.oc.oc import OC
 from benchmark_runner.common.oc.oc_exceptions import YAMLNotExist, LoginFailed, NodeNotReady
 
@@ -49,15 +51,33 @@ def test_short_uuid():
         assert oc._OC__get_short_uuid(workload='stressng_pod') == 'bb2be20e'
 
 
-def test_check_node_status_ready():
+def test_check_all_nodes_status_ready():
     oc = OC()
-    result = oc.check_node_status(nodes_list=['node-0 Ready', 'node-1 Ready', 'node-2 Ready'])
-    assert result
+    with patch.object(OC, 'get_node_status', return_value=['node-0 Ready', 'node-1 Ready', 'node-2 Ready']):
+        result = oc.wait_for_node_ready()
+        assert result
 
 
-def test_check_node_status_not_ready():
+def test_check_all_nodes_status_not_ready():
     oc = OC()
     with pytest.raises(NodeNotReady) as exc_info:
-        oc.check_node_status(nodes_list=['node-0 Ready', 'node-1 NotReady', 'node-2 Ready'])
+        with patch.object(OC, 'get_node_status', return_value=['node-0 NotReady', 'node-1 NotReady', 'node-2 Ready']):
+            oc.wait_for_node_ready(wait_time=3, timeout=10)
     # Check that the exception message is as expected
-    assert str(exc_info.value) == "Node node-1 is not ready. Current status: NotReady"
+    assert str(exc_info.value) == "Node not ready: {'node-0': 'NotReady', 'node-1': 'NotReady'}"
+
+
+def test_check_not_ready_node_status_not_ready():
+    oc = OC()
+    with pytest.raises(NodeNotReady) as exc_info:
+        with patch.object(OC, 'get_node_status', return_value=['node-0 NotReady', 'node-1 NotReady', 'node-2 Ready']):
+            oc.wait_for_node_ready(node='node-1', wait_time=3, timeout=10)
+    # Check that the exception message is as expected
+    assert str(exc_info.value) == "Node not ready: {'node-1': 'NotReady'}"
+
+
+def test_check_ready_node_status_not_ready():
+    oc = OC()
+    with patch.object(OC, 'get_node_status', return_value=['node-0 NotReady', 'node-1 NotReady', 'node-2 Ready']):
+        result = oc.wait_for_node_ready(node='node-2', wait_time=3, timeout=10)
+        assert result