Merge pull request #288 from fabric-testbed/287.maint

Maintenance Mode Issues

Author: kthare10

fabric_cf/actor/core/container/maintenance.py

@@ -24,7 +24,7 @@
 #
 # Author: Komal Thareja ([email protected])
 from datetime import datetime, timezone
-from typing import List, Dict, Tuple
+from typing import List, Dict, Tuple, Union
 
 from fim.slivers.maintenance_mode import MaintenanceInfo, MaintenanceState
 
@@ -92,16 +92,56 @@ def is_worker_in_maintenance(self, *, worker: str) -> bool:
     def __str__(self):
         return f"Name: {self.name} MaintInfo: {self.maintenance_info} Properties: {self.properties}"
 
+    def clone_maintenance_info(self) -> Union[MaintenanceInfo or None]:
+        if self.maintenance_info is not None:
+            return self.maintenance_info.copy()
+        return None
+
+    def update_maintenance_info(self, maint_info: MaintenanceInfo):
+        self.maintenance_info = maint_info
+
 
 class Maintenance:
     @staticmethod
     def update_maintenance_mode(*, database: ABCDatabase, properties: Dict[str, str], sites: List[Site] = None):
+        """
+        Update Maintenance Mode at Testbed/Site/Worker Level
+        - Tesbed level Maintenance - single Site object is passed with Name = ALL
+        - Site level Maintenance - single Site object per site is passed with Name = SiteName
+        - Worker level Maintenance - single Site object per site with one entry per worker
+        @param database database
+        @param properties properties container project ids/ user emails
+        @param sites Maintenance information for the sites
+        """
         for s in sites:
+            # Set the list of allowed projects/users at the site level
             if properties is not None:
                 s.set_properties(properties=properties)
 
-            if database.get_site(site_name=s.get_name()) is not None:
-                database.update_site(site=s)
+            # Get Current Maintenance mode for the Site
+            existing_site = database.get_site(site_name=s.get_name())
+            # Site entry exists
+            if existing_site is not None:
+                # Site level Maintenance Update
+                if s.get_maintenance_info().get(s.get_name()) is not None:
+                    database.update_site(site=s)
+                # Worker level Maintenance Update
+                else:
+                    new_maint_info = existing_site.clone_maintenance_info()
+                    if new_maint_info.get(s.get_name()):
+                        new_maint_info.rem(s.get_name())
+                    for worker_name, entry in s.get_maintenance_info().list_details():
+                        # Remove existing entry
+                        if new_maint_info.get(worker_name):
+                            new_maint_info.rem(worker_name)
+
+                        # Add worker entry using the new information only if worker is in Maintenance
+                        if entry.state != MaintenanceState.Active:
+                            new_maint_info.add(worker_name, entry)
+                    new_maint_info.finalize()
+                    existing_site.update_maintenance_info(maint_info=new_maint_info)
+                    database.update_site(site=existing_site)
+            # Adding Maintenance State First Time
             else:
                 database.add_site(site=s)
 
@@ -137,7 +177,7 @@ def is_sliver_provisioning_allowed(*, database: ABCDatabase, project: str, email
         """
         status, site = Maintenance.is_site_in_maintenance(database=database, site_name=site)
 
-        if not status:
+        if not status and site is None:
             return True, None
 
         projects = site.get_properties().get(Constants.PROJECT_ID)

fabric_cf/actor/core/policy/broker_simpler_units_policy.py

@@ -47,6 +47,7 @@
 from fabric_cf.actor.core.apis.abc_delegation import ABCDelegation
 from fabric_cf.actor.core.apis.abc_reservation_mixin import ABCReservationMixin
 from fabric_cf.actor.core.common.constants import Constants
+from fabric_cf.actor.core.container.maintenance import Maintenance
 from fabric_cf.actor.core.delegation.resource_ticket import ResourceTicketFactory
 from fabric_cf.actor.core.common.exceptions import BrokerException, ExceptionErrorCode
 from fabric_cf.actor.core.kernel.reservation_states import ReservationStates
@@ -508,6 +509,32 @@ def __candidate_nodes(self, *, sliver: NodeSliver) -> List[str]:
 
         return result
 
+    def __prune_nodes_in_maintenance(self, node_id_list: List[str], site: str, reservation: ABCBrokerReservation):
+        """
+        Prune the candidate node list to exclude the workers in Maintenance
+        @param node_id_list: Candidate Node List identified to allocate the reservation
+        @param site: Site Name
+        @param reservation: Reservation to be allocated
+        """
+        project_id = reservation.get_slice().get_project_id()
+        email = reservation.get_slice().get_owner().get_email()
+
+        nodes_to_remove = []
+        for node_id in node_id_list:
+            graph_node = self.get_network_node_from_graph(node_id=node_id)
+            status, error_message = Maintenance.is_sliver_provisioning_allowed(database=self.actor.get_plugin().get_database(),
+                                                                               project=project_id, site=site,
+                                                                               worker=graph_node.get_name(),
+                                                                               email=email)
+            if not status:
+                self.logger.info(f"Excluding {graph_node.get_name()} as allocation candidate due to {error_message}")
+                nodes_to_remove.append(node_id)
+
+        for x in nodes_to_remove:
+            node_id_list.remove(x)
+
+        return node_id_list
+
     def __find_first_fit(self, node_id_list: List[str], node_id_to_reservations: dict, inv: InventoryForType,
                          reservation: ABCBrokerReservation) -> Tuple[str, BaseSliver, Any]:
         """
@@ -571,6 +598,10 @@ def __allocate_nodes(self, *, reservation: ABCBrokerReservation, inv: NetworkNod
         delegation_id = None
         node_id_list = self.__candidate_nodes(sliver=sliver)
 
+        node_id_list = self.__prune_nodes_in_maintenance(node_id_list=node_id_list,
+                                                         site=sliver.site,
+                                                         reservation=reservation)
+
         # no candidate nodes found
         if len(node_id_list) == 0:
             error_msg = f'Insufficient resources: No candidates nodes found to serve {reservation}'