Management Server - Prepare for Maintenance and Cancel Maintenance improvements:

- Added new setting 'management.server.maintenance.ignore.maintenance.hosts' to ignore hosts in maintenance states  while preparing management server for maintenance. This skips agent transfer and agents count check for hosts in maintenance.
- Rebalance indirect agents after cancel maintenance, using rebalance parameter in cancelMaintenance API
- Force maintenance after maintenance window timeout, using forced parameter in prepareForMaintenance API.
- Propagate 'indirect.agent.lb.check.interval' setting change to the host agents.

Author: sureshanaparti

agent/src/main/java/com/cloud/agent/Agent.java

@@ -928,7 +928,7 @@ private Answer setupAgentCertificate(final SetupCertificateCommand cmd) {
         return new SetupCertificateAnswer(true);
     }
 
-    private void processManagementServerList(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) {
+    private void processManagementServerList(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval, final boolean triggerHostLB) {
         if (CollectionUtils.isNotEmpty(msList) && StringUtils.isNotEmpty(lbAlgorithm)) {
             try {
                 final String newMSHosts = String.format("%s%s%s", com.cloud.utils.StringUtils.toCSVList(msList), IAgentShell.hostLbAlgorithmSeparator, lbAlgorithm);
@@ -941,6 +941,12 @@ private void processManagementServerList(final List<String> msList, final List<S
             }
         }
         shell.setAvoidHosts(avoidMsList);
+        if (triggerHostLB) {
+            logger.info("Triggering preferred host task");
+            hostLbCheckExecutor = Executors.newSingleThreadScheduledExecutor((new NamedThreadFactory("HostLB-Executor")));
+            ScheduledExecutorService hostLbExecutor = Executors.newScheduledThreadPool(1);
+            hostLbExecutor.schedule(new PreferredHostCheckerTask(), 0, TimeUnit.MILLISECONDS);
+        }
         if ("shuffle".equals(lbAlgorithm)) {
             scheduleHostLBCheckerTask(0);
         } else {
@@ -949,14 +955,14 @@ private void processManagementServerList(final List<String> msList, final List<S
     }
 
     private Answer setupManagementServerList(final SetupMSListCommand cmd) {
-        processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval());
+        processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), cmd.getTriggerHostLb());
         return new SetupMSListAnswer(true);
     }
 
     private Answer migrateAgentToOtherMS(final MigrateAgentConnectionCommand cmd) {
         try {
             if (CollectionUtils.isNotEmpty(cmd.getMsList())) {
-                processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval());
+                processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), false);
             }
             Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job")).schedule(() -> {
                 migrateAgentConnection(cmd.getAvoidMsList());
@@ -1046,7 +1052,7 @@ public void processReadyCommand(final Command cmd) {
         }
 
         verifyAgentArch(ready.getArch());
-        processManagementServerList(ready.getMsHostList(), ready.getAvoidMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval());
+        processManagementServerList(ready.getMsHostList(), ready.getAvoidMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval(), false);
 
         logger.info("Ready command is processed for agent [id: {}, uuid: {}, name: {}]", getId(), getUuid(), getName());
     }

api/src/main/java/com/cloud/exception/OperationTimedoutException.java

@@ -40,7 +40,7 @@ public class OperationTimedoutException extends CloudException {
     boolean _isActive;
 
     public OperationTimedoutException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) {
-        super("Commands " + seqId + " to Host " + agentId + " timed out after " + time);
+        super("Commands " + seqId + " to Host " + agentId + " timed out after " + time + " secs");
         _agentId = agentId;
         _seqId = seqId;
         _time = time;

api/src/main/java/com/cloud/resource/ResourceState.java

@@ -76,6 +76,10 @@ public static Event toEvent(String e) {
         }
     }
 
+    public static List<ResourceState> s_maintenanceStates = List.of(ResourceState.Maintenance,
+            ResourceState.ErrorInMaintenance, ResourceState.PrepareForMaintenance,
+            ResourceState.ErrorInPrepareForMaintenance);
+
     public ResourceState getNextState(Event a) {
         return s_fsm.getNextState(this, a);
     }
@@ -98,8 +102,7 @@ public static String[] toString(ResourceState... states) {
     }
 
     public static boolean isMaintenanceState(ResourceState state) {
-        return Arrays.asList(ResourceState.Maintenance, ResourceState.ErrorInMaintenance,
-                ResourceState.PrepareForMaintenance, ResourceState.ErrorInPrepareForMaintenance).contains(state);
+        return s_maintenanceStates.contains(state);
     }
 
     public static boolean canAttemptMaintenance(ResourceState state) {

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -427,6 +427,7 @@ public class ApiConstants {
     public static final String PUBLIC_END_PORT = "publicendport";
     public static final String PUBLIC_ZONE = "publiczone";
     public static final String PURGE_RESOURCES = "purgeresources";
+    public static final String REBALANCE = "rebalance";
     public static final String RECEIVED_BYTES = "receivedbytes";
     public static final String RECONNECT = "reconnect";
     public static final String RECOVER = "recover";

core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java

@@ -29,13 +29,15 @@ public class SetupMSListCommand extends Command {
     private List<String> avoidMsList;
     private String lbAlgorithm;
     private Long lbCheckInterval;
+    private Boolean triggerHostLb;
 
-    public SetupMSListCommand(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) {
+    public SetupMSListCommand(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval, final Boolean triggerHostLb) {
         super();
         this.msList = msList;
         this.avoidMsList = avoidMsList;
         this.lbAlgorithm = lbAlgorithm;
         this.lbCheckInterval = lbCheckInterval;
+        this.triggerHostLb = triggerHostLb;
     }
 
     public List<String> getMsList() {
@@ -54,9 +56,12 @@ public Long getLbCheckInterval() {
         return lbCheckInterval;
     }
 
+    public boolean getTriggerHostLb() {
+        return triggerHostLb;
+    }
+
     @Override
     public boolean executeInSequence() {
         return false;
     }
-
 }

engine/components-api/src/main/java/com/cloud/agent/AgentManager.java

@@ -171,5 +171,5 @@ enum TapAgentsAction {
 
     void propagateChangeToAgents(Map<String, String> params);
 
-    boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs);
+    boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance);
 }

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -2193,7 +2193,7 @@ public void propagateChangeToAgents(Map<String, String> params) {
     }
 
     @Override
-    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs) {
+    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance) {
         return true;
     }
 

engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -42,6 +42,7 @@
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 
+import com.cloud.resource.ResourceState;
 import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.framework.config.ConfigDepot;
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -431,10 +432,10 @@ public boolean routeToPeer(final String peer, final byte[] bytes) {
             ch = connectToPeer(peer, ch);
             if (ch == null) {
                 try {
-                    logD(bytes, "Unable to route to peer: " + Request.parse(bytes));
+                    logD(bytes, "Unable to establish connection to route to peer: " + Request.parse(bytes));
                 } catch (ClassNotFoundException | UnsupportedVersionException e) {
                     // Request.parse thrown exception when we try to log it, log as much as we can
-                    logD(bytes, "Unable to route to peer, and Request.parse further caught exception" + e.getMessage());
+                    logD(bytes, "Unable to establish connection to route to peer, and Request.parse further caught exception" + e.getMessage());
                 }
                 return false;
             }
@@ -643,7 +644,6 @@ protected void doTask(final Task task) throws TaskExecutionException {
                 final Link link = task.getLink();
 
                 if (Request.fromServer(data)) {
-
                     final AgentAttache agent = findAttache(hostId);
 
                     if (Request.isControl(data)) {
@@ -691,7 +691,6 @@ protected void doTask(final Task task) throws TaskExecutionException {
                         cancel(Long.toString(Request.getManagementServerId(data)), hostId, Request.getSequence(data), e.getMessage());
                     }
                 } else {
-
                     final long mgmtId = Request.getManagementServerId(data);
                     if (mgmtId != -1 && mgmtId != _nodeId) {
                         routeToPeer(Long.toString(mgmtId), data);
@@ -1352,7 +1351,7 @@ private String handleShutdownManagementServerHostCommand(BaseShutdownManagementS
             if (cmd instanceof PrepareForMaintenanceManagementServerHostCommand) {
                 logger.debug("Received PrepareForMaintenanceManagementServerHostCommand - preparing for maintenance");
                 try {
-                    managementServerMaintenanceManager.prepareForMaintenance(((PrepareForMaintenanceManagementServerHostCommand) cmd).getLbAlgorithm());
+                    managementServerMaintenanceManager.prepareForMaintenance(((PrepareForMaintenanceManagementServerHostCommand) cmd).getLbAlgorithm(), ((PrepareForMaintenanceManagementServerHostCommand) cmd).isForced());
                     return "Successfully prepared for maintenance";
                 } catch(CloudRuntimeException e) {
                     return e.getMessage();
@@ -1399,14 +1398,14 @@ private String handleShutdownManagementServerHostCommand(BaseShutdownManagementS
     }
 
     @Override
-    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs) {
+    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance) {
         if (timeoutDurationInMs <= 0) {
             logger.debug("Not transferring direct agents from management server node {} (id: {}) to other nodes, invalid timeout duration", fromMsId, fromMsUuid);
             return false;
         }
 
         long transferStartTimeInMs = System.currentTimeMillis();
-        if (CollectionUtils.isEmpty(getDirectAgentHosts(fromMsId))) {
+        if (CollectionUtils.isEmpty(getDirectAgentHosts(fromMsId, excludeHostsInMaintenance))) {
             logger.info("No direct agent hosts available on management server node {} (id: {}), to transfer", fromMsId, fromMsUuid);
             return true;
         }
@@ -1421,7 +1420,7 @@ public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long
         int agentTransferFailedCount = 0;
         List<DataCenterVO> dataCenterList = dcDao.listAll();
         for (DataCenterVO dc : dataCenterList) {
-            List<HostVO> directAgentHostsInDc = getDirectAgentHostsInDc(fromMsId, dc.getId());
+            List<HostVO> directAgentHostsInDc = getDirectAgentHostsInDc(fromMsId, dc.getId(), excludeHostsInMaintenance);
             if (CollectionUtils.isEmpty(directAgentHostsInDc)) {
                 continue;
             }
@@ -1455,9 +1454,9 @@ public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long
         return (agentTransferFailedCount == 0);
     }
 
-    private List<HostVO> getDirectAgentHosts(long msId) {
+    private List<HostVO> getDirectAgentHosts(long msId, boolean excludeHostsInMaintenance) {
         List<HostVO> directAgentHosts = new ArrayList<>();
-        List<HostVO> hosts = _hostDao.listHostsByMs(msId);
+        List<HostVO> hosts = _hostDao.listHostsByMsResourceState(msId, null);
         for (HostVO host : hosts) {
             AgentAttache agent = findAttache(host.getId());
             if (agent instanceof DirectAgentAttache) {
@@ -1468,9 +1467,11 @@ private List<HostVO> getDirectAgentHosts(long msId) {
         return directAgentHosts;
     }
 
-    private List<HostVO> getDirectAgentHostsInDc(long msId, long dcId) {
+    private List<HostVO> getDirectAgentHostsInDc(long msId, long dcId, boolean excludeHostsInMaintenance) {
         List<HostVO> directAgentHosts = new ArrayList<>();
-        List<HostVO> hosts = _hostDao.listHostsByMsAndDc(msId, dcId);
+        // To exclude maintenance states use values from ResourceState as source of truth
+        List<ResourceState> statesToExclude = excludeHostsInMaintenance ? ResourceState.s_maintenanceStates : List.of();
+        List<HostVO> hosts = _hostDao.listHostsByMsDcResourceState(msId, dcId, statesToExclude);
         for (HostVO host : hosts) {
             AgentAttache agent = findAttache(host.getId());
             if (agent instanceof DirectAgentAttache) {
@@ -1506,6 +1507,10 @@ public void onManagementServerPreparingForMaintenance() {
     public void onManagementServerCancelPreparingForMaintenance() {
         logger.debug("Management server cancel preparing for maintenance");
         super.onManagementServerPreparingForMaintenance();
+
+        // needed for the case when Management Server in Preparing For Maintenance but didn't go to Maintenance state
+        // (where this variable will be reset)
+        _agentLbHappened = false;
     }
 
     @Override

engine/schema/src/main/java/com/cloud/host/dao/HostDao.java

@@ -177,14 +177,24 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<HostVO> listHostsByMsAndDc(long msId, long dcId);
 
+    List<HostVO> listHostsByMsDcResourceState(long msId, long dcId, List<ResourceState> excludedResourceStates);
+
     List<HostVO> listHostsByMs(long msId);
 
+    List<HostVO> listHostsByMsResourceState(long msId, List<ResourceState> excludedResourceStates);
+
     /**
-     * Retrieves the number of hosts/agents this {@see ManagementServer} has responsibility over.
-     * @param msId the id of the {@see ManagementServer}
-     * @return the number of hosts/agents this {@see ManagementServer} has responsibility over
+     * Count Hosts by given Management Server, Host and Hypervisor Types,
+     * and exclude Hosts with given Resource States.
+     *
+     * @param msId                   Management Server Id
+     * @param excludedResourceStates Resource States to be excluded
+     * @param hostTypes              Host Types
+     * @param hypervisorTypes        Hypervisor Types
+     * @return Hosts count
      */
-    int countByMs(long msId);
+    int countHostsByMsResourceStateTypeAndHypervisorType(long msId, List<ResourceState> excludedResourceStates,
+                                                         List<Type> hostTypes, List<HypervisorType> hypervisorTypes);
 
     /**
      * Retrieves the host ids/agents this {@see ManagementServer} has responsibility over.

engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java

@@ -72,6 +72,7 @@
 import com.cloud.utils.db.GenericSearchBuilder;
 import com.cloud.utils.db.JoinBuilder;
 import com.cloud.utils.db.JoinBuilder.JoinType;
+import com.cloud.utils.db.QueryBuilder;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.SearchCriteria.Func;
@@ -1600,6 +1601,17 @@ public List<HostVO> listHostsByMsAndDc(long msId, long dcId) {
         return listBy(sc);
     }
 
+    @Override
+    public List<HostVO> listHostsByMsDcResourceState(long msId, long dcId, List<ResourceState> excludedResourceStates) {
+        QueryBuilder<HostVO> sc = QueryBuilder.create(HostVO.class);
+        sc.and(sc.entity().getManagementServerId(), Op.EQ, msId);
+        sc.and(sc.entity().getDataCenterId(), Op.EQ, dcId);
+        if (CollectionUtils.isNotEmpty(excludedResourceStates)) {
+            sc.and(sc.entity().getResourceState(), Op.NIN, excludedResourceStates.toArray());
+        }
+        return listBy(sc.create());
+    }
+
     @Override
     public List<HostVO> listHostsByMs(long msId) {
         SearchCriteria<HostVO> sc = ResponsibleMsSearch.create();
@@ -1608,10 +1620,32 @@ public List<HostVO> listHostsByMs(long msId) {
     }
 
     @Override
-    public int countByMs(long msId) {
-        SearchCriteria<HostVO> sc = ResponsibleMsSearch.create();
-        sc.setParameters("managementServerId", msId);
-        return getCount(sc);
+    public List<HostVO> listHostsByMsResourceState(long msId, List<ResourceState> excludedResourceStates) {
+        QueryBuilder<HostVO> sc = QueryBuilder.create(HostVO.class);
+        sc.and(sc.entity().getManagementServerId(), Op.EQ, msId);
+        if (CollectionUtils.isNotEmpty(excludedResourceStates)) {
+            sc.and(sc.entity().getResourceState(), Op.NIN, excludedResourceStates.toArray());
+        }
+        return listBy(sc.create());
+    }
+
+    @Override
+    public int countHostsByMsResourceStateTypeAndHypervisorType(long msId,
+                                                                List<ResourceState> excludedResourceStates,
+                                                                List<Type> hostTypes,
+                                                                List<HypervisorType> hypervisorTypes) {
+        QueryBuilder<HostVO> sc = QueryBuilder.create(HostVO.class);
+        sc.and(sc.entity().getManagementServerId(), Op.EQ, msId);
+        if (CollectionUtils.isNotEmpty(excludedResourceStates)) {
+            sc.and(sc.entity().getResourceState(), Op.NIN, excludedResourceStates.toArray());
+        }
+        if (CollectionUtils.isNotEmpty(hostTypes)) {
+            sc.and(sc.entity().getType(), Op.IN, hostTypes.toArray());
+        }
+        if (CollectionUtils.isNotEmpty(hypervisorTypes)) {
+            sc.and(sc.entity().getHypervisorType(), Op.IN, hypervisorTypes.toArray());
+        }
+        return getCount(sc.create());
     }
 
     @Override

framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java

@@ -70,9 +70,11 @@ public interface IndirectAgentLB {
      */
     Long getLBPreferredHostCheckInterval(Long clusterId);
 
-    void propagateMSListToAgents();
+    void propagateMSListToAgents(boolean triggerHostLB);
 
-    boolean haveAgentBasedHosts(long msId);
+    void propagateMSListToAgentsInCluster(Long clusterId);
 
-    boolean migrateAgents(String fromMsUuid, long fromMsId, String lbAlgorithm, long timeoutDurationInMs);
+    boolean haveAgentBasedHosts(long msId, boolean excludeHostsInMaintenance);
+
+    boolean migrateAgents(String fromMsUuid, long fromMsId, String lbAlgorithm, long timeoutDurationInMs, boolean excludeHostsInMaintenance);
 }

plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/CancelMaintenanceCmd.java

@@ -18,12 +18,15 @@
 package org.apache.cloudstack.api.command;
 
 import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseCmd;
 
 import com.cloud.user.Account;
 
+import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.response.ManagementServerMaintenanceResponse;
 import org.apache.cloudstack.acl.RoleType;
+import org.apache.commons.lang3.BooleanUtils;
 
 @APICommand(name = CancelMaintenanceCmd.APINAME,
             description = "Cancels maintenance of the management server",
@@ -36,6 +39,13 @@ public class CancelMaintenanceCmd extends BaseMSMaintenanceActionCmd {
 
     public static final String APINAME = "cancelMaintenance";
 
+    @Parameter(name = ApiConstants.REBALANCE, type = CommandType.BOOLEAN, description = "Rebalance agents (applicable for indirect agents) after cancelling maintenance, default is true")
+    private Boolean rebalance;
+
+    public boolean getRebalance() {
+        return BooleanUtils.toBooleanDefaultIfNull(rebalance, true);
+    }
+
     @Override
     public String getCommandName() {
         return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;

plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/PrepareForMaintenanceCmd.java

@@ -26,6 +26,7 @@
 
 import org.apache.cloudstack.api.response.ManagementServerMaintenanceResponse;
 import org.apache.cloudstack.acl.RoleType;
+import org.apache.commons.lang3.BooleanUtils;
 
 @APICommand(name = PrepareForMaintenanceCmd.APINAME,
             description = "Prepares management server for maintenance by preventing new jobs from being accepted after completion of active jobs and migrating the agents",
@@ -40,6 +41,9 @@ public class PrepareForMaintenanceCmd extends BaseMSMaintenanceActionCmd {
             " when this is not set, already configured algorithm from setting 'indirect.agent.lb.algorithm' is considered")
     private String algorithm;
 
+    @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, description = "Force management server to maintenance after the maintenance window timeout, default is false")
+    private Boolean forced;
+
     public String getAlgorithm() {
         return algorithm;
     }
@@ -48,6 +52,10 @@ public void setAlgorithm(String algorithm) {
         this.algorithm = algorithm;
     }
 
+    public boolean isForced() {
+        return BooleanUtils.toBooleanDefaultIfNull(forced, false);
+    }
+
     @Override
     public String getCommandName() {
         return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;

plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManager.java

@@ -40,6 +40,15 @@ public interface ManagementServerMaintenanceManager {
             ConfigKey.Scope.Global,
             null);
 
+    ConfigKey<Boolean> ManagementServerMaintenanceIgnoreMaintenanceHosts = new ConfigKey<>(Boolean.class,
+            "management.server.maintenance.ignore.maintenance.hosts",
+            "Advanced",
+            String.valueOf(Boolean.FALSE),
+            "Host in Maintenance state can sometimes block Management Server to go to Maintenance; this setting skips Host(s) in Maintenance state during Management Server Maintenance, default: false.",
+            true,
+            ConfigKey.Scope.Global,
+            null);
+
     void registerListener(ManagementServerMaintenanceListener listener);
 
     void unregisterListener(ManagementServerMaintenanceListener listener);
@@ -83,7 +92,7 @@ public interface ManagementServerMaintenanceManager {
     String getLbAlgorithm();
 
     // Prepares the current management server for maintenance by migrating the agents and not accepting any more async jobs
-    void prepareForMaintenance(String lbAlorithm);
+    void prepareForMaintenance(String lbAlorithm, boolean forced);
 
     // Cancels maintenance of the current management server
     void cancelMaintenance();

plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java

@@ -26,7 +26,9 @@
 
 import javax.inject.Inject;
 
+import com.cloud.resource.ResourceState;
 import org.apache.cloudstack.agent.lb.IndirectAgentLB;
+import org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl;
 import org.apache.cloudstack.api.command.CancelMaintenanceCmd;
 import org.apache.cloudstack.api.command.CancelShutdownCmd;
 import org.apache.cloudstack.api.command.PrepareForMaintenanceCmd;
@@ -251,7 +253,7 @@ private void prepareForShutdown(boolean postTrigger) {
 
         this.preparingForShutdown = true;
         jobManager.disableAsyncJobs();
-        waitForPendingJobs();
+        waitForPendingJobs(false);
     }
 
     @Override
@@ -273,7 +275,7 @@ public void cancelShutdown() {
     }
 
     @Override
-    public void prepareForMaintenance(String lbAlorithm) {
+    public void prepareForMaintenance(String lbAlorithm, boolean forced) {
         if (this.preparingForShutdown) {
             throw new CloudRuntimeException("Shutdown has already been triggered, cancel shutdown and try again");
         }
@@ -286,7 +288,7 @@ public void prepareForMaintenance(String lbAlorithm) {
         this.lbAlgorithm = lbAlorithm;
         jobManager.disableAsyncJobs();
         onPreparingForMaintenance();
-        waitForPendingJobs();
+        waitForPendingJobs(forced);
     }
 
     @Override
@@ -310,12 +312,13 @@ public void cancelMaintenance() {
         }
     }
 
-    private void waitForPendingJobs() {
+    private void waitForPendingJobs(boolean forceMaintenance) {
         cancelWaitForPendingJobs();
         pendingJobsCheckTask = Executors.newScheduledThreadPool(1, new NamedThreadFactory("PendingJobsCheck"));
         long pendingJobsCheckDelayInSecs = 1L; // 1 sec
         long pendingJobsCheckPeriodInSecs = 3L; // every 3 secs, check more frequently for pending jobs
-        pendingJobsCheckTask.scheduleAtFixedRate(new CheckPendingJobsTask(this), pendingJobsCheckDelayInSecs, pendingJobsCheckPeriodInSecs, TimeUnit.SECONDS);
+        boolean ignoreMaintenanceHosts = ManagementServerMaintenanceIgnoreMaintenanceHosts.value();
+        pendingJobsCheckTask.scheduleAtFixedRate(new CheckPendingJobsTask(this, ignoreMaintenanceHosts, forceMaintenance), pendingJobsCheckDelayInSecs, pendingJobsCheckPeriodInSecs, TimeUnit.SECONDS);
     }
 
     @Override
@@ -426,7 +429,8 @@ public ManagementServerMaintenanceResponse prepareForMaintenance(PrepareForMaint
 
         checkAnyMsInPreparingStates("prepare for maintenance");
 
-        if (indirectAgentLB.haveAgentBasedHosts(msHost.getMsid())) {
+        boolean ignoreMaintenanceHosts = ManagementServerMaintenanceIgnoreMaintenanceHosts.value();
+        if (indirectAgentLB.haveAgentBasedHosts(msHost.getMsid(), ignoreMaintenanceHosts)) {
             List<String> indirectAgentMsList = indirectAgentLB.getManagementServerList();
             indirectAgentMsList.remove(msHost.getServiceIP());
             List<String> nonUpMsList = msHostDao.listNonUpStateMsIPs();
@@ -437,7 +441,7 @@ public ManagementServerMaintenanceResponse prepareForMaintenance(PrepareForMaint
         }
 
         final Command[] cmds = new Command[1];
-        cmds[0] = new PrepareForMaintenanceManagementServerHostCommand(msHost.getMsid(), cmd.getAlgorithm());
+        cmds[0] = new PrepareForMaintenanceManagementServerHostCommand(msHost.getMsid(), cmd.getAlgorithm(), cmd.isForced());
         executeCmd(msHost, cmds);
 
         msHostDao.updateState(msHost.getId(), State.PreparingForMaintenance);
@@ -457,10 +461,15 @@ public ManagementServerMaintenanceResponse cancelMaintenance(CancelMaintenanceCm
         }
 
         final Command[] cmds = new Command[1];
-        cmds[0] = new CancelMaintenanceManagementServerHostCommand(msHost.getMsid());
+        cmds[0] = new CancelMaintenanceManagementServerHostCommand(msHost.getMsid(), cmd.getRebalance());
         executeCmd(msHost, cmds);
 
         msHostDao.updateState(msHost.getId(), State.Up);
+
+        if (cmd.getRebalance()) {
+            logger.info("Propagate MS list and rebalance indirect agents");
+            indirectAgentLB.propagateMSListToAgents(true);
+        }
         return prepareMaintenanceResponse(cmd.getManagementServerId());
     }
 
@@ -546,17 +555,21 @@ public String getConfigComponentName() {
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[]{
-                ManagementServerMaintenanceTimeoutInMins
+                ManagementServerMaintenanceTimeoutInMins, ManagementServerMaintenanceIgnoreMaintenanceHosts
         };
     }
 
     private final class CheckPendingJobsTask extends ManagedContextRunnable {
 
         private ManagementServerMaintenanceManager managementServerMaintenanceManager;
+        private boolean ignoreMaintenanceHosts = false;
         private boolean agentsTransferTriggered = false;
+        private boolean forceMaintenance = false;
 
-        public CheckPendingJobsTask(ManagementServerMaintenanceManager managementServerMaintenanceManager) {
+        public CheckPendingJobsTask(ManagementServerMaintenanceManager managementServerMaintenanceManager, boolean ignoreMaintenanceHosts, boolean forceMaintenance) {
             this.managementServerMaintenanceManager = managementServerMaintenanceManager;
+            this.ignoreMaintenanceHosts = ignoreMaintenanceHosts;
+            this.forceMaintenance = forceMaintenance;
         }
 
         @Override
@@ -570,14 +583,25 @@ protected void runInContext() {
                 }
 
                 if (managementServerMaintenanceManager.isPreparingForMaintenance() && isMaintenanceWindowExpired()) {
+                    if (forceMaintenance) {
+                        logger.debug("Maintenance window timeout, MS is forced to Maintenance Mode");
+                        ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId());
+                        msHostDao.updateState(msHost.getId(), State.Maintenance);
+                        managementServerMaintenanceManager.onMaintenance();
+                        managementServerMaintenanceManager.cancelWaitForPendingJobs();
+                        return;
+                    }
+
                     logger.debug("Maintenance window timeout, terminating the pending jobs check timer task");
                     managementServerMaintenanceManager.cancelPreparingForMaintenance(null);
                     managementServerMaintenanceManager.cancelWaitForPendingJobs();
                     return;
                 }
 
                 long totalPendingJobs = managementServerMaintenanceManager.countPendingJobs(ManagementServerNode.getManagementServerId());
-                int totalAgents = hostDao.countByMs(ManagementServerNode.getManagementServerId());
+
+                long totalAgents = totalAgentsInMs();
+
                 String msg = String.format("Checking for triggered maintenance or shutdown... shutdownTriggered [%b] AllowAsyncJobs [%b] PendingJobCount [%d] AgentsCount [%d]",
                         managementServerMaintenanceManager.isShutdownTriggered(), managementServerMaintenanceManager.isAsyncJobsEnabled(), totalPendingJobs, totalAgents);
                 logger.debug(msg);
@@ -609,15 +633,15 @@ protected void runInContext() {
 
                     agentsTransferTriggered = true;
                     logger.info(String.format("Preparing for maintenance - migrating agents from management server node %d (id: %s)", ManagementServerNode.getManagementServerId(), msHost.getUuid()));
-                    boolean agentsMigrated = indirectAgentLB.migrateAgents(msHost.getUuid(), ManagementServerNode.getManagementServerId(), managementServerMaintenanceManager.getLbAlgorithm(), remainingMaintenanceWindowInMs());
+                    boolean agentsMigrated = indirectAgentLB.migrateAgents(msHost.getUuid(), ManagementServerNode.getManagementServerId(), managementServerMaintenanceManager.getLbAlgorithm(), remainingMaintenanceWindowInMs(), ignoreMaintenanceHosts);
                     if (!agentsMigrated) {
                         logger.warn(String.format("Unable to prepare for maintenance, cannot migrate indirect agents on this management server node %d (id: %s)", ManagementServerNode.getManagementServerId(), msHost.getUuid()));
                         managementServerMaintenanceManager.cancelPreparingForMaintenance(msHost);
                         managementServerMaintenanceManager.cancelWaitForPendingJobs();
                         return;
                     }
 
-                    if(!agentMgr.transferDirectAgentsFromMS(msHost.getUuid(), ManagementServerNode.getManagementServerId(), remainingMaintenanceWindowInMs())) {
+                    if(!agentMgr.transferDirectAgentsFromMS(msHost.getUuid(), ManagementServerNode.getManagementServerId(), remainingMaintenanceWindowInMs(), ignoreMaintenanceHosts)) {
                         logger.warn(String.format("Unable to prepare for maintenance, cannot transfer direct agents on this management server node %d (id: %s)", ManagementServerNode.getManagementServerId(), msHost.getUuid()));
                         managementServerMaintenanceManager.cancelPreparingForMaintenance(msHost);
                         managementServerMaintenanceManager.cancelWaitForPendingJobs();
@@ -648,5 +672,14 @@ private long remainingMaintenanceWindowInMs() {
             long remainingMaintenanceWindowTimeInMs = (ManagementServerMaintenanceTimeoutInMins.value().longValue() * 60 * 1000) - maintenanceElapsedTimeInMs;
             return (remainingMaintenanceWindowTimeInMs > 0) ? remainingMaintenanceWindowTimeInMs : 0;
         }
+
+        private long totalAgentsInMs() {
+            /* Any Host in Maintenance state could block moving Management Server to Maintenance state, exclude those Hosts from total agents count
+             * To exclude maintenance states use values from ResourceState as source of truth
+             */
+            List<ResourceState> statesToExclude = ignoreMaintenanceHosts ? ResourceState.s_maintenanceStates : List.of();
+            return hostDao.countHostsByMsResourceStateTypeAndHypervisorType(ManagementServerNode.getManagementServerId(), statesToExclude,
+                    IndirectAgentLBServiceImpl.agentValidHostTypes, null);
+        }
     }
 }

plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/command/CancelMaintenanceManagementServerHostCommand.java

@@ -19,8 +19,18 @@
 package org.apache.cloudstack.maintenance.command;
 
 public class CancelMaintenanceManagementServerHostCommand extends BaseShutdownManagementServerHostCommand {
+    boolean rebalance;
 
     public CancelMaintenanceManagementServerHostCommand(long msId) {
         super(msId);
     }
+
+    public CancelMaintenanceManagementServerHostCommand(long msId, boolean rebalance) {
+        super(msId);
+        this.rebalance = rebalance;
+    }
+
+    public boolean getRebalance() {
+        return rebalance;
+    }
 }

plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/command/PrepareForMaintenanceManagementServerHostCommand.java

@@ -20,17 +20,23 @@
 
 public class PrepareForMaintenanceManagementServerHostCommand extends BaseShutdownManagementServerHostCommand {
     String lbAlgorithm;
+    boolean forced;
 
     public PrepareForMaintenanceManagementServerHostCommand(long msId) {
         super(msId);
     }
 
-    public PrepareForMaintenanceManagementServerHostCommand(long msId, String lbAlgorithm) {
+    public PrepareForMaintenanceManagementServerHostCommand(long msId, String lbAlgorithm, boolean forced) {
         super(msId);
         this.lbAlgorithm = lbAlgorithm;
+        this.forced = forced;
     }
 
     public String getLbAlgorithm() {
         return lbAlgorithm;
     }
+
+    public boolean isForced() {
+        return forced;
+    }
 }

plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java

@@ -305,11 +305,11 @@ public void triggerShutdownCmd() {
     @Test
     public void prepareForMaintenanceAndCancelFromMaintenanceState() {
         Mockito.doNothing().when(jobManagerMock).disableAsyncJobs();
-        spy.prepareForMaintenance("static");
+        spy.prepareForMaintenance("static", false);
         Mockito.verify(jobManagerMock).disableAsyncJobs();
 
         Assert.assertThrows(CloudRuntimeException.class, () -> {
-            spy.prepareForMaintenance("static");
+            spy.prepareForMaintenance("static", false);
         });
 
         ManagementServerHostVO msHost = mock(ManagementServerHostVO.class);
@@ -324,11 +324,11 @@ public void prepareForMaintenanceAndCancelFromMaintenanceState() {
     @Test
     public void prepareForMaintenanceAndCancelFromPreparingForMaintenanceState() {
         Mockito.doNothing().when(jobManagerMock).disableAsyncJobs();
-        spy.prepareForMaintenance("static");
+        spy.prepareForMaintenance("static", false);
         Mockito.verify(jobManagerMock).disableAsyncJobs();
 
         Assert.assertThrows(CloudRuntimeException.class, () -> {
-            spy.prepareForMaintenance("static");
+            spy.prepareForMaintenance("static", false);
         });
 
         ManagementServerHostVO msHost = mock(ManagementServerHostVO.class);
@@ -455,7 +455,7 @@ public void prepareForMaintenanceCmdNoIndirectMsHosts() {
         Mockito.when(msHostDao.listNonUpStateMsIPs()).thenReturn(new ArrayList<>());
         PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class);
         Mockito.when(cmd.getManagementServerId()).thenReturn(1L);
-        Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(true);
+        Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong(), anyBoolean())).thenReturn(true);
         Mockito.when(indirectAgentLBMock.getManagementServerList()).thenReturn(new ArrayList<>());
 
         Assert.assertThrows(CloudRuntimeException.class, () -> {
@@ -476,7 +476,7 @@ public void prepareForMaintenanceCmdNullResponseFromClusterManager() {
         Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1);
         PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class);
         Mockito.when(cmd.getManagementServerId()).thenReturn(1L);
-        Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(false);
+        Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong(), anyBoolean())).thenReturn(false);
         Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn(null);
 
         Assert.assertThrows(CloudRuntimeException.class, () -> {
@@ -497,7 +497,7 @@ public void prepareForMaintenanceCmdFailedResponseFromClusterManager() {
         Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1);
         PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class);
         Mockito.when(cmd.getManagementServerId()).thenReturn(1L);
-        Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(false);
+        Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong(), anyBoolean())).thenReturn(false);
         Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Failed");
 
         Assert.assertThrows(CloudRuntimeException.class, () -> {
@@ -518,7 +518,7 @@ public void prepareForMaintenanceCmdSuccessResponseFromClusterManager() {
         Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1);
         PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class);
         Mockito.when(cmd.getManagementServerId()).thenReturn(1L);
-        Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(false);
+        Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong(), anyBoolean())).thenReturn(false);
         Mockito.when(hostDao.listByMs(anyLong())).thenReturn(new ArrayList<>());
         Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Success");
 

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -51,6 +51,7 @@
 import javax.naming.ConfigurationException;
 
 import com.cloud.resource.ResourceManager;
+import com.cloud.utils.Ternary;
 import org.apache.cloudstack.acl.SecurityChecker;
 import org.apache.cloudstack.affinity.AffinityGroup;
 import org.apache.cloudstack.affinity.AffinityGroupService;
@@ -616,19 +617,28 @@ private void initMessageBusListener() {
         messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, new MessageSubscriber() {
             @Override
             public void onPublishMessage(String serderAddress, String subject, Object args) {
-                String globalSettingUpdated = (String) args;
-                if (StringUtils.isEmpty(globalSettingUpdated)) {
+                Ternary<String, ConfigKey.Scope, Long> settingUpdated = (Ternary<String, ConfigKey.Scope, Long>) args;
+                String settingNameUpdated = settingUpdated.first();
+                if (StringUtils.isEmpty(settingNameUpdated)) {
                     return;
                 }
-                if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) ||
-                        globalSettingUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) {
-                    _indirectAgentLB.propagateMSListToAgents();
-                } else if (globalSettingUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString())
-                        ||  globalSettingUpdated.equals(Config.MigrateWait.toString())) {
+                if (settingNameUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) ||
+                        settingNameUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) {
+                    _indirectAgentLB.propagateMSListToAgents(false);
+                } else if (settingNameUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString())
+                        ||  settingNameUpdated.equals(Config.MigrateWait.toString())) {
                     Map<String, String> params = new HashMap<String, String>();
                     params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
                     params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
                     _agentManager.propagateChangeToAgents(params);
+                } else if (settingNameUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBCheckInterval.key())) {
+                    ConfigKey.Scope scope = settingUpdated.second();
+                    if (scope == ConfigKey.Scope.Global) {
+                        _indirectAgentLB.propagateMSListToAgents(false);
+                    } else if (scope == ConfigKey.Scope.Cluster) {
+                        Long clusterId = settingUpdated.third();
+                        _indirectAgentLB.propagateMSListToAgentsInCluster(clusterId);
+                    }
                 }
             }
         });
@@ -817,6 +827,7 @@ public String updateConfiguration(final long userId, final String name, final St
             CallContext.current().setEventDetails(String.format(" Name: %s, New Value: %s, Scope: %s", name, value, scope));
 
             _configDepot.invalidateConfigCache(name, scopeVal, resourceId);
+            messageBus.publish(_name, EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, PublishScope.GLOBAL, new Ternary<>(name, scopeVal, resourceId));
             return valueEncrypted ? DBEncryptionUtil.decrypt(value) : value;
         }
 
@@ -911,7 +922,7 @@ public String updateConfiguration(final long userId, final String name, final St
         }
 
         txn.commit();
-        messageBus.publish(_name, EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, PublishScope.GLOBAL, name);
+        messageBus.publish(_name, EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, PublishScope.GLOBAL, new Ternary<>(name, ConfigKey.Scope.Global, resourceId));
         return _configDao.getValue(name);
     }
 

server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java

@@ -204,7 +204,7 @@ public void testExceptionOnEmptyHostSetting() throws NoSuchFieldException, Illeg
     public void testGetOrderedRunningHostIdsEmptyList() {
         doReturn(Collections.emptyList()).when(hostDao).findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(
                 Mockito.eq(DC_1_ID), Mockito.eq(null), Mockito.eq(null), Mockito.anyList(), Mockito.anyList(), Mockito.anyList());
-        Assert.assertTrue(agentMSLB.getOrderedHostIdList(DC_1_ID).isEmpty());
+        Assert.assertTrue(agentMSLB.getOrderedHostIdList(DC_1_ID, false).isEmpty());
     }
 
     @Test
@@ -213,6 +213,6 @@ public void testGetOrderedRunningHostIdsOrderList() {
                 .findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(Mockito.eq(DC_1_ID), Mockito.eq(null), Mockito.eq(null),
                         Mockito.anyList(), Mockito.anyList(), Mockito.anyList());
         Assert.assertEquals(Arrays.asList(host1.getId(), host2.getId(), host3.getId(), host4.getId()),
-                agentMSLB.getOrderedHostIdList(DC_1_ID));
+                agentMSLB.getOrderedHostIdList(DC_1_ID, false));
     }
 }

server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java

@@ -1875,6 +1875,7 @@
 "label.read.io": "Read (IO)",
 "label.readonly": "Read-Only",
 "label.reason": "Reason",
+"label.rebalance": "Rebalance",
 "label.reboot": "Reboot",
 "label.recent.deliveries": "Recent deliveries",
 "label.receivedbytes": "Bytes received",

ui/public/locales/en.json

@@ -75,6 +75,7 @@ export default {
       message: 'message.cancel.maintenance',
       dataView: true,
       popup: true,
+      args: ['rebalance'],
       show: (record, store) => { return ['PreparingForMaintenance', 'Maintenance'].includes(record.state) },
       mapping: {
         managementserverid: {

ui/src/config/section/infra/managementServers.js

@@ -45,6 +45,12 @@
             </a-select-option>
           </a-select>
         </a-form-item>
+        <a-form-item name="forced" ref="forced">
+          <template #label>
+            <tooltip-label :title="$t('label.forced')" :tooltip="prepareForMaintenanceApiParams.forced.description"/>
+          </template>
+          <a-switch v-model:checked="form.forced" />
+        </a-form-item>
         <a-divider/>
         <a-alert type="error">
           <template #message>
@@ -135,6 +141,7 @@ export default {
         if (this.isPrepareForMaintenance && this.form.algorithm !== '') {
           params.algorithm = this.form.algorithm
         }
+        params.forced = this.form.forced
         api(this.action.currentAction.api, params).then(() => {
           this.$message.success(this.$t(this.action.currentAction.label) + ' : ' + this.resource.name)
           this.closeAction()

ui/src/views/infra/Confirmation.vue

@@ -323,7 +323,7 @@ protected void read(final SelectionKey key) throws IOException {
             logger.trace("Reading from: {}", socketChannel.socket().toString());
             final byte[] data = link.read(socketChannel);
             if (data == null) {
-                logger.trace("Packet is incomplete.  Waiting for more.");
+                logger.trace("Packet is incomplete. Waiting for more.");
                 return;
             }
             final Task task = _factory.create(Task.Type.DATA, link, data);