Management Server - Prepare for Maintenance and Cancel Maintenance improvements

agent/src/main/java/com/cloud/agent/Agent.java

@@ -453,22 +453,30 @@
         certExecutor.schedule(new PostCertificateRenewalTask(this), 5, TimeUnit.SECONDS);
     }
 
-    private void scheduleHostLBCheckerTask(final long checkInterval) {
+    private void scheduleHostLBCheckerTask(final String lbAlgorithm, final long checkInterval) {
         String name = "HostLBCheckerTask";
         if (hostLbCheckExecutor != null && !hostLbCheckExecutor.isShutdown()) {
+            logger.info("Shutting down the preferred host checker task {}", name);
             hostLbCheckExecutor.shutdown();
             try {
                 if (!hostLbCheckExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
                     hostLbCheckExecutor.shutdownNow();
                 }
             } catch (InterruptedException e) {
-                logger.debug("Forcing {} shutdown as it did not shutdown in the desired time due to: {}",
+                logger.debug("Forcing the preferred host checker task {} shutdown as it did not shutdown in the desired time due to: {}",
                         name, e.getMessage());
                 hostLbCheckExecutor.shutdownNow();
             }
         }
         if (checkInterval > 0L) {
-            logger.info("Scheduling preferred host task with host.lb.interval={}ms", checkInterval);
+            if ("shuffle".equalsIgnoreCase(lbAlgorithm)) {
+                logger.info("Scheduling the preferred host checker task to trigger once (to apply lb algorithm '{}') after host.lb.interval={} ms", lbAlgorithm, checkInterval);
+                hostLbCheckExecutor = Executors.newSingleThreadScheduledExecutor((new NamedThreadFactory(name)));
+                hostLbCheckExecutor.schedule(new PreferredHostCheckerTask(), checkInterval, TimeUnit.MILLISECONDS);
+                return;
+            }
+
+            logger.info("Scheduling a recurring preferred host checker task with lb algorithm '{}' and host.lb.interval={} ms", lbAlgorithm, checkInterval);
             hostLbCheckExecutor = Executors.newSingleThreadScheduledExecutor((new NamedThreadFactory(name)));
             hostLbCheckExecutor.scheduleAtFixedRate(new PreferredHostCheckerTask(), checkInterval, checkInterval,
                     TimeUnit.MILLISECONDS);
@@ -928,7 +936,7 @@
         return new SetupCertificateAnswer(true);
     }
 
-    private void processManagementServerList(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) {
+    private void processManagementServerList(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval, final boolean triggerHostLB) {
         if (CollectionUtils.isNotEmpty(msList) && StringUtils.isNotEmpty(lbAlgorithm)) {
             try {
                 final String newMSHosts = String.format("%s%s%s", com.cloud.utils.StringUtils.toCSVList(msList), IAgentShell.hostLbAlgorithmSeparator, lbAlgorithm);
@@ -941,22 +949,24 @@
             }
         }
         shell.setAvoidHosts(avoidMsList);
-        if ("shuffle".equals(lbAlgorithm)) {
-            scheduleHostLBCheckerTask(0);
-        } else {
-            scheduleHostLBCheckerTask(shell.getLbCheckerInterval(lbCheckInterval));
+        if (triggerHostLB) {
+            logger.info("Triggering the preferred host checker task now");
+            ScheduledExecutorService hostLbExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("HostLB-Executor"));
+            hostLbExecutor.schedule(new PreferredHostCheckerTask(), 0, TimeUnit.MILLISECONDS);
+            hostLbExecutor.shutdown();
         }
+        scheduleHostLBCheckerTask(lbAlgorithm, shell.getLbCheckerInterval(lbCheckInterval));
     }
 
     private Answer setupManagementServerList(final SetupMSListCommand cmd) {
-        processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval());
+        processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), cmd.getTriggerHostLb());
         return new SetupMSListAnswer(true);
     }
 
     private Answer migrateAgentToOtherMS(final MigrateAgentConnectionCommand cmd) {
         try {
             if (CollectionUtils.isNotEmpty(cmd.getMsList())) {
-                processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval());
+                processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), false);
             }
             Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job")).schedule(() -> {
                 migrateAgentConnection(cmd.getAvoidMsList());
@@ -1046,7 +1056,7 @@
         }
 
         verifyAgentArch(ready.getArch());
-        processManagementServerList(ready.getMsHostList(), ready.getAvoidMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval());
+        processManagementServerList(ready.getMsHostList(), ready.getAvoidMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval(), false);
 
         logger.info("Ready command is processed for agent [id: {}, uuid: {}, name: {}]", getId(), getUuid(), getName());
     }

api/src/main/java/com/cloud/exception/OperationTimedoutException.java

@@ -40,7 +40,7 @@ public class OperationTimedoutException extends CloudException {
     boolean _isActive;
 
     public OperationTimedoutException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) {
-        super("Commands " + seqId + " to Host " + agentId + " timed out after " + time);
+        super("Commands " + seqId + " to Host " + agentId + " timed out after " + time + " secs");
         _agentId = agentId;
         _seqId = seqId;
         _time = time;

api/src/main/java/com/cloud/resource/ResourceState.java

@@ -76,6 +76,10 @@ public static Event toEvent(String e) {
         }
     }
 
+    public static List<ResourceState> s_maintenanceStates = List.of(ResourceState.Maintenance,
+            ResourceState.ErrorInMaintenance, ResourceState.PrepareForMaintenance,
+            ResourceState.ErrorInPrepareForMaintenance);
+
     public ResourceState getNextState(Event a) {
         return s_fsm.getNextState(this, a);
     }
@@ -98,8 +102,7 @@ public static String[] toString(ResourceState... states) {
     }
 
     public static boolean isMaintenanceState(ResourceState state) {
-        return Arrays.asList(ResourceState.Maintenance, ResourceState.ErrorInMaintenance,
-                ResourceState.PrepareForMaintenance, ResourceState.ErrorInPrepareForMaintenance).contains(state);
+        return s_maintenanceStates.contains(state);
     }
 
     public static boolean canAttemptMaintenance(ResourceState state) {

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -432,6 +432,7 @@ public class ApiConstants {
     public static final String PUBLIC_END_PORT = "publicendport";
     public static final String PUBLIC_ZONE = "publiczone";
     public static final String PURGE_RESOURCES = "purgeresources";
+    public static final String REBALANCE = "rebalance";
     public static final String RECEIVED_BYTES = "receivedbytes";
     public static final String RECONNECT = "reconnect";
     public static final String RECOVER = "recover";

api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/PatchSystemVMCmd.java

@@ -46,7 +46,7 @@ public class PatchSystemVMCmd extends BaseAsyncCmd {
     @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN,
             description = "If true, initiates copy of scripts and restart of the agent, even if the scripts version matches." +
                     "To be used with ID parameter only")
-    private Boolean force;
+    private Boolean forced;
 
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
@@ -58,7 +58,7 @@ public Long getId() {
     }
 
     public boolean isForced() {
-        return force != null && force;
+        return forced != null && forced;
     }
 
     /////////////////////////////////////////////////////

core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java

@@ -29,13 +29,15 @@
     private List<String> avoidMsList;
     private String lbAlgorithm;
     private Long lbCheckInterval;
+    private Boolean triggerHostLb;
 
-    public SetupMSListCommand(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) {
+    public SetupMSListCommand(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval, final Boolean triggerHostLb) {
         super();
         this.msList = msList;
         this.avoidMsList = avoidMsList;
         this.lbAlgorithm = lbAlgorithm;
         this.lbCheckInterval = lbCheckInterval;
+        this.triggerHostLb = triggerHostLb;
     }
 
     public List<String> getMsList() {
@@ -54,9 +56,12 @@
         return lbCheckInterval;
     }
 
+    public boolean getTriggerHostLb() {
+        return triggerHostLb;
+    }
+
     @Override
     public boolean executeInSequence() {
         return false;
     }
-
 }

engine/components-api/src/main/java/com/cloud/agent/AgentManager.java

@@ -171,5 +171,5 @@ enum TapAgentsAction {
 
     void propagateChangeToAgents(Map<String, String> params);
 
-    boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs);
+    boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance);
 }

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -273,8 +273,6 @@
 
         _executor = new ThreadPoolExecutor(agentTaskThreads, agentTaskThreads, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentTaskPool"));
 
-        initConnectExecutor();
-
         maxConcurrentNewAgentConnections = RemoteAgentMaxConcurrentNewConnections.value();
 
         _connection = new NioServer("AgentManager", Port.value(), Workers.value() + 10,
@@ -828,6 +826,7 @@
             return true;
         }
 
+        initConnectExecutor();
         startDirectlyConnectedHosts(false);
 
         if (_connection != null) {
@@ -2193,7 +2192,7 @@
     }
 
     @Override
-    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs) {
+    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance) {
         return true;
     }
 

engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -42,6 +42,7 @@
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 
+import com.cloud.resource.ResourceState;
 import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.framework.config.ConfigDepot;
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -431,10 +432,10 @@
             ch = connectToPeer(peer, ch);
             if (ch == null) {
                 try {
-                    logD(bytes, "Unable to route to peer: " + Request.parse(bytes));
+                    logD(bytes, "Unable to establish connection to route to peer: " + Request.parse(bytes));
                 } catch (ClassNotFoundException | UnsupportedVersionException e) {
                     // Request.parse thrown exception when we try to log it, log as much as we can
-                    logD(bytes, "Unable to route to peer, and Request.parse further caught exception" + e.getMessage());
+                    logD(bytes, "Unable to establish connection to route to peer, and Request.parse further caught exception" + e.getMessage());
                 }
                 return false;
             }
@@ -643,7 +644,6 @@
                 final Link link = task.getLink();
 
                 if (Request.fromServer(data)) {
-
                     final AgentAttache agent = findAttache(hostId);
 
                     if (Request.isControl(data)) {
@@ -691,7 +691,6 @@
                         cancel(Long.toString(Request.getManagementServerId(data)), hostId, Request.getSequence(data), e.getMessage());
                     }
                 } else {
-
                     final long mgmtId = Request.getManagementServerId(data);
                     if (mgmtId != -1 && mgmtId != _nodeId) {
                         routeToPeer(Long.toString(mgmtId), data);
@@ -1352,7 +1351,7 @@
             if (cmd instanceof PrepareForMaintenanceManagementServerHostCommand) {
                 logger.debug("Received PrepareForMaintenanceManagementServerHostCommand - preparing for maintenance");
                 try {
-                    managementServerMaintenanceManager.prepareForMaintenance(((PrepareForMaintenanceManagementServerHostCommand) cmd).getLbAlgorithm());
+                    managementServerMaintenanceManager.prepareForMaintenance(((PrepareForMaintenanceManagementServerHostCommand) cmd).getLbAlgorithm(), ((PrepareForMaintenanceManagementServerHostCommand) cmd).isForced());
                     return "Successfully prepared for maintenance";
                 } catch(CloudRuntimeException e) {
                     return e.getMessage();
@@ -1399,14 +1398,14 @@
     }
 
     @Override
-    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs) {
+    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance) {
         if (timeoutDurationInMs <= 0) {
             logger.debug("Not transferring direct agents from management server node {} (id: {}) to other nodes, invalid timeout duration", fromMsId, fromMsUuid);
             return false;
         }
 
         long transferStartTimeInMs = System.currentTimeMillis();
-        if (CollectionUtils.isEmpty(getDirectAgentHosts(fromMsId))) {
+        if (CollectionUtils.isEmpty(getDirectAgentHosts(fromMsId, excludeHostsInMaintenance))) {
             logger.info("No direct agent hosts available on management server node {} (id: {}), to transfer", fromMsId, fromMsUuid);
             return true;
         }
@@ -1421,7 +1420,7 @@
         int agentTransferFailedCount = 0;
         List<DataCenterVO> dataCenterList = dcDao.listAll();
         for (DataCenterVO dc : dataCenterList) {
-            List<HostVO> directAgentHostsInDc = getDirectAgentHostsInDc(fromMsId, dc.getId());
+            List<HostVO> directAgentHostsInDc = getDirectAgentHostsInDc(fromMsId, dc.getId(), excludeHostsInMaintenance);
             if (CollectionUtils.isEmpty(directAgentHostsInDc)) {
                 continue;
             }
@@ -1455,9 +1454,10 @@
         return (agentTransferFailedCount == 0);
     }
 
-    private List<HostVO> getDirectAgentHosts(long msId) {
+    private List<HostVO> getDirectAgentHosts(long msId, boolean excludeHostsInMaintenance) {
         List<HostVO> directAgentHosts = new ArrayList<>();
-        List<HostVO> hosts = _hostDao.listHostsByMs(msId);
+        List<ResourceState> statesToExclude = excludeHostsInMaintenance ? ResourceState.s_maintenanceStates : List.of();
+        List<HostVO> hosts = _hostDao.listHostsByMsResourceState(msId, statesToExclude);
         for (HostVO host : hosts) {
             AgentAttache agent = findAttache(host.getId());
             if (agent instanceof DirectAgentAttache) {
@@ -1468,9 +1468,11 @@
         return directAgentHosts;
     }
 
-    private List<HostVO> getDirectAgentHostsInDc(long msId, long dcId) {
+    private List<HostVO> getDirectAgentHostsInDc(long msId, long dcId, boolean excludeHostsInMaintenance) {
         List<HostVO> directAgentHosts = new ArrayList<>();
-        List<HostVO> hosts = _hostDao.listHostsByMsAndDc(msId, dcId);
+        // To exclude maintenance states use values from ResourceState as source of truth
+        List<ResourceState> statesToExclude = excludeHostsInMaintenance ? ResourceState.s_maintenanceStates : List.of();
+        List<HostVO> hosts = _hostDao.listHostsByMsDcResourceState(msId, dcId, statesToExclude);
         for (HostVO host : hosts) {
             AgentAttache agent = findAttache(host.getId());
             if (agent instanceof DirectAgentAttache) {
@@ -1506,6 +1508,10 @@
     public void onManagementServerCancelPreparingForMaintenance() {
         logger.debug("Management server cancel preparing for maintenance");
         super.onManagementServerPreparingForMaintenance();
+
+        // needed for the case when Management Server in Preparing For Maintenance but didn't go to Maintenance state
+        // (where this variable will be reset)
+        _agentLbHappened = false;
     }
 
     @Override

engine/schema/src/main/java/com/cloud/host/dao/HostDao.java

@@ -177,14 +177,24 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<HostVO> listHostsByMsAndDc(long msId, long dcId);
 
+    List<HostVO> listHostsByMsDcResourceState(long msId, long dcId, List<ResourceState> excludedResourceStates);
+
     List<HostVO> listHostsByMs(long msId);
 
+    List<HostVO> listHostsByMsResourceState(long msId, List<ResourceState> excludedResourceStates);
+
     /**
-     * Retrieves the number of hosts/agents this {@see ManagementServer} has responsibility over.
-     * @param msId the id of the {@see ManagementServer}
-     * @return the number of hosts/agents this {@see ManagementServer} has responsibility over
+     * Count Hosts by given Management Server, Host and Hypervisor Types,
+     * and exclude Hosts with given Resource States.
+     *
+     * @param msId                   Management Server Id
+     * @param excludedResourceStates Resource States to be excluded
+     * @param hostTypes              Host Types
+     * @param hypervisorTypes        Hypervisor Types
+     * @return Hosts count
      */
-    int countByMs(long msId);
+    int countHostsByMsResourceStateTypeAndHypervisorType(long msId, List<ResourceState> excludedResourceStates,
+                                                         List<Type> hostTypes, List<HypervisorType> hypervisorTypes);
 
     /**
      * Retrieves the host ids/agents this {@see ManagementServer} has responsibility over.