Support for custom SSH port for KVM hosts from the host url on add host and the configuration

api/src/main/java/com/cloud/host/Host.java

@@ -59,6 +59,7 @@ public static String[] toStrings(Host.Type... types) {
     String HOST_INSTANCE_CONVERSION = "host.instance.conversion";
     String HOST_OVFTOOL_VERSION = "host.ovftool.version";
     String HOST_VIRTV2V_VERSION = "host.virtv2v.version";
+    String HOST_SSH_POST = "host.ssh.port";
 
     /**
      * @return name of the machine.

api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddHostCmd.java

@@ -60,7 +60,8 @@ public class AddHostCmd extends BaseCmd {
     @Parameter(name = ApiConstants.POD_ID, type = CommandType.UUID, entityType = PodResponse.class, required = true, description = "The Pod ID for the host")
     private Long podId;
 
-    @Parameter(name = ApiConstants.URL, type = CommandType.STRING, required = true, description = "The host URL")
+    @Parameter(name = ApiConstants.URL, type = CommandType.STRING, required = true, description = "The host URL, optionally add ssh port for KVM hosts," +
+            " otherwise falls back to the port defined at the config 'kvm.host.discovery.ssh.port'")
     private String url;
 
     @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "The Zone ID for the host")

engine/components-api/src/main/java/com/cloud/agent/AgentManager.java

@@ -54,6 +54,10 @@ public interface AgentManager {
             "This timeout overrides the wait global config. This holds a comma separated key value pairs containing timeout (in seconds) for specific commands. " +
                     "For example: DhcpEntryCommand=600, SavePasswordCommand=300, VmDataCommand=300", false);
 
+    ConfigKey<Integer> KVMHostDiscoverySshPort = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class,
+            "kvm.host.discovery.ssh.port", "22", "SSH port used for KVM host discovery and any other operations on host (using SSH)." +
+                " Please note that this is applicable when port is not defined through host url while adding the KVM host.", true, ConfigKey.Scope.Cluster);
+
     enum TapAgentsAction {
         Add, Del, Contains,
     }
@@ -170,4 +174,6 @@ enum TapAgentsAction {
     void notifyMonitorsOfRemovedHost(long hostId, long clusterId);
 
     void propagateChangeToAgents(Map<String, String> params);
+
+    int getHostSshPort(HostVO host);
 }

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -40,6 +40,7 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.utils.StringUtils;
 import org.apache.cloudstack.agent.lb.IndirectAgentLB;
 import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
@@ -55,7 +56,6 @@
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.ThreadContext;
 
 import com.cloud.agent.AgentManager;
@@ -1977,7 +1977,7 @@ public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] { CheckTxnBeforeSending, Workers, Port, Wait, AlertWait, DirectAgentLoadSize,
                 DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait,
                 GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections,
-                RemoteAgentNewConnectionsMonitorInterval };
+                RemoteAgentNewConnectionsMonitorInterval, KVMHostDiscoverySshPort };
     }
 
     protected class SetHostParamsListener implements Listener {
@@ -2093,6 +2093,24 @@ public void propagateChangeToAgents(Map<String, String> params) {
         }
     }
 
+    @Override
+    public int getHostSshPort(HostVO host) {
+        if (host == null) {
+            return KVMHostDiscoverySshPort.value();
+        }
+
+        _hostDao.loadDetails(host);
+        String hostPort = host.getDetail(Host.HOST_SSH_POST);
+        int sshPort;
+        if (StringUtils.isBlank(hostPort)) {
+            sshPort = KVMHostDiscoverySshPort.valueIn(host.getClusterId());
+        } else {
+            sshPort = Integer.parseInt(hostPort);
+        }
+
+        return sshPort;
+    }
+
     private GlobalLock getHostJoinLock(Long hostId) {
         return GlobalLock.getInternLock(String.format("%s-%s", "Host-Join", hostId));
     }

plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.backup;
 
+import com.cloud.agent.AgentManager;
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.host.HostVO;
 import com.cloud.host.Status;
@@ -117,6 +118,9 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid
     @Inject
     private VMInstanceDao vmInstanceDao;
 
+    @Inject
+    private AgentManager agentMgr;
+
     private static String getUrlDomain(String url) throws URISyntaxException {
         URI uri;
         try {
@@ -229,8 +233,13 @@ private String executeBackupCommand(HostVO host, String username, String passwor
         String nstRegex = "\\bcompleted savetime=([0-9]{10})";
         Pattern saveTimePattern = Pattern.compile(nstRegex);
 
+        if (host == null) {
+            LOG.warn("Unable to take backup, host is null");
+            return null;
+        }
+
         try {
-            Pair<Boolean, String> response = SshHelper.sshExecute(host.getPrivateIpAddress(), 22,
+            Pair<Boolean, String> response = SshHelper.sshExecute(host.getPrivateIpAddress(), agentMgr.getHostSshPort(host),
                     username, null, password, command, 120000, 120000, 3600000);
             if (!response.first()) {
                 LOG.error(String.format("Backup Script failed on HYPERVISOR %s due to: %s", host, response.second()));
@@ -249,9 +258,13 @@ private String executeBackupCommand(HostVO host, String username, String passwor
         return null;
     }
     private boolean executeRestoreCommand(HostVO host, String username, String password, String command) {
+        if (host == null) {
+            LOG.warn("Unable to restore backup, host is null");
+            return false;
+        }
 
         try {
-            Pair<Boolean, String> response = SshHelper.sshExecute(host.getPrivateIpAddress(), 22,
+            Pair<Boolean, String> response = SshHelper.sshExecute(host.getPrivateIpAddress(), agentMgr.getHostSshPort(host),
                 username, null, password, command, 120000, 120000, 3600000);
 
             if (!response.first()) {

server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java

@@ -272,7 +272,12 @@ private void setupAgentSecurity(final Connection sshConnection, final String age
                 }
             }
 
-            sshConnection = new Connection(agentIp, 22);
+            int port = uri.getPort();
+            if (port <= 0) {
+                port = AgentManager.KVMHostDiscoverySshPort.valueIn(clusterId);
+            }
+
+            sshConnection = new Connection(agentIp, port);
 
             sshConnection.connect(null, 60000, 60000);
 
@@ -380,6 +385,9 @@ private void setupAgentSecurity(final Connection sshConnection, final String age
             Map<String, String> hostDetails = connectedHost.getDetails();
             hostDetails.put("password", password);
             hostDetails.put("username", username);
+            if (uri.getPort() > 0) {
+                hostDetails.put(Host.HOST_SSH_POST, String.valueOf(uri.getPort()));
+            }
             _hostDao.saveDetails(connectedHost);
             return resources;
         } catch (DiscoveredWithErrorException e) {

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -776,7 +776,6 @@ private List<HostVO> discoverHostsFull(final Long dcId, final Long podId, Long c
                 _clusterDetailsDao.persist(cluster_cpu_detail);
                 _clusterDetailsDao.persist(cluster_memory_detail);
             }
-
         }
 
         try {
@@ -871,7 +870,6 @@ private List<HostVO> discoverHostsFull(final Long dcId, final Long podId, Long c
                         hosts.add(host);
                     }
                     discoverer.postDiscovery(hosts, _nodeId);
-
                 }
                 logger.info("server resources successfully discovered by " + discoverer.getName());
                 return hosts;
@@ -2949,7 +2947,7 @@ protected Ternary<String, String, String> getHostCredentials(HostVO host) {
      */
     protected void connectAndRestartAgentOnHost(HostVO host, String username, String password, String privateKey) {
         final com.trilead.ssh2.Connection connection = SSHCmdHelper.acquireAuthorizedConnection(
-                host.getPrivateIpAddress(), 22, username, password, privateKey);
+                host.getPrivateIpAddress(), _agentMgr.getHostSshPort(host), username, password, privateKey);
         if (connection == null) {
             throw new CloudRuntimeException(String.format("SSH to agent is enabled, but failed to connect to %s via IP address [%s].", host, host.getPrivateIpAddress()));
         }

server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java

@@ -360,12 +360,14 @@ public void testConnectAndRestartAgentOnHostCannotRestart() throws Exception {
 
     @Test
     public void testConnectAndRestartAgentOnHost() {
+        when(agentManager.getHostSshPort(any())).thenReturn(22);
         resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword, hostPrivateKey);
     }
 
     @Test
     public void testHandleAgentSSHEnabledNotConnectedAgent() {
         when(host.getStatus()).thenReturn(Status.Disconnected);
+        when(agentManager.getHostSshPort(any())).thenReturn(22);
         resourceManager.handleAgentIfNotConnected(host, false);
         verify(resourceManager).getHostCredentials(eq(host));
         verify(resourceManager).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword), eq(hostPrivateKey));

utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java

@@ -77,7 +77,7 @@ public static com.trilead.ssh2.Connection acquireAuthorizedConnection(String ip,
     }
 
     public static com.trilead.ssh2.Connection acquireAuthorizedConnection(String ip, int port, String username, String password) {
-        return acquireAuthorizedConnection(ip, 22, username, password, null);
+        return acquireAuthorizedConnection(ip, port, username, password, null);
     }
 
     public static boolean acquireAuthorizedConnectionWithPublicKey(final com.trilead.ssh2.Connection sshConnection, final String username, final String privateKey) {