Merge release branch 4.20 to main

* 4.20:
  linstor: Fix ZFS snapshot backup (#10219)
  fix listing of VMs by network (#10204)
  Configure org.eclipse.jetty.server.Request.maxFormKeys from server.properties and increase the default value (#10214)
  api: fix access for listSystemVmUsageHistory (#10032)
  Fix NPE issues during host rolling maintenance, due to host tags and custom constrained/unconstrained service offering (#9844)

Author: DaanHoogland

client/conf/server.properties.in

@@ -32,6 +32,9 @@ session.timeout=30
 # Max allowed API request payload/content size in bytes
 request.content.size=1048576
 
+# Max allowed API request form keys
+request.max.form.keys=5000
+
 # Options to configure and enable HTTPS on the management server
 #
 # For the management server to pick up these configuration settings, the configured

client/src/main/java/org/apache/cloudstack/ServerDaemon.java

@@ -82,6 +82,8 @@ public class ServerDaemon implements Daemon {
     private static final String ACCESS_LOG = "access.log";
     private static final String REQUEST_CONTENT_SIZE_KEY = "request.content.size";
     private static final int DEFAULT_REQUEST_CONTENT_SIZE = 1048576;
+    private static final String REQUEST_MAX_FORM_KEYS_KEY = "request.max.form.keys";
+    private static final int DEFAULT_REQUEST_MAX_FORM_KEYS = 5000;
 
     ////////////////////////////////////////////////////////
     /////////////// Server Configuration ///////////////////
@@ -94,6 +96,7 @@ public class ServerDaemon implements Daemon {
     private int httpsPort = 8443;
     private int sessionTimeout = 30;
     private int maxFormContentSize = DEFAULT_REQUEST_CONTENT_SIZE;
+    private int maxFormKeys = DEFAULT_REQUEST_MAX_FORM_KEYS;
     private boolean httpsEnable = false;
     private String accessLogFile = "access.log";
     private String bindInterface = null;
@@ -141,6 +144,7 @@ public void init(final DaemonContext context) {
             setAccessLogFile(properties.getProperty(ACCESS_LOG, "access.log"));
             setSessionTimeout(Integer.valueOf(properties.getProperty(SESSION_TIMEOUT, "30")));
             setMaxFormContentSize(Integer.valueOf(properties.getProperty(REQUEST_CONTENT_SIZE_KEY, String.valueOf(DEFAULT_REQUEST_CONTENT_SIZE))));
+            setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS))));
         } catch (final IOException e) {
             logger.warn("Failed to read configuration from server.properties file", e);
         } finally {
@@ -192,6 +196,7 @@ public void start() throws Exception {
         // Extra config options
         server.setStopAtShutdown(true);
         server.setAttribute(ContextHandler.MAX_FORM_CONTENT_SIZE_KEY, maxFormContentSize);
+        server.setAttribute(ContextHandler.MAX_FORM_KEYS_KEY, maxFormKeys);
 
         // HTTPS Connector
         createHttpsConnector(httpConfig);
@@ -264,6 +269,7 @@ private Pair<SessionHandler,HandlerCollection> createHandlers() {
         webApp.setContextPath(contextPath);
         webApp.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
         webApp.setMaxFormContentSize(maxFormContentSize);
+        webApp.setMaxFormKeys(maxFormKeys);
 
         // GZIP handler
         final GzipHandler gzipHandler = new GzipHandler();
@@ -366,4 +372,8 @@ public void setSessionTimeout(int sessionTimeout) {
     public void setMaxFormContentSize(int maxFormContentSize) {
         this.maxFormContentSize = maxFormContentSize;
     }
+
+    public void setMaxFormKeys(int maxFormKeys) {
+        this.maxFormKeys = maxFormKeys;
+    }
 }

plugins/metrics/src/main/java/org/apache/cloudstack/api/ListSystemVMsUsageHistoryCmd.java

@@ -26,7 +26,7 @@
 
 @APICommand(name = "listSystemVmsUsageHistory", description = "Lists System VM stats", responseObject = VmMetricsStatsResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.18.0",
-        authorized = {RoleType.Admin,  RoleType.ResourceAdmin, RoleType.DomainAdmin})
+        authorized = {RoleType.Admin})
 public class ListSystemVMsUsageHistoryCmd extends BaseResourceUsageHistoryCmd {
 
     /////////////////////////////////////////////////////

plugins/storage/volume/linstor/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2025-01-20]
+
+### Fixed
+
+- Volume snapshots on zfs used the wrong dataset path to hide/unhide snapdev
+
 ## [2024-12-13]
 
 ### Fixed

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java

@@ -46,11 +46,17 @@ public final class LinstorBackupSnapshotCommandWrapper
 {
     protected static Logger LOGGER = LogManager.getLogger(LinstorBackupSnapshotCommandWrapper.class);
 
+    private static String zfsDatasetName(String zfsFullSnapshotUrl) {
+        String zfsFullPath = zfsFullSnapshotUrl.substring(6);
+        int atPos = zfsFullPath.indexOf('@');
+        return atPos >= 0 ? zfsFullPath.substring(0, atPos) : zfsFullPath;
+    }
+
     private String zfsSnapdev(boolean hide, String zfsUrl) {
-        Script script = new Script("/usr/bin/zfs", Duration.millis(5000));
+        Script script = new Script("zfs", Duration.millis(5000));
         script.add("set");
         script.add("snapdev=" + (hide ? "hidden" : "visible"));
-        script.add(zfsUrl.substring(6));  // cutting zfs://
+        script.add(zfsDatasetName(zfsUrl));  // cutting zfs:// and @snapshotname
         return script.execute();
     }
 
@@ -134,10 +140,10 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR
             LOGGER.info("Src: " + srcPath + " | " + src.getName());
             if (srcPath.startsWith("zfs://")) {
                 zfsHidden = true;
-                if (zfsSnapdev(false, srcPath) != null) {
+                if (zfsSnapdev(false, src.getPath()) != null) {
                     return new CopyCmdAnswer("Unable to unhide zfs snapshot device.");
                 }
-                srcPath = "/dev/" + srcPath.substring(6);
+                srcPath = "/dev/zvol/" + srcPath.substring(6);
             }
 
             secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl());

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -1413,6 +1413,7 @@ private Pair<List<Long>, Integer> searchForUserVMIdsAndCount(ListVMsCmd cmd) {
         if (networkId != null || vpcId != null) {
             SearchBuilder<NicVO> nicSearch = nicDao.createSearchBuilder();
             nicSearch.and("networkId", nicSearch.entity().getNetworkId(), Op.EQ);
+            nicSearch.and("removed", nicSearch.entity().getRemoved(), Op.NULL);
             if (vpcId != null) {
                 SearchBuilder<NetworkVO> networkSearch = networkDao.createSearchBuilder();
                 networkSearch.and("vpcId", networkSearch.entity().getVpcId(), Op.EQ);

server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java

@@ -37,6 +37,7 @@
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ObjectUtils;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
@@ -64,12 +65,16 @@
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.utils.Pair;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.UserVmDetailVO;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine.State;
 import com.cloud.vm.VirtualMachineProfileImpl;
+import com.cloud.vm.VmDetailConstants;
+import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
 
 public class RollingMaintenanceManagerImpl extends ManagerBase implements RollingMaintenanceManager {
@@ -85,6 +90,8 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin
     @Inject
     private VMInstanceDao vmInstanceDao;
     @Inject
+    protected UserVmDetailsDao userVmDetailsDao;
+    @Inject
     private ServiceOfferingDao serviceOfferingDao;
     @Inject
     private ClusterDetailsDao clusterDetailsDao;
@@ -619,10 +626,19 @@ private Pair<Boolean, String> performCapacityChecksBeforeHostInMaintenance(Host
         int successfullyCheckedVmMigrations = 0;
         for (VMInstanceVO runningVM : vmsRunning) {
             boolean canMigrateVm = false;
+            Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = getComputeResourcesCpuSpeedAndRamSize(runningVM);
+            Integer cpu = cpuSpeedAndRamSize.first();
+            Integer speed = cpuSpeedAndRamSize.second();
+            Integer ramSize = cpuSpeedAndRamSize.third();
+            if (ObjectUtils.anyNull(cpu, speed, ramSize)) {
+                logger.warn("Cannot fetch compute resources for the VM {}, skipping it from the capacity check", runningVM);
+                continue;
+            }
+
             ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(runningVM.getServiceOfferingId());
             for (Host hostInCluster : hostsInCluster) {
                 if (!checkHostTags(hostTags, hostTagsDao.getHostTags(hostInCluster.getId()), serviceOffering.getHostTag())) {
-                    logger.debug(String.format("Host tags mismatch between %s and %s Skipping it from the capacity check", host, hostInCluster));
+                    logger.debug("Host tags mismatch between {} and {} Skipping it from the capacity check", host, hostInCluster);
                     continue;
                 }
                 DeployDestination deployDestination = new DeployDestination(null, null, null, host);
@@ -632,13 +648,13 @@ private Pair<Boolean, String> performCapacityChecksBeforeHostInMaintenance(Host
                     affinityChecks = affinityChecks && affinityProcessor.check(vmProfile, deployDestination);
                 }
                 if (!affinityChecks) {
-                    logger.debug(String.format("Affinity check failed between %s and %s Skipping it from the capacity check", host, hostInCluster));
+                    logger.debug("Affinity check failed between {} and {} Skipping it from the capacity check", host, hostInCluster);
                     continue;
                 }
                 boolean maxGuestLimit = capacityManager.checkIfHostReachMaxGuestLimit(host);
-                boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster.getId(), serviceOffering.getCpu(), serviceOffering.getSpeed());
-                int cpuRequested = serviceOffering.getCpu() * serviceOffering.getSpeed();
-                long ramRequested = serviceOffering.getRamSize() * 1024L * 1024L;
+                boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster.getId(), cpu, speed);
+                int cpuRequested = cpu * speed;
+                long ramRequested = ramSize * 1024L * 1024L;
                 ClusterDetailsVO clusterDetailsCpuOvercommit = clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
                 ClusterDetailsVO clusterDetailsRamOvercommmt = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
                 Float cpuOvercommitRatio = Float.parseFloat(clusterDetailsCpuOvercommit.getValue());
@@ -664,11 +680,42 @@ private Pair<Boolean, String> performCapacityChecksBeforeHostInMaintenance(Host
         return new Pair<>(true, "OK");
     }
 
+    protected Ternary<Integer, Integer, Integer> getComputeResourcesCpuSpeedAndRamSize(VMInstanceVO runningVM) {
+        ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(runningVM.getServiceOfferingId());
+        Integer cpu = serviceOffering.getCpu();
+        Integer speed = serviceOffering.getSpeed();
+        Integer ramSize = serviceOffering.getRamSize();
+        if (!serviceOffering.isDynamic()) {
+            return new Ternary<>(cpu, speed, ramSize);
+        }
+
+        List<UserVmDetailVO> vmDetails = userVmDetailsDao.listDetails(runningVM.getId());
+        if (CollectionUtils.isEmpty(vmDetails)) {
+            return new Ternary<>(cpu, speed, ramSize);
+        }
+
+        for (UserVmDetailVO vmDetail : vmDetails) {
+            if (StringUtils.isBlank(vmDetail.getName()) || StringUtils.isBlank(vmDetail.getValue())) {
+                continue;
+            }
+
+            if (cpu == null && VmDetailConstants.CPU_NUMBER.equals(vmDetail.getName())) {
+                cpu = Integer.valueOf(vmDetail.getValue());
+            } else if (speed == null && VmDetailConstants.CPU_SPEED.equals(vmDetail.getName())) {
+                speed = Integer.valueOf(vmDetail.getValue());
+            } else if (ramSize == null && VmDetailConstants.MEMORY.equals(vmDetail.getName())) {
+                ramSize = Integer.valueOf(vmDetail.getValue());
+            }
+        }
+
+        return new Ternary<>(cpu, speed, ramSize);
+    }
+
     /**
      * Check hosts tags
      */
     private boolean checkHostTags(List<HostTagVO> hostTags, List<HostTagVO> hostInClusterTags, String offeringTag) {
-        if (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) {
+        if ((CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) || StringUtils.isBlank(offeringTag)) {
             return true;
         } else if ((CollectionUtils.isNotEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) ||
                 (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isNotEmpty(hostInClusterTags))) {

server/src/test/java/com/cloud/resource/RollingMaintenanceManagerImplTest.java

@@ -23,7 +23,15 @@
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.org.Cluster;
+import com.cloud.service.ServiceOfferingVO;
+import com.cloud.service.dao.ServiceOfferingDao;
+import com.cloud.utils.Ternary;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.UserVmDetailVO;
+import com.cloud.vm.VMInstanceVO;
+import com.cloud.vm.VmDetailConstants;
+import com.cloud.vm.dao.UserVmDetailsDao;
+
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -54,6 +62,12 @@ public class RollingMaintenanceManagerImplTest {
     HostVO host4;
     @Mock
     Cluster cluster;
+    @Mock
+    VMInstanceVO vm;
+    @Mock
+    ServiceOfferingDao serviceOfferingDao;
+    @Mock
+    UserVmDetailsDao userVmDetailsDao;
 
     @Spy
     @InjectMocks
@@ -172,4 +186,50 @@ public void testPerformStateChecksForce() {
 
         Assert.assertEquals(1, hosts.size());
     }
+
+    @Test
+    public void testGetComputeResourcesCpuSpeedAndRamSize_ForNormalOffering() {
+        ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class);
+        Mockito.when(serviceOffering.isDynamic()).thenReturn(false);
+        Mockito.when(serviceOffering.getCpu()).thenReturn(1);
+        Mockito.when(serviceOffering.getSpeed()).thenReturn(500);
+        Mockito.when(serviceOffering.getRamSize()).thenReturn(512);
+
+        Mockito.when(vm.getServiceOfferingId()).thenReturn(1L);
+        Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering);
+
+        Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = manager.getComputeResourcesCpuSpeedAndRamSize(vm);
+
+        Assert.assertEquals(1, cpuSpeedAndRamSize.first().intValue());
+        Assert.assertEquals(500, cpuSpeedAndRamSize.second().intValue());
+        Assert.assertEquals(512, cpuSpeedAndRamSize.third().intValue());
+    }
+
+    @Test
+    public void testGetComputeResourcesCpuSpeedAndRamSize_ForCustomOffering() {
+        ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class);
+        Mockito.when(serviceOffering.isDynamic()).thenReturn(true);
+        Mockito.when(serviceOffering.getCpu()).thenReturn(null);
+        Mockito.when(serviceOffering.getSpeed()).thenReturn(null);
+        Mockito.when(serviceOffering.getRamSize()).thenReturn(null);
+
+        List<UserVmDetailVO> vmDetails = new ArrayList<>();
+        UserVmDetailVO cpuDetail = new UserVmDetailVO(1L, VmDetailConstants.CPU_NUMBER, "2",  false);
+        vmDetails.add(cpuDetail);
+        UserVmDetailVO speedDetail = new UserVmDetailVO(1L, VmDetailConstants.CPU_SPEED, "1000",  false);
+        vmDetails.add(speedDetail);
+        UserVmDetailVO ramSizeDetail = new UserVmDetailVO(1L, VmDetailConstants.MEMORY, "1024",  false);
+        vmDetails.add(ramSizeDetail);
+
+        Mockito.when(vm.getId()).thenReturn(1L);
+        Mockito.when(vm.getServiceOfferingId()).thenReturn(1L);
+        Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering);
+        Mockito.when(userVmDetailsDao.listDetails(1L)).thenReturn(vmDetails);
+
+        Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = manager.getComputeResourcesCpuSpeedAndRamSize(vm);
+
+        Assert.assertEquals(2, cpuSpeedAndRamSize.first().intValue());
+        Assert.assertEquals(1000, cpuSpeedAndRamSize.second().intValue());
+        Assert.assertEquals(1024, cpuSpeedAndRamSize.third().intValue());
+    }
 }