Merge branch 'main' into pre-commit-update-codespell

Author: jbampton

.github/workflows/ci.yml

@@ -164,7 +164,8 @@ jobs:
                   component/test_cpu_limits
                   component/test_cpu_max_limits
                   component/test_cpu_project_limits
-                  component/test_deploy_vm_userdata_multi_nic",
+                  component/test_deploy_vm_userdata_multi_nic
+                  component/test_deploy_vm_lease",
                 "component/test_egress_fw_rules
                   component/test_invalid_gw_nm
                   component/test_ip_reservation",
@@ -236,7 +237,7 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid pynose
+          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose
 
       - name: Install jacoco dependencies
         run: |

.github/workflows/main-sonar-check.yml

@@ -54,7 +54,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-m2
 

.github/workflows/sonar-check.yml

@@ -56,7 +56,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-m2
 

INSTALL.md

@@ -20,10 +20,10 @@ Install tools and dependencies used for development:
 
 Set up Maven (3.6.0):
 
-    # wget http://www.us.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
-    # tar -zxvf apache-maven-3.6.3-bin.tar.gz -C /usr/local
+    # wget https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz
+    # tar -zxvf apache-maven-3.9.9-bin.tar.gz -C /usr/local
     # cd /usr/local
-    # ln -s apache-maven-3.6.3 maven
+    # ln -s apache-maven-3.9.9 maven
     # echo export M2_HOME=/usr/local/maven >> ~/.bashrc # or .zshrc or .profile
     # echo export PATH=/usr/local/maven/bin:${PATH} >> ~/.bashrc # or .zshrc or .profile
     # source ~/.bashrc

agent/conf/agent.properties

@@ -441,3 +441,9 @@ iscsi.session.cleanup.enabled=false
 
 # Wait(in seconds) during agent reconnections. When no value is set then default value of 5s will be used
 #backoff.seconds=
+
+# Timeout (in seconds) to wait for the snapshot reversion to complete.
+# revert.snapshot.timeout=10800
+
+# Timeout (in seconds) to wait for the incremental snapshot to complete.
+# incremental.snapshot.timeout=10800

agent/src/main/java/com/cloud/agent/Agent.java

@@ -800,6 +800,9 @@ protected void processRequest(final Request request, final Link link) {
                         }
                         commandsInProgress.incrementAndGet();
                         try {
+                            if (cmd.isReconcile()) {
+                                cmd.setRequestSequence(request.getSequence());
+                            }
                             answer = serverResource.executeRequest(cmd);
                         } finally {
                             commandsInProgress.decrementAndGet();
@@ -1021,6 +1024,8 @@ private void processPingAnswer(final PingAnswer answer) {
         if ((answer.isSendStartup()) && reconnectAllowed) {
             logger.info("Management server requested startup command to reinitialize the agent");
             sendStartup(link);
+        } else {
+            serverResource.processPingAnswer((PingAnswer) answer);
         }
         shell.setAvoidHosts(answer.getAvoidMsList());
     }
@@ -1087,6 +1092,9 @@ public void processOtherTask(final Task task) {
             Answer answer = null;
             commandsInProgress.incrementAndGet();
             try {
+                if (command.isReconcile()) {
+                    command.setRequestSequence(req.getSequence());
+                }
                 answer = serverResource.executeRequest(command);
             } finally {
                 commandsInProgress.decrementAndGet();

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -818,6 +818,16 @@ public Property<Integer> getWorkers() {
      */
     public static final Property<Integer> SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", 30, Integer.class);
 
+    /**
+     * Timeout (in seconds) to wait for the incremental snapshot to complete.
+     * */
+    public static final Property<Integer> INCREMENTAL_SNAPSHOT_TIMEOUT = new Property<>("incremental.snapshot.timeout", 10800);
+
+    /**
+     * Timeout (in seconds) to wait for the snapshot reversion to complete.
+     * */
+    public static final Property<Integer> REVERT_SNAPSHOT_TIMEOUT = new Property<>("revert.snapshot.timeout", 10800);
+
     public static class Property <T>{
         private String name;
         private T defaultValue;

api/src/main/java/com/cloud/agent/api/Command.java

@@ -35,13 +35,31 @@ public static enum OnError {
         Continue, Stop
     }
 
+    public enum State {
+        CREATED,        // Command is created by management server
+        STARTED,        // Command is started by agent
+        PROCESSING,     // Processing by agent
+        PROCESSING_IN_BACKEND,  // Processing in backend by agent
+        COMPLETED,      // Operation succeeds by agent or management server
+        FAILED,         // Operation fails by agent
+        RECONCILE_RETRY,        // Ready for retry of reconciliation
+        RECONCILING,    // Being reconciled by management server
+        RECONCILED,     // Reconciled by management server
+        RECONCILE_SKIPPED,  // Skip the reconciliation as the resource state is inconsistent with the command
+        RECONCILE_FAILED,       // Fail to reconcile by management server
+        TIMED_OUT,      // Timed out on management server or agent
+        INTERRUPTED,    // Interrupted by management server or agent (for example agent is restarted),
+        DANGLED_IN_BACKEND     // Backend process which cannot be processed normally (for example agent is restarted)
+    }
+
     public static final String HYPERVISOR_TYPE = "hypervisorType";
 
     // allow command to carry over hypervisor or other environment related context info
     @LogLevel(Log4jLevel.Trace)
     protected Map<String, String> contextMap = new HashMap<String, String>();
     private int wait;  //in second
     private boolean bypassHostMaintenance = false;
+    private transient long requestSequence = 0L;
 
     protected Command() {
         this.wait = 0;
@@ -82,6 +100,10 @@ public String getContextParam(String name) {
         return contextMap.get(name);
     }
 
+    public Map<String, String> getContextMap() {
+        return contextMap;
+    }
+
     public boolean allowCaching() {
         return true;
     }
@@ -94,6 +116,18 @@ public void setBypassHostMaintenance(boolean bypassHostMaintenance) {
         this.bypassHostMaintenance = bypassHostMaintenance;
     }
 
+    public boolean isReconcile() {
+        return false;
+    }
+
+    public long getRequestSequence() {
+        return requestSequence;
+    }
+
+    public void setRequestSequence(long requestSequence) {
+        this.requestSequence = requestSequence;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

api/src/main/java/com/cloud/agent/api/to/DiskTO.java

@@ -46,7 +46,7 @@ public class DiskTO {
     private Long diskSeq;
     private String path;
     private Volume.Type type;
-    private Map<String, String> _details;
+    private Map<String, String> details;
 
     public DiskTO() {
 
@@ -92,10 +92,10 @@ public void setType(Volume.Type type) {
     }
 
     public void setDetails(Map<String, String> details) {
-        _details = details;
+        this.details = details;
     }
 
     public Map<String, String> getDetails() {
-        return _details;
+        return details;
     }
 }

api/src/main/java/com/cloud/agent/api/to/NetworkTO.java

@@ -36,7 +36,7 @@ public class NetworkTO {
     protected TrafficType type;
     protected URI broadcastUri;
     protected URI isolationUri;
-    protected boolean isSecurityGroupEnabled;
+    protected boolean securityGroupEnabled;
     protected String name;
     protected String ip6address;
     protected String ip6gateway;
@@ -112,7 +112,7 @@ public String getName() {
     }
 
     public void setSecurityGroupEnabled(boolean enabled) {
-        this.isSecurityGroupEnabled = enabled;
+        this.securityGroupEnabled = enabled;
     }
 
     /**
@@ -221,7 +221,7 @@ public void setIsolationuri(URI isolationUri) {
     }
 
     public boolean isSecurityGroupEnabled() {
-        return this.isSecurityGroupEnabled;
+        return this.securityGroupEnabled;
     }
 
     public void setIp6Dns1(String ip6Dns1) {