Fix write retries, Pipe resource cleanup, subscription progress, and snapshot recovery

iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java

@@ -1384,12 +1384,14 @@ private ProcedureMessages() {}
   public static final String LOG_MODE_ARG_SKIPPING_PIPE_CREATION_5F4D1026 = "(mode={}), skipping pipe creation";
   public static final String LOG_CREATESUBSCRIPTIONPROCEDURE_CONSENSUS_BASED_TOPICS_ARG_WILL_HANDLED_DATANODE_90A9C2FD = "CreateSubscriptionProcedure: consensus-based topics {} will be handled by DataNode ";
   public static final String LOG_VIA_CONSUMER_GROUP_META_PUSH_NO_PIPE_CREATION_NEEDED_D56CFE31 = "via consumer group meta push (no pipe creation needed)";
+  public static final String LOG_CREATESUBSCRIPTIONPROCEDURE_SYNCHRONIZING_COMMIT_PROGRESS_AFTER_CONSUMER_GROUP_NEWLY_SUBSCRIBED_CONSENSUS_TOPICS_ARG_F5687D36 = "CreateSubscriptionProcedure: synchronizing commit progress after consumer group newly subscribed consensus topics {}";
   public static final String LOG_DROPSUBSCRIPTIONPROCEDURE_TOPIC_ARG_USES_CONSENSUS_SUBSCRIPTION_MODE_6962D13C = "DropSubscriptionProcedure: topic [{}] uses consensus subscription mode ";
   public static final String LOG_MODE_ARG_SKIPPING_PIPE_REMOVAL_133B0CD6 = "(mode={}), skipping pipe removal";
   public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_ACQUIRELOCK_SKIP_PROCEDURE_LAST_EXECUTION_TIME_ARG_CE3DD247 = "CommitProgressSyncProcedure: acquireLock, skip the procedure due to the last execution time {}";
   public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMVALIDATE_CF220E1F = "CommitProgressSyncProcedure: executeFromValidate";
   public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMOPERATEONCONFIGNODES_0DC818CA = "CommitProgressSyncProcedure: executeFromOperateOnConfigNodes";
   public static final String LOG_FAILED_PULL_COMMIT_PROGRESS_DATANODE_ARG_STATUS_ARG_33037B29 = "Failed to pull commit progress from DataNode {}, status: {}";
+  public static final String EXCEPTION_FAILED_TO_PULL_REQUIRED_COMMIT_PROGRESS_FROM_DATANODE_ARG_STATUS_ARG_E08A2450 = "Failed to pull required commit progress from DataNode %s, status: %s";
   public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMOPERATEONDATANODES_NO_OP_34420360 = "CommitProgressSyncProcedure: executeFromOperateOnDataNodes (no-op)";
   public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_ROLLBACKFROMVALIDATE_2309D4D2 = "CommitProgressSyncProcedure: rollbackFromValidate";
   public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_ROLLBACKFROMOPERATEONCONFIGNODES_57CB907B = "CommitProgressSyncProcedure: rollbackFromOperateOnConfigNodes";

iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java

@@ -1431,6 +1431,9 @@ private ProcedureMessages() {}
           "CreateSubscriptionProcedure：基于共识的 topics {} 将由 DataNode 处理";
   public static final String LOG_VIA_CONSUMER_GROUP_META_PUSH_NO_PIPE_CREATION_NEEDED_D56CFE31 =
       "通过 consumer group 元数据推送（无需创建 pipe）";
+  public static final String
+      LOG_CREATESUBSCRIPTIONPROCEDURE_SYNCHRONIZING_COMMIT_PROGRESS_AFTER_CONSUMER_GROUP_NEWLY_SUBSCRIBED_CONSENSUS_TOPICS_ARG_F5687D36 =
+          "CreateSubscriptionProcedure：consumer group 新订阅共识 topic {} 后同步提交进度";
   public static final String
       LOG_DROPSUBSCRIPTIONPROCEDURE_TOPIC_ARG_USES_CONSENSUS_SUBSCRIPTION_MODE_6962D13C =
           "DropSubscriptionProcedure: topic [{}] 使用共识订阅模式 ";
@@ -1445,6 +1448,9 @@ private ProcedureMessages() {}
           "CommitProgressSyncProcedure: executeFromOperateOnConfigNodes";
   public static final String LOG_FAILED_PULL_COMMIT_PROGRESS_DATANODE_ARG_STATUS_ARG_33037B29 =
       "拉取 DataNode {} 的提交进度失败，状态：{}";
+  public static final String
+      EXCEPTION_FAILED_TO_PULL_REQUIRED_COMMIT_PROGRESS_FROM_DATANODE_ARG_STATUS_ARG_E08A2450 =
+          "从 DataNode %s 拉取必要的提交进度失败，状态：%s";
   public static final String
       LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMOPERATEONDATANODES_NO_OP_34420360 =
           "CommitProgressSyncProcedure: executeFromOperateOnDataNodes（无操作）";

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java

@@ -47,6 +47,7 @@
 import org.apache.iotdb.commons.conf.ConfigurationFileUtils;
 import org.apache.iotdb.commons.conf.IoTDBConstant;
 import org.apache.iotdb.commons.conf.TrimProperties;
+import org.apache.iotdb.commons.consensus.DataRegionId;
 import org.apache.iotdb.commons.enums.RepairDataPartitionTableProgressState;
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.exception.MetadataException;
@@ -66,6 +67,7 @@
 import org.apache.iotdb.commons.schema.tree.AlterTimeSeriesOperationType;
 import org.apache.iotdb.commons.schema.ttl.TTLCache;
 import org.apache.iotdb.commons.service.metric.MetricService;
+import org.apache.iotdb.commons.subscription.meta.consumer.CommitProgressKeeper;
 import org.apache.iotdb.commons.utils.AuthUtils;
 import org.apache.iotdb.commons.utils.PathUtils;
 import org.apache.iotdb.commons.utils.StatusUtils;
@@ -2679,19 +2681,93 @@ public TGetCommitProgressResp getCommitProgress(TGetCommitProgressReq req) {
     if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
       return new TGetCommitProgressResp(status);
     }
-    final String keyPrefix =
-        req.getConsumerGroupId() + "##" + req.getTopicName() + "##" + req.getRegionId() + "##";
-    final org.apache.iotdb.commons.subscription.meta.consumer.CommitProgressKeeper keeper =
+    final CommitProgressKeeper keeper =
         subscriptionManager
             .getSubscriptionCoordinator()
             .getSubscriptionInfo()
             .getCommitProgressKeeper();
-    final Map<WriterId, WriterProgress> mergedWriterPositions = new LinkedHashMap<>();
+    return buildCommitProgressResponse(
+        keeper.getAllRegionProgress(),
+        req.getConsumerGroupId(),
+        req.getTopicName(),
+        req.getRegionId());
+  }
+
+  static TGetCommitProgressResp buildCommitProgressResponse(
+      final Map<String, ByteBuffer> allRegionProgress,
+      final String consumerGroupId,
+      final String topicName,
+      final int regionId) {
+    final CommitProgressMergeResult mergeResult =
+        mergeCommitProgressWithPresence(allRegionProgress, consumerGroupId, topicName, regionId);
+    final TGetCommitProgressResp resp =
+        new TGetCommitProgressResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+    if (mergeResult.hasMatchingProgress) {
+      resp.setCommittedRegionProgress(serializeRegionProgress(mergeResult.regionProgress));
+    }
+    return resp;
+  }
 
-    for (final Map.Entry<String, ByteBuffer> entry : keeper.getAllRegionProgress().entrySet()) {
-      if (!entry.getKey().startsWith(keyPrefix)) {
+  static RegionProgress mergeCommitProgress(
+      final Map<String, ByteBuffer> allRegionProgress,
+      final String consumerGroupId,
+      final String topicName,
+      final int regionId) {
+    return mergeCommitProgressWithPresence(allRegionProgress, consumerGroupId, topicName, regionId)
+        .regionProgress;
+  }
+
+  private static CommitProgressMergeResult mergeCommitProgressWithPresence(
+      final Map<String, ByteBuffer> allRegionProgress,
+      final String consumerGroupId,
+      final String topicName,
+      final int regionId) {
+    final String regionIdString = new DataRegionId(regionId).toString();
+    final Map<WriterId, WriterProgress> mergedWriterPositions = new LinkedHashMap<>();
+    final boolean hasVersionedProgress =
+        mergeCommitProgressForPrefix(
+            allRegionProgress,
+            CommitProgressKeeper.generateRegionKeyPrefix(
+                consumerGroupId, topicName, regionIdString),
+            mergedWriterPositions);
+    boolean hasMatchingProgress = hasVersionedProgress;
+    if (!hasVersionedProgress
+        || CommitProgressKeeper.isLegacyKeyUnambiguous(
+            consumerGroupId, topicName, regionIdString)) {
+      final boolean hasLegacyProgress =
+          mergeCommitProgressForPrefix(
+              allRegionProgress,
+              CommitProgressKeeper.generateLegacyRegionKeyPrefix(
+                  consumerGroupId, topicName, regionIdString),
+              mergedWriterPositions);
+      hasMatchingProgress = hasMatchingProgress || hasLegacyProgress;
+    }
+    return new CommitProgressMergeResult(
+        new RegionProgress(mergedWriterPositions), hasMatchingProgress);
+  }
+
+  private static final class CommitProgressMergeResult {
+
+    private final RegionProgress regionProgress;
+    private final boolean hasMatchingProgress;
+
+    private CommitProgressMergeResult(
+        final RegionProgress regionProgress, final boolean hasMatchingProgress) {
+      this.regionProgress = regionProgress;
+      this.hasMatchingProgress = hasMatchingProgress;
+    }
+  }
+
+  private static boolean mergeCommitProgressForPrefix(
+      final Map<String, ByteBuffer> allRegionProgress,
+      final String keyPrefix,
+      final Map<WriterId, WriterProgress> mergedWriterPositions) {
+    boolean hasMatchingProgress = false;
+    for (final Map.Entry<String, ByteBuffer> entry : allRegionProgress.entrySet()) {
+      if (!CommitProgressKeeper.isValidDataNodeProgressKey(entry.getKey(), keyPrefix)) {
         continue;
       }
+      hasMatchingProgress = true;
       final RegionProgress regionProgress = deserializeRegionProgress(entry.getValue());
       if (Objects.isNull(regionProgress)) {
         continue;
@@ -2705,13 +2781,7 @@ public TGetCommitProgressResp getCommitProgress(TGetCommitProgressReq req) {
                 compareWriterProgress(newProgress, oldProgress) > 0 ? newProgress : oldProgress);
       }
     }
-    final TGetCommitProgressResp resp =
-        new TGetCommitProgressResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
-    if (!mergedWriterPositions.isEmpty()) {
-      resp.setCommittedRegionProgress(
-          serializeRegionProgress(new RegionProgress(mergedWriterPositions)));
-    }
-    return resp;
+    return hasMatchingProgress;
   }
 
   private static RegionProgress deserializeRegionProgress(final ByteBuffer buffer) {
@@ -2727,7 +2797,7 @@ private static ByteBuffer serializeRegionProgress(final RegionProgress regionPro
     try (final PublicBAOS baos = new PublicBAOS();
         final DataOutputStream dos = new DataOutputStream(baos)) {
       regionProgress.serialize(dos);
-      return ByteBuffer.wrap(baos.getBuf(), 0, baos.size()).asReadOnlyBuffer();
+      return ByteBuffer.wrap(baos.getBuf(), 0, baos.size());
     } catch (final IOException e) {
       throw new RuntimeException(
           ManagerMessages.EXCEPTION_FAILED_SERIALIZE_REGION_PROGRESS_1769D6F1 + regionProgress, e);

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/externalservice/ExternalServiceInfo.java

@@ -285,7 +285,7 @@ public void processLoadSnapshot(File snapshotDir) throws IOException {
     File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME);
 
     if (!snapshotFile.exists()) {
-      // do nothing if the snapshot file is not existed
+      clear();
       return;
     }
 

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigNodeRuntimeAgent.java

@@ -24,6 +24,7 @@
 import org.apache.iotdb.commons.log.LoggerPeriodicalLogReducer;
 import org.apache.iotdb.commons.pipe.agent.runtime.PipePeriodicalJobExecutor;
 import org.apache.iotdb.commons.pipe.agent.runtime.PipePeriodicalPhantomReferenceCleaner;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
 import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
 import org.apache.iotdb.commons.pipe.config.PipeConfig;
 import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
@@ -127,6 +128,11 @@ public void decreaseListenerReference(final PipeParameters parameters)
     regionListener.decreaseReference(parameters);
   }
 
+  public void reconcileListenerReferences(final Iterable<PipeMeta> pipeMetas)
+      throws IllegalPathException {
+    regionListener.reconcileReferences(pipeMetas);
+  }
+
   /** Notify the region listener that the leader is ready to allow pipe operations. */
   public void notifyLeaderReady() {
     regionListener.notifyLeaderReady();

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigRegionListener.java

@@ -20,6 +20,7 @@
 package org.apache.iotdb.confignode.manager.pipe.agent.runtime;
 
 import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
 import org.apache.iotdb.confignode.manager.pipe.source.ConfigRegionListeningFilter;
 import org.apache.iotdb.confignode.manager.pipe.source.ConfigRegionListeningQueue;
 import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
@@ -57,6 +58,24 @@ public synchronized void decreaseReference(final PipeParameters parameters)
     }
   }
 
+  public synchronized void reconcileReferences(final Iterable<PipeMeta> pipeMetas)
+      throws IllegalPathException {
+    int referenceCount = 0;
+    for (final PipeMeta pipeMeta : pipeMetas) {
+      if (!ConfigRegionListeningFilter.parseListeningPlanTypeSet(
+              pipeMeta.getStaticMeta().getSourceParameters())
+          .isEmpty()) {
+        referenceCount++;
+      }
+    }
+    listeningQueueReferenceCount = referenceCount;
+    if (referenceCount == 0) {
+      listeningQueue.close();
+    } else {
+      listeningQueue.open();
+    }
+  }
+
   public synchronized boolean isLeaderReady() {
     return isLeaderReady.get();
   }

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java

@@ -23,6 +23,7 @@
 import org.apache.iotdb.common.rpc.thrift.TSStatus;
 import org.apache.iotdb.common.rpc.thrift.TSchemaNode;
 import org.apache.iotdb.commons.auth.AuthException;
+import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.schema.node.MNodeType;
 import org.apache.iotdb.commons.schema.ttl.TTLCache;
@@ -804,10 +805,17 @@ public boolean loadSnapshot(final File latestSnapshotRootDir) {
               }
             });
     if (result.get()) {
-      pipeInfo.getPipeTaskInfo().enrichPipeMetasWithRootUserForCompatibility();
-      LOGGER.info(
-          ConfigNodeMessages.CONFIGNODESNAPSHOT_LOAD_SNAPSHOT_SUCCESS_LATESTSNAPSHOTROOTDIR,
-          latestSnapshotRootDir);
+      try {
+        PipeConfigNodeAgent.runtime()
+            .reconcileListenerReferences(pipeInfo.getPipeTaskInfo().getPipeMetaList());
+        pipeInfo.getPipeTaskInfo().enrichPipeMetasWithRootUserForCompatibility();
+        LOGGER.info(
+            ConfigNodeMessages.CONFIGNODESNAPSHOT_LOAD_SNAPSHOT_SUCCESS_LATESTSNAPSHOTROOTDIR,
+            latestSnapshotRootDir);
+      } catch (final IllegalPathException e) {
+        result.set(false);
+        LOGGER.error(ConfigNodeMessages.LOAD_SNAPSHOT_ERROR, e);
+      }
     }
     // Propagate any snapshot-load failure so callers (e.g. the AddPeer flow) do not treat a
     // partially or wholly failed load as success.

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java

@@ -311,11 +311,6 @@ public void processLoadSnapshot(final File snapshotDir) throws IOException {
 
     try {
       pipeTaskInfo.processLoadSnapshot(snapshotDir);
-
-      for (final PipeMeta pipeMeta : pipeTaskInfo.getPipeMetaList()) {
-        PipeConfigNodeAgent.runtime()
-            .increaseListenerReference(pipeMeta.getStaticMeta().getSourceParameters());
-      }
     } catch (final Exception ex) {
       LOGGER.error(ConfigNodeMessages.FAILED_TO_LOAD_PIPE_TASK_INFO_FROM_SNAPSHOT, ex);
       loadPipeTaskInfoException = ex;

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/quota/QuotaInfo.java

@@ -259,6 +259,7 @@ public Map<String, TThrottleQuota> getThrottleQuotaLimit() {
 
   public void clear() {
     spaceQuotaLimit.clear();
+    spaceQuotaUsage.clear();
     throttleQuotaLimit.clear();
   }
 }

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplatePreSetTable.java

@@ -149,6 +149,8 @@ public void processLoadSnapshot(File snapshotDir) throws IOException {
     try {
       File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME);
       if (!snapshotFile.exists()) {
+        // Empty preset tables are represented by the absence of a snapshot file.
+        templatePreSetMap.clear();
         return;
       }
 

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplateTable.java

@@ -262,16 +262,22 @@ public void processLoadSnapshot(File snapshotDir) throws IOException {
         BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) {
       // Load snapshot of template
       this.templateMap.clear();
+      this.templateIdMap.clear();
       deserialize(bufferedInputStream);
-      bufferedInputStream.close();
-      fileInputStream.close();
     } finally {
       templateReadWriteLock.writeLock().unlock();
     }
   }
 
   @TestOnly
   public void clear() {
-    this.templateMap.clear();
+    templateReadWriteLock.writeLock().lock();
+    try {
+      this.templateMap.clear();
+      this.templateIdMap.clear();
+      this.templateIdGenerator.set(0);
+    } finally {
+      templateReadWriteLock.writeLock().unlock();
+    }
   }
 }

iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java

@@ -864,9 +864,19 @@ public boolean isTopicSubscribedByConsumerGroup(
   public TSStatus alterConsumerGroup(AlterConsumerGroupPlan plan) {
     acquireWriteLock();
     try {
-      ConsumerGroupMeta consumerGroupMeta = plan.getConsumerGroupMeta();
+      final ConsumerGroupMeta consumerGroupMeta = plan.getConsumerGroupMeta();
       if (Objects.nonNull(consumerGroupMeta)) {
-        String consumerGroupId = consumerGroupMeta.getConsumerGroupId();
+        final String consumerGroupId = consumerGroupMeta.getConsumerGroupId();
+        final ConsumerGroupMeta currentConsumerGroupMeta =
+            consumerGroupMetaKeeper.containsConsumerGroupMeta(consumerGroupId)
+                ? consumerGroupMetaKeeper.getConsumerGroupMeta(consumerGroupId)
+                : null;
+        if (Objects.nonNull(currentConsumerGroupMeta)) {
+          ConsumerGroupMeta.getTopicsUnsubByGroup(currentConsumerGroupMeta, consumerGroupMeta)
+              .forEach(
+                  topicName ->
+                      commitProgressKeeper.removeTopicProgress(consumerGroupId, topicName));
+        }
         consumerGroupMetaKeeper.removeConsumerGroupMeta(consumerGroupId);
         if (!consumerGroupMeta.isEmpty()) {
           consumerGroupMetaKeeper.addConsumerGroupMeta(consumerGroupId, consumerGroupMeta);