CASS-2050 delete empty incremental backup files rather than throwing …

…when we try to rate limit them. (#968)
Netflix · Aug 30, 2021 · 6e127b3 · 6e127b3
1 parent 9db49fd
commit 6e127b3
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 3 deletions.
diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java
@@ -29,16 +29,16 @@
 import com.netflix.priam.scheduler.Task;
 import com.netflix.priam.utils.SystemUtils;
 import java.io.File;
+import java.io.FileFilter;
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.Future;
 import java.util.stream.Stream;
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -227,4 +227,11 @@ public static Set<Path> getBackupDirectories(IConfiguration config, String monit
     protected static boolean isAReadableDirectory(File dir) {
         return dir.exists() && dir.isDirectory() && dir.canRead();
     }
+
+    protected static void deleteEmptyFiles(File dir) {
+        FileFilter filter = (file) -> file.isFile() && file.canWrite() && file.length() == 0L;
+        for (File file : Optional.ofNullable(dir.listFiles(filter)).orElse(new File[] {})) {
+            FileUtils.deleteQuietly(file);
+        }
+    }
 }
diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java
@@ -113,6 +113,7 @@ public String getName() {
     protected void processColumnFamily(File backupDir) throws Exception {
         BackupFileType fileType =
                 backupRestoreConfig.enableV2Backups() ? BackupFileType.SST_V2 : BackupFileType.SST;
+        deleteEmptyFiles(backupDir);
         upload(backupDir, fileType, config.enableAsyncIncremental(), true);
     }
 }
diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java
@@ -113,6 +113,23 @@ public void testClusterSpecificColumnFamiliesSkippedFrom21() throws Exception {
         testClusterSpecificColumnFamiliesSkipped(columnFamilyDirs);
     }
 
+    @Test
+    public void testSkippingEmptyFiles() throws Exception {
+        filesystem.cleanup();
+        File tmp = new File("target/data/");
+        if (tmp.exists()) cleanup(tmp);
+        File emptyFile =
+                new File(
+                        "target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-1-Data.db");
+        File parent = emptyFile.getParentFile();
+        if (!parent.exists()) parent.mkdirs();
+        Assert.assertTrue(emptyFile.createNewFile());
+        IncrementalBackup backup = injector.getInstance(IncrementalBackup.class);
+        backup.execute();
+        Assert.assertTrue(filesystem.uploadedFiles.isEmpty());
+        Assert.assertFalse(emptyFile.exists());
+    }
+
     private void testClusterSpecificColumnFamiliesSkipped(String[] columnFamilyDirs)
             throws Exception {
         filesystem.cleanup();