add some features

1. update cos sdk
2. support  migrate csp
3. migrateUrl support skip head
4. migrateLocal support skip empty file and skip suffix

Author: kitmanzheng

conf/config.ini

@@ -60,7 +60,10 @@ exeludes=
 # 默认不设置, 表示不根据lastmodified时间进行筛选
 # 适用于客户在更新文件的同时又在运行迁移工具, 不准备把正在更新的文件迁移上传到COS, 比如设置为300, 表示只上传更新了5分钟以上的文件
 ignoreModifiedTimeLessThanSeconds=
-
+#多个后缀用;隔开，例如:.txt;.tmp;
+ignoreSuffix=
+#on：忽略空文件，off：不忽略。默认不忽略
+ignoreEmptyFile=
 
 ## 从阿里迁移到COS的配置分节
 [migrateAli]
@@ -103,6 +106,9 @@ needSign=true
 # 对于linux绝对路径, 如/a/b/c, 对于windows绝对路径，注意分隔符为两个反斜杠，如E:\\a\\b\\c
 urllistPath=/data/mydata/url
 
+#on：迁移时不去head url，off:会head url，默认off。不去head可能导致源更新了，迁移工具却不知道，适用于确定源不更新，加快迁移速度。
+isSkipHead=
+
 ## 从AWS迁移到COS的配置分节
 [migrateAws]
 # aws的bucket
@@ -135,3 +141,13 @@ srcSecretId=xxxxxxxxxxx
 srcSecretKey=yyyyyyyyyyyyyyyy
 # 要迁移的cos路径的 
 srcCosPath=/
+
+[migrateCsp]
+bucket=xxx
+accessKeyId=xxx
+accessKeySecret=xxx
+endPoint=xxx.com
+
+#指定了uriList之后，就只迁uri列表中的数据，不指定则是自动list全部数据然后迁移。uri格式不带域名，如：/dir1/subdir2/a.txt
+uriList=/data/uriList.txt
+prefix=

dep/cos_migrate_tool-1.0-jar-with-dependencies.jar

@@ -1,5 +1,11 @@
 package com.qcloud.cos_migrate_tool.app;
 
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -8,14 +14,17 @@
 import com.qcloud.cos_migrate_tool.config.CopyBucketConfig;
 import com.qcloud.cos_migrate_tool.config.CopyFromAliConfig;
 import com.qcloud.cos_migrate_tool.config.CopyFromAwsConfig;
+import com.qcloud.cos_migrate_tool.config.CopyFromCompetitorConfig;
 import com.qcloud.cos_migrate_tool.config.CopyFromLocalConfig;
 import com.qcloud.cos_migrate_tool.config.CopyFromQiniuConfig;
 import com.qcloud.cos_migrate_tool.config.CopyFromUrllistConfig;
+import com.qcloud.cos_migrate_tool.config.CopyFromCspConfig;
 import com.qcloud.cos_migrate_tool.config.MigrateType;
 import com.qcloud.cos_migrate_tool.meta.TaskStatics;
 import com.qcloud.cos_migrate_tool.task.MigrateAliTaskExecutor;
 import com.qcloud.cos_migrate_tool.task.MigrateAwsTaskExecutor;
 import com.qcloud.cos_migrate_tool.task.MigrateCopyBucketTaskExecutor;
+import com.qcloud.cos_migrate_tool.task.MigrateCspTaskExecutor;
 import com.qcloud.cos_migrate_tool.task.MigrateLocalTaskExecutor;
 import com.qcloud.cos_migrate_tool.task.MigrateQiniuTaskExecutor;
 import com.qcloud.cos_migrate_tool.task.MigrateUrllistTaskExecutor;
@@ -40,6 +49,8 @@ private static TaskExecutor buildTaskExecutor(CommonConfig config) {
             return new MigrateUrllistTaskExecutor((CopyFromUrllistConfig) config);
         } else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_QINIU)) {
             return new MigrateQiniuTaskExecutor((CopyFromQiniuConfig) config);
+        } else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_CSP)) {
+            return new MigrateCspTaskExecutor((CopyFromCspConfig) config);
         } else {
             System.out.println("unknown migrate type");
         }
@@ -48,26 +59,80 @@ private static TaskExecutor buildTaskExecutor(CommonConfig config) {
     }
 
     public static void main(String[] args) {
+
         if (!ConfigParser.instance.parse()) {
             return;
         }
 
         CommonConfig config = ConfigParser.instance.getConfig();
-        while (true) {
-            TaskStatics.instance.reset();
-            
-            TaskExecutor taskExecutor = buildTaskExecutor(config);
-            taskExecutor.run();
-            taskExecutor.waitTaskOver();
 
-            if (!config.isDamonMode())
-                break;
+        String batchTaskPath = config.getBatchTaskPath();
+        if (!batchTaskPath.isEmpty()) {
+
+            FileInputStream inputStream = null;
+            try {
+                inputStream = new FileInputStream(batchTaskPath);
+            } catch (FileNotFoundException e) {
+                e.printStackTrace();
+                return;
+            }
+            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
 
+            String str = null;
             try {
-                Thread.sleep(config.getDamonInterVal() * 1000);
-            } catch (InterruptedException e) {
-                log.error("the program is interrupted!", e);
-                break;
+                while (true) {
+                    while ((str = bufferedReader.readLine()) != null) {
+                        String[] task = str.split("\t");
+                        if (!((CopyFromCompetitorConfig) config).setTask(task)) {
+                            System.out.printf("task:%s invalid\n", str);
+                            log.error("task:{} invalid", str);
+                            continue;
+                        }
+
+                        TaskStatics.instance.reset();
+
+                        TaskExecutor taskExecutor = buildTaskExecutor(config);
+                        taskExecutor.run();
+                        taskExecutor.waitTaskOver();
+                    }
+                    
+                    if (!config.isDamonMode())
+                        break;
+
+                    try {
+                        Thread.sleep(config.getDamonInterVal() * 1000);
+                    } catch (InterruptedException e) {
+                        log.error("the program is interrupted!", e);
+                        break;
+                    }
+                }
+
+                inputStream.close();
+                bufferedReader.close();
+            } catch (IOException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+                return;
+            }
+
+        } else {
+
+            while (true) {
+                TaskStatics.instance.reset();
+
+                TaskExecutor taskExecutor = buildTaskExecutor(config);
+                taskExecutor.run();
+                taskExecutor.waitTaskOver();
+
+                if (!config.isDamonMode())
+                    break;
+
+                try {
+                    Thread.sleep(config.getDamonInterVal() * 1000);
+                } catch (InterruptedException e) {
+                    log.error("the program is interrupted!", e);
+                    break;
+                }
             }
         }
     }

src/main/java/com/qcloud/cos_migrate_tool/app/App.java

@@ -20,7 +20,7 @@ public class CommonConfig {
     private String cosPath;
     private boolean enableHttps;
     private boolean entireFileMd5Attached;
-    private int taskExecutorNumber = 64;
+    private int taskExecutorNumber = 68;
     private StorageClass storageClass = StorageClass.Standard;
     private int smallFileExecutorNumber = 64;
     private int bigFileExecutorNum = 4;
@@ -33,6 +33,30 @@ public class CommonConfig {
     private String cosProxyHost = "";
     private int cosProxyPort = -1;
     private String encryptionType = "";
+    private String batchTaskPath = "";    
+    private boolean realTimeCompare = false;
+    
+    public void setRealTimeCompare(String realTimeCompare)  throws IllegalArgumentException {
+        if (realTimeCompare.equalsIgnoreCase("on")) {
+            this.realTimeCompare = true;
+        } else if (realTimeCompare.equalsIgnoreCase("off")) {
+            this.realTimeCompare = false;
+        } else {
+            throw new IllegalArgumentException("invalid realTimeCompare config. only support on/off");
+        }
+    }
+    
+    public boolean getRealTimeCompare() {
+        return this.realTimeCompare;
+    }
+
+    public void setBatchTaskPath(String batchTaskPath) {
+        this.batchTaskPath = batchTaskPath;
+    }
+    
+    public String getBatchTaskPath() {
+        return this.batchTaskPath;  
+    }
 
     public void setEncryptionType(String encryptionType) {
         if (!encryptionType.equals("sse-cos")) {
@@ -71,11 +95,6 @@ public String getBucketName() {
 
     public void setBucketName(String bucketName) {
         bucketName = bucketName.trim();
-        String parrtern = ".*-(125|100|20)[0-9]{3,}$";
-        if (!Pattern.matches(parrtern, bucketName)) {
-            throw new IllegalArgumentException(
-                    "bucketName must contain appid. example: test-1250001000");
-        }
         this.bucketName = bucketName;
     }