() {
@Override
public TimeZone initialValue() {
@@ -91,8 +109,9 @@ public static String timestampToString(long us) {
String timestampString = ts.toString();
String formatted = THREAD_LOCAL_TIMESTAMP_FORMAT.get().format(ts);
- if(timestampString.length() > 19 && !".0".equals(timestampString.substring(19))) {
- formatted += timestampString.substring(19);
+ if(timestampString.length() > TIMESTAMP_STRING_LENGTH
+ && !TIMESTAMP_ZERO_SUFFIX.equals(timestampString.substring(TIMESTAMP_STRING_LENGTH))) {
+ formatted += timestampString.substring(TIMESTAMP_STRING_LENGTH);
}
return formatted;
}
@@ -231,33 +250,33 @@ public static int getMonth(int date) {
int year = tuple2.getField(0);
int dayInYear = tuple2.getField(1);
if (isLeapYear(year)) {
- if (dayInYear == 60) {
+ if (dayInYear == DAY_IN_YEAR_60) {
return 2;
- } else if (dayInYear > 60) {
+ } else if (dayInYear > DAY_IN_YEAR_60) {
dayInYear = dayInYear - 1;
}
}
- if (dayInYear <= 31) {
+ if (dayInYear <= DAY_IN_YEAR_31) {
return 1;
- } else if (dayInYear <= 59) {
+ } else if (dayInYear <= DAY_IN_YEAR_59) {
return 2;
- } else if (dayInYear <= 90) {
+ } else if (dayInYear <= DAY_IN_YEAR_90) {
return 3;
- } else if (dayInYear <= 120) {
+ } else if (dayInYear <= DAY_IN_YEAR_120) {
return 4;
- } else if (dayInYear <= 151) {
+ } else if (dayInYear <= DAY_IN_YEAR_151) {
return 5;
- } else if (dayInYear <= 181) {
+ } else if (dayInYear <= DAY_IN_YEAR_181) {
return 6;
- } else if (dayInYear <= 212) {
+ } else if (dayInYear <= DAY_IN_YEAR_212) {
return 7;
- } else if (dayInYear <= 243) {
+ } else if (dayInYear <= DAY_IN_YEAR_243) {
return 8;
- } else if (dayInYear <= 273) {
+ } else if (dayInYear <= DAY_IN_YEAR_273) {
return 9;
- } else if (dayInYear <= 304) {
+ } else if (dayInYear <= DAY_IN_YEAR_304) {
return 10;
- } else if (dayInYear <= 334) {
+ } else if (dayInYear <= DAY_IN_YEAR_334) {
return 11;
} else {
return 12;
@@ -274,39 +293,38 @@ public static int getDayOfMonth(int date) {
int year = tuple2.getField(0);
int dayInYear = tuple2.getField(1);
if (isLeapYear(year)) {
- if (dayInYear == 60) {
- return 29;
- } else if (dayInYear > 60) {
+ if (dayInYear == DAY_IN_YEAR_60) {
+ return DAY_IN_YEAR_29;
+ } else if (dayInYear > DAY_IN_YEAR_60) {
dayInYear = dayInYear - 1;
}
}
- if (dayInYear <= 31) {
+ if (dayInYear <= DAY_IN_YEAR_31) {
return dayInYear;
- } else if (dayInYear <= 59) {
- return dayInYear - 31;
- } else if (dayInYear <= 90) {
- return dayInYear - 59;
- } else if (dayInYear <= 120) {
- return dayInYear - 90;
- } else if (dayInYear <= 151) {
- return dayInYear - 120;
- } else if (dayInYear <= 181) {
- return dayInYear - 151;
- } else if (dayInYear <= 212) {
- return dayInYear - 181;
- } else if (dayInYear <= 243) {
- return dayInYear - 212;
- } else if (dayInYear <= 273) {
- return dayInYear - 243;
- } else if (dayInYear <= 304) {
- return dayInYear - 273;
- } else if (dayInYear <= 334) {
- return dayInYear - 304;
+ } else if (dayInYear <= DAY_IN_YEAR_59) {
+ return dayInYear - DAY_IN_YEAR_31;
+ } else if (dayInYear <= DAY_IN_YEAR_90) {
+ return dayInYear - DAY_IN_YEAR_59;
+ } else if (dayInYear <= DAY_IN_YEAR_120) {
+ return dayInYear - DAY_IN_YEAR_90;
+ } else if (dayInYear <= DAY_IN_YEAR_151) {
+ return dayInYear - DAY_IN_YEAR_120;
+ } else if (dayInYear <= DAY_IN_YEAR_181) {
+ return dayInYear - DAY_IN_YEAR_151;
+ } else if (dayInYear <= DAY_IN_YEAR_212) {
+ return dayInYear - DAY_IN_YEAR_181;
+ } else if (dayInYear <= DAY_IN_YEAR_243) {
+ return dayInYear - DAY_IN_YEAR_212;
+ } else if (dayInYear <= DAY_IN_YEAR_273) {
+ return dayInYear - DAY_IN_YEAR_243;
+ } else if (dayInYear <= DAY_IN_YEAR_304) {
+ return dayInYear - DAY_IN_YEAR_273;
+ } else if (dayInYear <= DAY_IN_YEAR_334) {
+ return dayInYear - DAY_IN_YEAR_304;
} else {
- return dayInYear - 334;
+ return dayInYear - DAY_IN_YEAR_334;
}
-
}
diff --git a/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/dict/ExternalCatalogUtils.java b/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/dict/ExternalCatalogUtils.java
index 1b83ff80fe..dbff692bbb 100644
--- a/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/dict/ExternalCatalogUtils.java
+++ b/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/dict/ExternalCatalogUtils.java
@@ -68,11 +68,4 @@ public static String escapePathName(String path) {
private static boolean needsEscaping(char c) {
return c >= 0 && c < charToEscape.size() && charToEscape.get(c);
}
-
- public static void main(String[] args) {
- String s = "2018-01-01 08:08:08";
- String d = escapePathName(s);
- System.out.println(d);
- }
-
}
diff --git a/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/AbstractRecordWriter.java b/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/AbstractRecordWriter.java
index 188601b450..63e01e580c 100644
--- a/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/AbstractRecordWriter.java
+++ b/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/AbstractRecordWriter.java
@@ -39,11 +39,22 @@
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import java.io.IOException;
-import java.util.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Random;
+import java.util.UUID;
+import java.util.Map;
+import java.util.Collections;
+import java.util.HashMap;
/**
* Abstract record writer wrapper
diff --git a/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/CarbonPartitionRecordWriter.java b/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/CarbonPartitionRecordWriter.java
index 3a3236ca76..b2a6b20d30 100644
--- a/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/CarbonPartitionRecordWriter.java
+++ b/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/CarbonPartitionRecordWriter.java
@@ -21,10 +21,7 @@
import com.dtstack.flinkx.carbondata.writer.dict.CarbonTypeConverter;
-import com.dtstack.flinkx.util.DateUtil;
-import com.dtstack.flinkx.util.StringUtil;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
diff --git a/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/HivePartitionRecordWriter.java b/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/HivePartitionRecordWriter.java
index 38dc8e42e8..a74982de1b 100644
--- a/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/HivePartitionRecordWriter.java
+++ b/flinkx-carbondata/flinkx-carbondata-writer/src/main/java/com/dtstack/flinkx/carbondata/writer/recordwriter/HivePartitionRecordWriter.java
@@ -34,11 +34,21 @@
import org.apache.carbondata.processing.util.CarbonLoaderUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import java.io.IOException;
-import java.util.*;
+import java.util.List;
+import java.util.Map;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Random;
+import java.util.UUID;
import java.util.stream.Collectors;
diff --git a/flinkx-emqx/flinkx-emqx-core/src/main/java/com/dtstack/flinkx/emqx/decoder/IDecode.java b/flinkx-carbondata/flinkx-carbondata-writer/src/test/java/com/dtstack/flinkx/carbondata/writer/dict/DateTimeUtilsTest.java
similarity index 73%
rename from flinkx-emqx/flinkx-emqx-core/src/main/java/com/dtstack/flinkx/emqx/decoder/IDecode.java
rename to flinkx-carbondata/flinkx-carbondata-writer/src/test/java/com/dtstack/flinkx/carbondata/writer/dict/DateTimeUtilsTest.java
index ce63e6fd15..11a8f7136e 100644
--- a/flinkx-emqx/flinkx-emqx-core/src/main/java/com/dtstack/flinkx/emqx/decoder/IDecode.java
+++ b/flinkx-carbondata/flinkx-carbondata-writer/src/test/java/com/dtstack/flinkx/carbondata/writer/dict/DateTimeUtilsTest.java
@@ -6,32 +6,29 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.dtstack.flinkx.emqx.decoder;
-import java.util.Map;
+
+package com.dtstack.flinkx.carbondata.writer.dict;
+
+import org.junit.Test;
/**
- * Date: 2020/02/12
- * Company: www.dtstack.com
- *
- * @author tudou
+ * @author jiangbo
+ * @date 2020/3/27
*/
-public interface IDecode {
-
- /**
- * 消息解码
- * @param message
- * @return
- */
- Map decode(String message);
+public class DateTimeUtilsTest {
+ @Test
+ public void testGetMonth() {
+ DateTimeUtils.getMonth(10);
+ }
}
diff --git a/flinkx-cassandra/flinkx-cassandra-core/pom.xml b/flinkx-cassandra/flinkx-cassandra-core/pom.xml
index 6dc4304d67..86b7f12abf 100644
--- a/flinkx-cassandra/flinkx-cassandra-core/pom.xml
+++ b/flinkx-cassandra/flinkx-cassandra-core/pom.xml
@@ -1,6 +1,6 @@
-
flinkx-cassandra
diff --git a/flinkx-cassandra/flinkx-cassandra-core/src/main/java/com/dtstack/flinkx/cassandra/CassandraConfigKeys.java b/flinkx-cassandra/flinkx-cassandra-core/src/main/java/com/dtstack/flinkx/cassandra/CassandraConfigKeys.java
index 072a2b0096..a5a67df83d 100644
--- a/flinkx-cassandra/flinkx-cassandra-core/src/main/java/com/dtstack/flinkx/cassandra/CassandraConfigKeys.java
+++ b/flinkx-cassandra/flinkx-cassandra-core/src/main/java/com/dtstack/flinkx/cassandra/CassandraConfigKeys.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.cassandra;
/**
diff --git a/flinkx-cassandra/flinkx-cassandra-core/src/main/java/com/dtstack/flinkx/cassandra/CassandraUtil.java b/flinkx-cassandra/flinkx-cassandra-core/src/main/java/com/dtstack/flinkx/cassandra/CassandraUtil.java
index fa89243336..bc0e631faa 100644
--- a/flinkx-cassandra/flinkx-cassandra-core/src/main/java/com/dtstack/flinkx/cassandra/CassandraUtil.java
+++ b/flinkx-cassandra/flinkx-cassandra-core/src/main/java/com/dtstack/flinkx/cassandra/CassandraUtil.java
@@ -32,7 +32,6 @@
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.InetAddress;
-import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.sql.Time;
import java.util.Date;
@@ -222,7 +221,7 @@ private static Optional objectToBytes(T obj){
* @param pos 位置
* @param sqlType cql类型
* @param value 值
- * @throws RuntimeException 对于不支持的数据类型，抛出异常
+ * @throws Exception 对于不支持的数据类型，抛出异常
*/
public static void bindColumn(BoundStatement ps, int pos, DataType sqlType, Object value) throws Exception {
if (value != null) {
diff --git a/flinkx-cassandra/flinkx-cassandra-core/src/test/java/com/dtstack/flinkx/cassandra/TestCassandraUtil.java b/flinkx-cassandra/flinkx-cassandra-core/src/test/java/com/dtstack/flinkx/cassandra/TestCassandraUtil.java
index 179e420193..6d5dde5398 100644
--- a/flinkx-cassandra/flinkx-cassandra-core/src/test/java/com/dtstack/flinkx/cassandra/TestCassandraUtil.java
+++ b/flinkx-cassandra/flinkx-cassandra-core/src/test/java/com/dtstack/flinkx/cassandra/TestCassandraUtil.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.cassandra;
import com.datastax.driver.core.*;
diff --git a/flinkx-cassandra/flinkx-cassandra-reader/pom.xml b/flinkx-cassandra/flinkx-cassandra-reader/pom.xml
index feca24f0f9..804af7c5d6 100644
--- a/flinkx-cassandra/flinkx-cassandra-reader/pom.xml
+++ b/flinkx-cassandra/flinkx-cassandra-reader/pom.xml
@@ -1,6 +1,6 @@
-
flinkx-cassandra
diff --git a/flinkx-cassandra/flinkx-cassandra-reader/src/main/java/com/dtstack/flinkx/cassandra/reader/CassandraConstants.java b/flinkx-cassandra/flinkx-cassandra-reader/src/main/java/com/dtstack/flinkx/cassandra/reader/CassandraConstants.java
index cd8da1f553..df6dd94a03 100644
--- a/flinkx-cassandra/flinkx-cassandra-reader/src/main/java/com/dtstack/flinkx/cassandra/reader/CassandraConstants.java
+++ b/flinkx-cassandra/flinkx-cassandra-reader/src/main/java/com/dtstack/flinkx/cassandra/reader/CassandraConstants.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.cassandra.reader;
/**
diff --git a/flinkx-cassandra/flinkx-cassandra-reader/src/main/java/com/dtstack/flinkx/cassandra/reader/CassandraInputFormat.java b/flinkx-cassandra/flinkx-cassandra-reader/src/main/java/com/dtstack/flinkx/cassandra/reader/CassandraInputFormat.java
index f5fdbe4ae2..52f1311861 100644
--- a/flinkx-cassandra/flinkx-cassandra-reader/src/main/java/com/dtstack/flinkx/cassandra/reader/CassandraInputFormat.java
+++ b/flinkx-cassandra/flinkx-cassandra-reader/src/main/java/com/dtstack/flinkx/cassandra/reader/CassandraInputFormat.java
@@ -20,7 +20,6 @@
import com.datastax.driver.core.*;
import com.dtstack.flinkx.cassandra.CassandraUtil;
-import com.dtstack.flinkx.constants.ConstantValue;
import com.dtstack.flinkx.inputformat.BaseRichInputFormat;
import com.dtstack.flinkx.reader.MetaColumn;
import com.google.common.base.Preconditions;
@@ -97,7 +96,6 @@ public Row nextRecordInternal(Row row) {
Object value = CassandraUtil.getData(cqlRow, definitions.get(i).getType(), definitions.get(i).getName());
row.setField(i, value);
}
- LOG.info(row.toString());
return row;
}
@@ -108,7 +106,7 @@ protected void closeInternal() {
}
@Override
- public InputSplit[] createInputSplitsInternal(int minNumSplits) {
+ protected InputSplit[] createInputSplitsInternal(int minNumSplits) {
ArrayList splits = new ArrayList<>();
try {
@@ -130,12 +128,12 @@ public InputSplit[] createInputSplitsInternal(int minNumSplits) {
private InputSplit[] splitJob(int minNumSplits, ArrayList splits) {
if(minNumSplits <= 1) {
splits.add(new CassandraInputSplit());
- return splits.toArray(new CassandraInputSplit[splits.size()]);
+ return splits.toArray(new CassandraInputSplit[0]);
}
if(whereString != null && whereString.toLowerCase().contains(CassandraConstants.TOKEN)) {
splits.add(new CassandraInputSplit());
- return splits.toArray(new CassandraInputSplit[splits.size()]);
+ return splits.toArray(new CassandraInputSplit[0]);
}
Session session = CassandraUtil.getSession(cassandraConfig, "");
String partitioner = session.getCluster().getMetadata().getPartitioner();
@@ -152,8 +150,7 @@ private InputSplit[] splitJob(int minNumSplits, ArrayList s
}
splits.add(new CassandraInputSplit(l.toString(), r.toString()));
}
- }
- else if(partitioner.endsWith(CassandraConstants.MURMUR3_PARTITIONER)) {
+ }else if(partitioner.endsWith(CassandraConstants.MURMUR3_PARTITIONER)) {
BigDecimal minToken = BigDecimal.valueOf(Long.MIN_VALUE);
BigDecimal maxToken = BigDecimal.valueOf(Long.MAX_VALUE);
BigDecimal step = maxToken.subtract(minToken)
@@ -166,11 +163,10 @@ else if(partitioner.endsWith(CassandraConstants.MURMUR3_PARTITIONER)) {
}
splits.add(new CassandraInputSplit(String.valueOf(l), String.valueOf(r)));
}
- }
- else {
+ }else {
splits.add(new CassandraInputSplit());
}
- return splits.toArray(new CassandraInputSplit[splits.size()]);
+ return splits.toArray(new CassandraInputSplit[0]);
}
/**
@@ -181,7 +177,7 @@ else if(partitioner.endsWith(CassandraConstants.MURMUR3_PARTITIONER)) {
private String getQueryString(CassandraInputSplit inputSplit) {
StringBuilder columns = new StringBuilder();
if (columnMeta == null) {
- columns.append(ConstantValue.STAR_SYMBOL);
+ columns.append("*");
} else {
for(MetaColumn column : columnMeta) {
if(columns.length() > 0 ) {
diff --git a/flinkx-cassandra/flinkx-cassandra-writer/pom.xml b/flinkx-cassandra/flinkx-cassandra-writer/pom.xml
index 565ef3ca4f..73593ae1c9 100644
--- a/flinkx-cassandra/flinkx-cassandra-writer/pom.xml
+++ b/flinkx-cassandra/flinkx-cassandra-writer/pom.xml
@@ -1,6 +1,6 @@
-
flinkx-cassandra
diff --git a/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraOutputFormat.java b/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraOutputFormat.java
index f16eca03ab..a280dbf059 100644
--- a/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraOutputFormat.java
+++ b/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraOutputFormat.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.cassandra.writer;
import com.datastax.driver.core.*;
@@ -102,7 +119,6 @@ protected void writeSingleRecordInternal(Row row) throws WriteRecordException {
throw new WriteRecordException("类型转换失败", e.getCause(), i, row);
}
}
- LOG.info("insertSql: {}" + boundStatement);
session.execute(boundStatement);
}
@@ -110,12 +126,14 @@ protected void writeSingleRecordInternal(Row row) throws WriteRecordException {
protected void writeMultipleRecordsInternal() throws Exception {
if (batchSize > 1) {
BoundStatement boundStatement = pstmt.bind();
- for (Row row : rows) {
- for (int i = 0; i < columnMeta.size(); i++) {
- Object value = row.getField(i);
- CassandraUtil.bindColumn(boundStatement, i, columnTypes.get(i), value);
+ for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
+ for (int columnIndex = 0; columnIndex < columnMeta.size(); columnIndex++) {
+ Object value = rows.get(rowIndex).getField(columnIndex);
+ CassandraUtil.bindColumn(boundStatement, columnIndex, columnTypes.get(columnIndex), value);
+ }
+ if ((rowIndex % 1000) == 0) {
+ LOG.info("insertSql: {}", boundStatement);
}
- LOG.info("insertSql: {}" + boundStatement);
if(asyncWrite) {
unConfirmedWrite.add(session.executeAsync(boundStatement));
if (unConfirmedWrite.size() >= batchSize) {
@@ -152,7 +170,7 @@ protected void writeMultipleRecordsInternal() throws Exception {
}
@Override
- public void closeInternal() throws IOException {
+ public void closeInternal() {
CassandraUtil.close(session);
}
}
diff --git a/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraOutputFormatBuilder.java b/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraOutputFormatBuilder.java
index 4446f1633f..56a170811e 100644
--- a/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraOutputFormatBuilder.java
+++ b/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraOutputFormatBuilder.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.cassandra.writer;
import com.dtstack.flinkx.outputformat.BaseRichOutputFormatBuilder;
diff --git a/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraWriter.java b/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraWriter.java
index 2e2f0bbc9c..1e89082846 100644
--- a/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraWriter.java
+++ b/flinkx-cassandra/flinkx-cassandra-writer/src/main/java/com/dtstack/flinkx/cassandra/writer/CassandraWriter.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.cassandra.writer;
import com.dtstack.flinkx.config.DataTransferConfig;
@@ -13,8 +30,6 @@
import java.util.Map;
import static com.dtstack.flinkx.cassandra.CassandraConfigKeys.*;
-import static com.dtstack.flinkx.cassandra.CassandraConfigKeys.KEY_CONSITANCY_LEVEL;
-import static com.dtstack.flinkx.cassandra.CassandraConfigKeys.KEY_MAX_PENDING_CONNECTION;
/**
*
diff --git a/flinkx-cassandra/pom.xml b/flinkx-cassandra/pom.xml
index d788a4bea1..917b1dc9f8 100644
--- a/flinkx-cassandra/pom.xml
+++ b/flinkx-cassandra/pom.xml
@@ -1,6 +1,6 @@
-
flinkx-all
diff --git a/flinkx-clickhouse/flinkx-clickhouse-core/src/main/java/com/dtstack/flinkx/clickhouse/core/ClickhouseUtil.java b/flinkx-clickhouse/flinkx-clickhouse-core/src/main/java/com/dtstack/flinkx/clickhouse/core/ClickhouseUtil.java
index b33160d10d..c6103d9dfa 100644
--- a/flinkx-clickhouse/flinkx-clickhouse-core/src/main/java/com/dtstack/flinkx/clickhouse/core/ClickhouseUtil.java
+++ b/flinkx-clickhouse/flinkx-clickhouse-core/src/main/java/com/dtstack/flinkx/clickhouse/core/ClickhouseUtil.java
@@ -23,6 +23,7 @@
import java.sql.Connection;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.Properties;
/**
@@ -43,8 +44,10 @@ public static Connection getConnection(String url, String username, String passw
for (int i = 0; i < MAX_RETRY_TIMES && failed; ++i) {
try {
conn = new BalancedClickhouseDataSource(url, properties).getConnection();
- conn.createStatement().execute("select 111");
- failed = false;
+ try (Statement statement = conn.createStatement()) {
+ statement.execute("select 111");
+ failed = false;
+ }
} catch (Exception e) {
if (conn != null) {
conn.close();
diff --git a/flinkx-clickhouse/flinkx-clickhouse-reader/pom.xml b/flinkx-clickhouse/flinkx-clickhouse-reader/pom.xml
index 01d798e35d..4c5ef8cf4c 100644
--- a/flinkx-clickhouse/flinkx-clickhouse-reader/pom.xml
+++ b/flinkx-clickhouse/flinkx-clickhouse-reader/pom.xml
@@ -56,6 +56,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-clickhouse/flinkx-clickhouse-writer/pom.xml b/flinkx-clickhouse/flinkx-clickhouse-writer/pom.xml
index 94e72635a6..8c3d65ad12 100644
--- a/flinkx-clickhouse/flinkx-clickhouse-writer/pom.xml
+++ b/flinkx-clickhouse/flinkx-clickhouse-writer/pom.xml
@@ -56,6 +56,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-core/pom.xml b/flinkx-core/pom.xml
index 223aae3824..a0bd729335 100644
--- a/flinkx-core/pom.xml
+++ b/flinkx-core/pom.xml
@@ -110,33 +110,6 @@
${flink.version}
-
- junit
- junit
- 4.11
- test
-
-
-
- org.powermock
- powermock-api-mockito
- 1.7.0
- test
-
-
-
- org.powermock
- powermock-module-junit4
- 1.7.0
- test
-
-
-
- org.hamcrest
- hamcrest-all
- 1.3
- test
-
org.apache.httpcomponents
httpcore
@@ -200,6 +173,10 @@
org.slf4j:*
org.apache.httpcomponents:*
io.prometheus:*
+ org.apache.avro:*
+ org.codehaus.jackson:*
+ com.fasterxml.jackson.core:*
+ commons-*:*
@@ -207,6 +184,10 @@
com.google.common
shade.core.com.google.common
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
org.apache.http
shade.core.org.apache.http
diff --git a/flinkx-core/src/main/java/com/dtstack/flink/api/java/MyLocalStreamEnvironment.java b/flinkx-core/src/main/java/com/dtstack/flink/api/java/MyLocalStreamEnvironment.java
index 56fb01de1c..c3dcf4ca9e 100644
--- a/flinkx-core/src/main/java/com/dtstack/flink/api/java/MyLocalStreamEnvironment.java
+++ b/flinkx-core/src/main/java/com/dtstack/flink/api/java/MyLocalStreamEnvironment.java
@@ -22,7 +22,6 @@
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.RestOptions;
import org.apache.flink.configuration.TaskManagerOptions;
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/Main.java b/flinkx-core/src/main/java/com/dtstack/flinkx/Main.java
index bcac73e949..b36613bdb7 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/Main.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/Main.java
@@ -30,7 +30,7 @@
import com.dtstack.flinkx.writer.DataWriterFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.Charsets;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/authenticate/KerberosUtil.java b/flinkx-core/src/main/java/com/dtstack/flinkx/authenticate/KerberosUtil.java
index c89f0b2274..8fde360336 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/authenticate/KerberosUtil.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/authenticate/KerberosUtil.java
@@ -46,6 +46,7 @@ public class KerberosUtil {
private static final String SP = "/";
private static final String KEY_SFTP_CONF = "sftpConf";
+ private static final String KEY_PRINCIPAL = "principal";
private static final String KEY_REMOTE_DIR = "remoteDir";
private static final String KEY_USE_LOCAL_FILE = "useLocalFile";
public static final String KEY_PRINCIPAL_FILE = "principalFile";
@@ -89,6 +90,15 @@ public static UserGroupInformation loginAndReturnUgi(Configuration conf, String
return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab);
}
+ public static String getPrincipal(Map configMap, String keytabPath) {
+ String principal = MapUtils.getString(configMap, KEY_PRINCIPAL);
+ if (StringUtils.isEmpty(principal)) {
+ principal = findPrincipalFromKeytab(keytabPath);
+ }
+
+ return principal;
+ }
+
private static void reloadKrb5Conf(Configuration conf){
String krb5File = conf.get(KEY_JAVA_SECURITY_KRB5_CONF);
LOG.info("set krb5 file:{}", krb5File);
@@ -188,7 +198,7 @@ private static String loadFromSftp(Map config, String fileName){
}
}
- public static String findPrincipalFromKeytab(String keytabFile) {
+ private static String findPrincipalFromKeytab(String keytabFile) {
KeyTab keyTab = KeyTab.getInstance(keytabFile);
for (KeyTabEntry entry : keyTab.getEntries()) {
String principal = entry.getService().getName();
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/authenticate/SftpHandler.java b/flinkx-core/src/main/java/com/dtstack/flinkx/authenticate/SftpHandler.java
index 90c0e127fb..dc1a70a1b4 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/authenticate/SftpHandler.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/authenticate/SftpHandler.java
@@ -20,13 +20,18 @@
package com.dtstack.flinkx.authenticate;
import com.dtstack.flinkx.util.RetryUtil;
-import com.jcraft.jsch.*;
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.Session;
+import com.jcraft.jsch.SftpException;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
@@ -131,21 +136,11 @@ private void downloadFile(String ftpPath, String localPath){
throw new RuntimeException("File not exist on sftp:" + ftpPath);
}
- OutputStream os = null;
- try {
- os = new FileOutputStream(new File(localPath));
+ try (OutputStream os = new FileOutputStream(new File(localPath))){
channelSftp.get(ftpPath, os);
+ os.flush();
} catch (Exception e){
throw new RuntimeException("download file from sftp error", e);
- } finally {
- if(os != null){
- try {
- os.flush();
- os.close();
- } catch (IOException e) {
- LOG.warn("", e);
- }
- }
}
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/classloader/ClassLoaderManager.java b/flinkx-core/src/main/java/com/dtstack/flinkx/classloader/ClassLoaderManager.java
index 9cb8f0220a..a6276ff42a 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/classloader/ClassLoaderManager.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/classloader/ClassLoaderManager.java
@@ -25,7 +25,13 @@
import java.net.URL;
import java.net.URLClassLoader;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
/**
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/classloader/PluginUtil.java b/flinkx-core/src/main/java/com/dtstack/flinkx/classloader/PluginUtil.java
index fd51f7e65c..efee645720 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/classloader/PluginUtil.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/classloader/PluginUtil.java
@@ -24,9 +24,7 @@
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
-import java.util.ArrayList;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
/**
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/config/AbstractConfig.java b/flinkx-core/src/main/java/com/dtstack/flinkx/config/AbstractConfig.java
index df46175089..087c5302bd 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/config/AbstractConfig.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/config/AbstractConfig.java
@@ -76,9 +76,9 @@ public Object getVal(String key) {
Object obj = internalMap.get(key);
if (obj instanceof LinkedTreeMap) {
LinkedTreeMap treeMap = (LinkedTreeMap) obj;
- Map newMap = new HashMap<>(treeMap.size());
- newMap.putAll(treeMap);
- return newMap;
+ Map map = new HashMap<>(Math.max((int) (treeMap.size()/.75f) + 1, 16));
+ map.putAll(treeMap);
+ return map;
}
return obj;
}
@@ -109,10 +109,10 @@ public int getIntVal(String key, int defaultValue) {
return defaultValue;
}
if(ret instanceof Integer) {
- return ((Integer)ret).intValue();
+ return (Integer) ret;
}
if(ret instanceof String) {
- return Integer.valueOf((String)ret).intValue();
+ return Integer.parseInt((String) ret);
}
if(ret instanceof Long) {
return ((Long)ret).intValue();
@@ -144,7 +144,7 @@ public long getLongVal(String key, long defaultValue) {
return ((Integer)ret).longValue();
}
if(ret instanceof String) {
- return Long.valueOf((String)ret);
+ return Long.parseLong((String)ret);
}
if(ret instanceof Float) {
return ((Float)ret).longValue();
@@ -176,7 +176,7 @@ public double getDoubleVal(String key, double defaultValue) {
return ((Integer) ret).doubleValue();
}
if (ret instanceof String) {
- return Double.valueOf((String) ret);
+ return Double.parseDouble((String) ret);
}
if (ret instanceof Float) {
return ((Float) ret).doubleValue();
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/config/DataTransferConfig.java b/flinkx-core/src/main/java/com/dtstack/flinkx/config/DataTransferConfig.java
index 5e58d85e5a..f3e732da0b 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/config/DataTransferConfig.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/config/DataTransferConfig.java
@@ -90,7 +90,7 @@ private static void checkConfig(DataTransferConfig config) {
Preconditions.checkNotNull(readerParameter, "Must specify parameter for reader");
- // 检查我writer配置
+ // 检查writer配置
WriterConfig writerConfig = content.getWriter();
Preconditions.checkNotNull(writerConfig, "Must specify a writer element");
Preconditions.checkNotNull(writerConfig.getName(), "Must specify the writer name");
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/config/RestartConfig.java b/flinkx-core/src/main/java/com/dtstack/flinkx/config/RestartConfig.java
index 35068840ab..f83a394399 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/config/RestartConfig.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/config/RestartConfig.java
@@ -49,7 +49,7 @@ public RestartConfig(Map map) {
}
public static RestartConfig defaultConfig(){
- Map map = new HashMap<>(1);
+ Map map = new HashMap<>(16);
return new RestartConfig(map);
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/config/RestoreConfig.java b/flinkx-core/src/main/java/com/dtstack/flinkx/config/RestoreConfig.java
index c85c1bb376..ef886bf7e7 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/config/RestoreConfig.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/config/RestoreConfig.java
@@ -18,7 +18,7 @@
package com.dtstack.flinkx.config;
-import java.util.HashMap;
+import java.util.Collections;
import java.util.Map;
/**
@@ -40,8 +40,7 @@ public RestoreConfig(Map map) {
}
public static RestoreConfig defaultConfig(){
- Map map = new HashMap<>(1);
- map.put(KEY_IS_RESTORE, false);
+ Map map = Collections.singletonMap(KEY_IS_RESTORE, false);
return new RestoreConfig(map);
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/config/TestConfig.java b/flinkx-core/src/main/java/com/dtstack/flinkx/config/TestConfig.java
index 2a15601654..dbcd774858 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/config/TestConfig.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/config/TestConfig.java
@@ -37,7 +37,7 @@ public TestConfig(Map map) {
}
public static TestConfig defaultConfig(){
- Map map = new HashMap<>(1);
+ Map map = new HashMap<>(16);
return new TestConfig(map);
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/constants/ConstantValue.java b/flinkx-core/src/main/java/com/dtstack/flinkx/constants/ConstantValue.java
index 46de3749fb..d94589fbc6 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/constants/ConstantValue.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/constants/ConstantValue.java
@@ -51,4 +51,15 @@ public class ConstantValue {
public static final String SYSTEM_PROPERTIES_KEY_FILE_ENCODING = "file.encoding";
public static final String OS_WINDOWS = "windows";
+
+ public static final String TIME_SECOND_SUFFIX = "sss";
+ public static final String TIME_MILLISECOND_SUFFIX = "SSS";
+
+ public static final String FILE_SUFFIX_XML = ".xml";
+
+ public static final int MAX_BATCH_SIZE = 200000;
+
+ public static final long STORE_SIZE_G = 1024L * 1024 * 1024;
+
+ public static final long STORE_SIZE_M = 1024L * 1024;
}
diff --git a/flinkx-emqx/flinkx-emqx-core/src/main/java/com/dtstack/flinkx/emqx/decoder/PlainDecoder.java b/flinkx-core/src/main/java/com/dtstack/flinkx/decoder/DecodeEnum.java
similarity index 69%
rename from flinkx-emqx/flinkx-emqx-core/src/main/java/com/dtstack/flinkx/emqx/decoder/PlainDecoder.java
rename to flinkx-core/src/main/java/com/dtstack/flinkx/decoder/DecodeEnum.java
index 63d92b8463..ce360c6bc0 100644
--- a/flinkx-emqx/flinkx-emqx-core/src/main/java/com/dtstack/flinkx/emqx/decoder/PlainDecoder.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/decoder/DecodeEnum.java
@@ -6,31 +6,41 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.dtstack.flinkx.emqx.decoder;
-import java.util.Collections;
-import java.util.Map;
+package com.dtstack.flinkx.decoder;
/**
- * Date: 2020/02/12
- * Company: www.dtstack.com
- *
- * @author tudou
+ * @author jiangbo
+ * @date 2020/3/23
*/
-public class PlainDecoder implements IDecode {
+public enum DecodeEnum {
+
+ /**
+ * json format
+ */
+ JSON("json"),
- @Override
- public Map decode(final String message) {
- return Collections.singletonMap("message", message);
+ /**
+ * text format
+ */
+ PLAIN("plain");
+
+ private String name;
+
+ DecodeEnum(String name) {
+ this.name = name;
}
+ public String getName() {
+ return name;
+ }
}
diff --git a/flinkx-kb/flinkx-kb-core/src/main/java/com/dtstack/flinkx/kafkabase/decoder/IDecode.java b/flinkx-core/src/main/java/com/dtstack/flinkx/decoder/IDecode.java
similarity index 96%
rename from flinkx-kb/flinkx-kb-core/src/main/java/com/dtstack/flinkx/kafkabase/decoder/IDecode.java
rename to flinkx-core/src/main/java/com/dtstack/flinkx/decoder/IDecode.java
index 0d117d3956..5b195bd668 100644
--- a/flinkx-kb/flinkx-kb-core/src/main/java/com/dtstack/flinkx/kafkabase/decoder/IDecode.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/decoder/IDecode.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.dtstack.flinkx.kafkabase.decoder;
+package com.dtstack.flinkx.decoder;
import java.util.Map;
diff --git a/flinkx-kb/flinkx-kb-core/src/main/java/com/dtstack/flinkx/kafkabase/decoder/JsonDecoder.java b/flinkx-core/src/main/java/com/dtstack/flinkx/decoder/JsonDecoder.java
similarity index 97%
rename from flinkx-kb/flinkx-kb-core/src/main/java/com/dtstack/flinkx/kafkabase/decoder/JsonDecoder.java
rename to flinkx-core/src/main/java/com/dtstack/flinkx/decoder/JsonDecoder.java
index 8d6fad179e..10a954e601 100644
--- a/flinkx-kb/flinkx-kb-core/src/main/java/com/dtstack/flinkx/kafkabase/decoder/JsonDecoder.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/decoder/JsonDecoder.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.dtstack.flinkx.kafkabase.decoder;
+package com.dtstack.flinkx.decoder;
import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
@@ -31,12 +31,10 @@
* @author tudou
*/
public class JsonDecoder implements IDecode {
+ private static final String KEY_MESSAGE = "message";
private static Logger LOG = LoggerFactory.getLogger(JsonDecoder.class);
-
private static ObjectMapper objectMapper = new ObjectMapper();
- private static final String KEY_MESSAGE = "message";
-
@Override
@SuppressWarnings("unchecked")
public Map decode(final String message) {
diff --git a/flinkx-kb/flinkx-kb-core/src/main/java/com/dtstack/flinkx/kafkabase/decoder/PlainDecoder.java b/flinkx-core/src/main/java/com/dtstack/flinkx/decoder/PlainDecoder.java
similarity index 96%
rename from flinkx-kb/flinkx-kb-core/src/main/java/com/dtstack/flinkx/kafkabase/decoder/PlainDecoder.java
rename to flinkx-core/src/main/java/com/dtstack/flinkx/decoder/PlainDecoder.java
index 5a2a027369..63771cfd6e 100644
--- a/flinkx-kb/flinkx-kb-core/src/main/java/com/dtstack/flinkx/kafkabase/decoder/PlainDecoder.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/decoder/PlainDecoder.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.dtstack.flinkx.kafkabase.decoder;
+package com.dtstack.flinkx.decoder;
import java.util.Collections;
import java.util.Map;
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/enums/EDatabaseType.java b/flinkx-core/src/main/java/com/dtstack/flinkx/enums/EDatabaseType.java
index 475f6923d1..8de25bd051 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/enums/EDatabaseType.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/enums/EDatabaseType.java
@@ -33,6 +33,7 @@ public enum EDatabaseType {
SQLServer,
Oracle,
PostgreSQL,
+ Greenplum,
DB2,
MongoDB,
Redis,
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/inputformat/BaseRichInputFormat.java b/flinkx-core/src/main/java/com/dtstack/flinkx/inputformat/BaseRichInputFormat.java
index 1ce9844175..2c55ee3a19 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/inputformat/BaseRichInputFormat.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/inputformat/BaseRichInputFormat.java
@@ -28,7 +28,6 @@
import com.dtstack.flinkx.metrics.CustomPrometheusReporter;
import com.dtstack.flinkx.reader.ByteRateLimiter;
import com.dtstack.flinkx.restore.FormatState;
-import org.apache.commons.lang.StringUtils;
import com.dtstack.flinkx.util.ExceptionUtil;
import org.apache.flink.api.common.accumulators.LongCounter;
import org.apache.flink.api.common.io.DefaultInputSplitAssigner;
@@ -376,14 +375,13 @@ public RestoreConfig getRestoreConfig() {
return restoreConfig;
}
- public void setLogConfig(LogConfig logConfig) {
- this.logConfig = logConfig;
- }
-
public void setRestoreConfig(RestoreConfig restoreConfig) {
this.restoreConfig = restoreConfig;
}
+ public void setLogConfig(LogConfig logConfig) {
+ this.logConfig = logConfig;
+ }
public void setTestConfig(TestConfig testConfig) {
this.testConfig = testConfig;
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/metrics/AccumulatorCollector.java b/flinkx-core/src/main/java/com/dtstack/flinkx/metrics/AccumulatorCollector.java
index c701707ade..a1f67a983f 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/metrics/AccumulatorCollector.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/metrics/AccumulatorCollector.java
@@ -25,7 +25,7 @@
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.google.gson.internal.LinkedTreeMap;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.accumulators.LongCounter;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -35,8 +35,13 @@
import org.slf4j.LoggerFactory;
import java.io.InputStream;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
/**
* Regularly get statistics from the flink API
@@ -247,7 +252,7 @@ private void checkErrorTimes() {
}
}
- class ValueAccumulator{
+ static class ValueAccumulator{
private long global;
private LongCounter local;
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/metrics/CustomPrometheusReporter.java b/flinkx-core/src/main/java/com/dtstack/flinkx/metrics/CustomPrometheusReporter.java
index 3735cc7c64..dfe35d84fe 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/metrics/CustomPrometheusReporter.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/metrics/CustomPrometheusReporter.java
@@ -27,7 +27,13 @@
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.metrics.*;
+import org.apache.flink.metrics.CharacterFilter;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Histogram;
+import org.apache.flink.metrics.Meter;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.metrics.groups.AbstractMetricGroup;
import org.apache.flink.runtime.metrics.groups.FrontMetricGroup;
@@ -39,7 +45,14 @@
import java.io.IOException;
import java.lang.reflect.Field;
-import java.util.*;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
import java.util.regex.Pattern;
/**
@@ -121,7 +134,6 @@ public void open() {
if (StringUtils.isNullOrWhitespaceOnly(host) || port < 1) {
return;
-// throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
}
if (randomSuffix) {
@@ -190,6 +202,10 @@ private void notifyOfAddedMetric(final Metric metric, final String metricName, f
count = collectorWithCount.getValue();
} else {
collector = createCollector(metric, dimensionKeys, dimensionValues, scopedMetricName, helpString);
+ if (null == collector) {
+ return;
+ }
+
try {
collector.register(defaultRegistry);
} catch (Exception e) {
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/options/OptionParser.java b/flinkx-core/src/main/java/com/dtstack/flinkx/options/OptionParser.java
index f5991f1f9c..c46d451f48 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/options/OptionParser.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/options/OptionParser.java
@@ -100,19 +100,15 @@ public List getProgramExeArgList() throws Exception {
continue;
}else if(OPTION_JOB.equalsIgnoreCase(key)){
File file = new File(value.toString());
- FileInputStream in = new FileInputStream(file);
- byte[] filecontent = new byte[(int) file.length()];
- in.read(filecontent);
- value = new String(filecontent, Charsets.UTF_8.name());
+ try (FileInputStream in = new FileInputStream(file)) {
+ byte[] filecontent = new byte[(int) file.length()];
+ in.read(filecontent);
+ value = new String(filecontent, Charsets.UTF_8.name());
+ }
}
args.add("-" + key);
args.add(value.toString());
}
return args;
}
-
- private void printUsage() {
- System.out.print(options.toString());
- }
-
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/options/Options.java b/flinkx-core/src/main/java/com/dtstack/flinkx/options/Options.java
index 0123d3f955..26a1189ecd 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/options/Options.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/options/Options.java
@@ -64,9 +64,9 @@ public class Options {
@OptionRequired(description = "env properties")
private String confProp = "{}";
- /**
- * savepoint
- */
+ @OptionRequired(description = "json modify")
+ private String p = "";
+
@OptionRequired(description = "savepoint path")
private String s;
@@ -195,4 +195,12 @@ public String getPluginLoadMode() {
public void setPluginLoadMode(String pluginLoadMode) {
this.pluginLoadMode = pluginLoadMode;
}
+
+ public String getP() {
+ return p;
+ }
+
+ public void setP(String p) {
+ this.p = p;
+ }
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseFileOutputFormat.java b/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseFileOutputFormat.java
index 6d92035700..f9735c0ab3 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseFileOutputFormat.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseFileOutputFormat.java
@@ -294,11 +294,12 @@ protected void afterCloseInternal() {
moveAllTemporaryDataFileToDirectory();
LOG.info("The task ran successfully,clear temporary data files");
+ closeSource();
clearTemporaryDataFiles();
}
+ }else{
+ closeSource();
}
-
- closeSource();
} catch(Exception ex) {
throw new RuntimeException(ex);
}
@@ -328,11 +329,6 @@ public void tryCleanupOnError() throws Exception {
}
}
- @Override
- protected void writeMultipleRecordsInternal() throws Exception {
- // CAN NOT HAPPEN
- }
-
@Override
protected boolean needWaitAfterCloseInternal() {
return true;
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseRichOutputFormat.java b/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseRichOutputFormat.java
index 1b42017236..39ce43bc7b 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseRichOutputFormat.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseRichOutputFormat.java
@@ -34,7 +34,7 @@
import com.dtstack.flinkx.writer.ErrorLimiter;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.accumulators.LongCounter;
import org.apache.flink.api.common.io.CleanupWhenUnsuccessful;
import org.apache.flink.configuration.Configuration;
@@ -51,7 +51,9 @@
import java.util.List;
import java.util.Map;
-import static com.dtstack.flinkx.writer.WriteErrorTypes.*;
+import static com.dtstack.flinkx.writer.WriteErrorTypes.ERR_FORMAT_TRANSFORM;
+import static com.dtstack.flinkx.writer.WriteErrorTypes.ERR_NULL_POINTER;
+import static com.dtstack.flinkx.writer.WriteErrorTypes.ERR_PRIMARY_CONFLICT;
/**
* Abstract Specification for all the OutputFormat defined in flinkx plugins
@@ -418,6 +420,10 @@ protected void writeMultipleRecords() throws Exception {
*/
protected abstract void writeMultipleRecordsInternal() throws Exception;
+ protected void notSupportBatchWrite(String writerName) {
+ throw new UnsupportedOperationException(writerName + "不支持批量写入");
+ }
+
protected void writeRecordInternal() {
try {
writeMultipleRecords();
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseRichOutputFormatBuilder.java b/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseRichOutputFormatBuilder.java
index 0bdcca624a..38306365f2 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseRichOutputFormatBuilder.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/outputformat/BaseRichOutputFormatBuilder.java
@@ -19,6 +19,7 @@
package com.dtstack.flinkx.outputformat;
import com.dtstack.flinkx.config.RestoreConfig;
+import com.dtstack.flinkx.constants.ConstantValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
@@ -71,6 +72,12 @@ public void setInitAccumulatorAndDirty(boolean initAccumulatorAndDirty) {
this.format.initAccumulatorAndDirty = initAccumulatorAndDirty;
}
+ protected void notSupportBatchWrite(String writerName) {
+ if (this.format.getBatchInterval() > 1) {
+ throw new IllegalArgumentException(writerName + "不支持批量写入");
+ }
+ }
+
/**
* Check the value of parameters
*/
@@ -78,7 +85,17 @@ public void setInitAccumulatorAndDirty(boolean initAccumulatorAndDirty) {
public BaseRichOutputFormat finish() {
checkFormat();
+
+ /**
+ * 200000条限制的原因：
+ * 按照目前的使用情况以及部署配置，假设写入字段数量平均为50个，一个单slot的TaskManager内存为1G，
+ * 在不考虑各插件批量写入对内存特殊要求并且只考虑插件缓存这么多条数据的情况下，batchInterval为400000条时出现fullGC，
+ * 为了避免fullGC以及OOM，并且保证batchInterval有足够的配置空间，取最大值的一半200000。
+ */
+ if (this.format.getBatchInterval() > ConstantValue.MAX_BATCH_SIZE) {
+ throw new IllegalArgumentException("批量写入条数必须小于[200000]条");
+ }
+
return format;
}
-
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/reader/ByteRateLimiter.java b/flinkx-core/src/main/java/com/dtstack/flinkx/reader/ByteRateLimiter.java
index f66d45b7f9..13c21f8129 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/reader/ByteRateLimiter.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/reader/ByteRateLimiter.java
@@ -25,7 +25,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.concurrent.*;
+import java.math.BigDecimal;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
/**
* This class is user for speed control
@@ -80,11 +84,12 @@ private void updateRate(){
long thisRecords = accumulatorCollector.getLocalAccumulatorValue(Metrics.NUM_READS);
long totalRecords = accumulatorCollector.getAccumulatorValue(Metrics.NUM_READS);
- double thisWriteRatio = (totalRecords == 0 ? 0 : thisRecords / (double)totalRecords);
+ BigDecimal thisWriteRatio = BigDecimal.valueOf(totalRecords == 0 ? 0 : thisRecords / (double) totalRecords);
- if (totalRecords > MIN_RECORD_NUMBER_UPDATE_RATE && totalBytes != 0 && thisWriteRatio != 0) {
- double bpr = totalBytes / totalRecords;
- double permitsPerSecond = expectedBytePerSecond / bpr * thisWriteRatio;
+ if (totalRecords > MIN_RECORD_NUMBER_UPDATE_RATE && totalBytes != 0
+ && thisWriteRatio.compareTo(new BigDecimal(0)) == 0) {
+ double bpr = totalBytes / (double)totalRecords;
+ double permitsPerSecond = expectedBytePerSecond / bpr * thisWriteRatio.doubleValue();
rateLimiter.setRate(permitsPerSecond);
}
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/reader/MetaColumn.java b/flinkx-core/src/main/java/com/dtstack/flinkx/reader/MetaColumn.java
index d328115334..a7436f894f 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/reader/MetaColumn.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/reader/MetaColumn.java
@@ -20,7 +20,6 @@
import com.dtstack.flinkx.constants.ConstantValue;
import com.dtstack.flinkx.util.DateUtil;
-import com.dtstack.flinkx.util.StringUtil;
import org.apache.commons.lang.StringUtils;
import java.io.Serializable;
@@ -48,6 +47,8 @@ public class MetaColumn implements Serializable {
private String splitter;
+ private Boolean isPart;
+
public String getSplitter() {
return splitter;
}
@@ -124,6 +125,7 @@ public static List getMetaColumns(List columns, boolean generateInde
mc.setType(sm.get("type") != null ? String.valueOf(sm.get("type")) : null);
mc.setValue(sm.get("value") != null ? String.valueOf(sm.get("value")) : null);
mc.setSplitter(sm.get("splitter") != null ? String.valueOf(sm.get("splitter")) : null);
+ mc.setPart(sm.get("isPart") != null ? (Boolean) sm.get("isPart") : false);
if(sm.get("format") != null && String.valueOf(sm.get("format")).trim().length() > 0){
mc.setTimeFormat(DateUtil.buildDateFormatter(String.valueOf(sm.get("format"))));
@@ -151,6 +153,14 @@ public static List getMetaColumns(List columns, boolean generateInde
return metaColumns;
}
+ public Boolean getPart() {
+ return isPart;
+ }
+
+ public void setPart(Boolean part) {
+ isPart = part;
+ }
+
public static List getMetaColumns(List columns){
return getMetaColumns(columns, true);
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/util/ClassUtil.java b/flinkx-core/src/main/java/com/dtstack/flinkx/util/ClassUtil.java
index bba853e451..b073524412 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/util/ClassUtil.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/util/ClassUtil.java
@@ -33,7 +33,7 @@ public class ClassUtil {
private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class);
- public final static String LOCK_STR = "jdbc_lock_str";
+ public final static Object LOCK_STR = new Object();
public static void forName(String clazz, ClassLoader classLoader) {
synchronized (LOCK_STR){
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/util/Clock.java b/flinkx-core/src/main/java/com/dtstack/flinkx/util/Clock.java
index 62f3a7e98c..c003af9457 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/util/Clock.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/util/Clock.java
@@ -46,9 +46,14 @@ static Clock system() {
return SYSTEM;
}
+ /**
+ * Get current time
+ * @return time
+ */
default Instant currentTime() {
return Instant.ofEpochMilli(currentTimeInMillis());
}
+
/**
* Get the current time in nanoseconds.
* @return the current time in nanoseconds.
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/util/DateUtil.java b/flinkx-core/src/main/java/com/dtstack/flinkx/util/DateUtil.java
index 8ce417196d..a9b2ce6e53 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/util/DateUtil.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/util/DateUtil.java
@@ -21,7 +21,11 @@
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TimeZone;
/**
* Date Utilities
@@ -93,7 +97,12 @@ public static java.sql.Date columnToDate(Object column,SimpleDateFormat customTi
if (((String) column).length() == 0){
return null;
}
- return new java.sql.Date(stringToDate((String)column,customTimeFormat).getTime());
+
+ Date date = stringToDate((String)column, customTimeFormat);
+ if (null == date) {
+ return null;
+ }
+ return new java.sql.Date(date.getTime());
} else if (column instanceof Integer) {
Integer rawData = (Integer) column;
return new java.sql.Date(getMillSecond(rawData.toString()));
@@ -120,7 +129,12 @@ public static java.sql.Timestamp columnToTimestamp(Object column,SimpleDateForma
if (((String) column).length() == 0){
return null;
}
- return new java.sql.Timestamp(stringToDate((String)column,customTimeFormat).getTime());
+
+ Date date = stringToDate((String)column,customTimeFormat);
+ if (null == date) {
+ return null;
+ }
+ return new java.sql.Timestamp(date.getTime());
} else if (column instanceof Integer) {
Integer rawData = (Integer) column;
return new java.sql.Timestamp(getMillSecond(rawData.toString()));
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/util/ExceptionUtil.java b/flinkx-core/src/main/java/com/dtstack/flinkx/util/ExceptionUtil.java
index cddcf1ba29..d770364db7 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/util/ExceptionUtil.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/util/ExceptionUtil.java
@@ -33,30 +33,19 @@ public class ExceptionUtil {
private static Logger logger = LoggerFactory.getLogger(ExceptionUtil.class);
public static String getErrorMessage(Throwable e) {
- StringWriter stringWriter = null;
- PrintWriter writer = null;
- try{
- stringWriter= new StringWriter();
- writer = new PrintWriter(stringWriter);
- e.printStackTrace(writer);
- writer.flush();
- stringWriter.flush();
- StringBuffer buffer= stringWriter.getBuffer();
- return buffer.toString();
- }catch(Throwable ee){
- logger.error("",ee);
+ if (null == e) {
+ return null;
+ }
- }finally {
- if(writer!=null){
- writer.close();
- }
- if(stringWriter!=null){
- try{
- stringWriter.close();
- }catch (Throwable ee){
- logger.error("",ee);
- }
- }
+ try (StringWriter stringWriter = new StringWriter();
+ PrintWriter writer = new PrintWriter(stringWriter)) {
+ e.printStackTrace(writer);
+ writer.flush();
+ stringWriter.flush();
+ StringBuffer buffer = stringWriter.getBuffer();
+ return buffer.toString();
+ } catch (Throwable ee) {
+ logger.error("", ee);
}
return null;
}
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/util/FileSystemUtil.java b/flinkx-core/src/main/java/com/dtstack/flinkx/util/FileSystemUtil.java
index c545ef3d94..0e7214fb58 100644
--- a/flinkx-core/src/main/java/com/dtstack/flinkx/util/FileSystemUtil.java
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/util/FileSystemUtil.java
@@ -29,6 +29,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.security.PrivilegedAction;
import java.util.HashMap;
import java.util.Map;
@@ -76,7 +77,7 @@ public static void setHadoopUserName(Configuration conf){
}
}
- private static boolean isOpenKerberos(Map hadoopConfig){
+ public static boolean isOpenKerberos(Map hadoopConfig){
if(!MapUtils.getBoolean(hadoopConfig, KEY_HADOOP_SECURITY_AUTHORIZATION, false)){
return false;
}
@@ -85,13 +86,7 @@ private static boolean isOpenKerberos(Map hadoopConfig){
}
private static FileSystem getFsWithKerberos(Map hadoopConfig, String defaultFs) throws Exception{
- String keytabFileName = KerberosUtil.getPrincipalFileName(hadoopConfig);
-
- keytabFileName = KerberosUtil.loadFile(hadoopConfig, keytabFileName);
- String principal = KerberosUtil.findPrincipalFromKeytab(keytabFileName);
- KerberosUtil.loadKrb5Conf(hadoopConfig);
-
- UserGroupInformation ugi = KerberosUtil.loginAndReturnUgi(getConfiguration(hadoopConfig, defaultFs), principal, keytabFileName);
+ UserGroupInformation ugi = getUGI(hadoopConfig, defaultFs);
UserGroupInformation.setLoginUser(ugi);
return ugi.doAs(new PrivilegedAction() {
@@ -106,6 +101,19 @@ public FileSystem run(){
});
}
+ public static UserGroupInformation getUGI(Map hadoopConfig, String defaultFs) throws IOException {
+ String keytabFileName = KerberosUtil.getPrincipalFileName(hadoopConfig);
+
+ keytabFileName = KerberosUtil.loadFile(hadoopConfig, keytabFileName);
+ String principal = KerberosUtil.getPrincipal(hadoopConfig, keytabFileName);
+ KerberosUtil.loadKrb5Conf(hadoopConfig);
+
+ UserGroupInformation ugi = KerberosUtil.loginAndReturnUgi(getConfiguration(hadoopConfig, defaultFs), principal, keytabFileName);
+ UserGroupInformation.setLoginUser(ugi);
+
+ return ugi;
+ }
+
public static Configuration getConfiguration(Map confMap, String defaultFs) {
confMap = fillConfig(confMap, defaultFs);
@@ -134,7 +142,7 @@ public static JobConf getJobConf(Map confMap, String defaultFs){
private static Map fillConfig(Map confMap, String defaultFs) {
if (confMap == null) {
- confMap = new HashMap<>(8);
+ confMap = new HashMap<>();
}
if (isHaMode(confMap)) {
diff --git a/flinkx-core/src/main/java/com/dtstack/flinkx/util/GsonUtil.java b/flinkx-core/src/main/java/com/dtstack/flinkx/util/GsonUtil.java
new file mode 100644
index 0000000000..b94765dc4a
--- /dev/null
+++ b/flinkx-core/src/main/java/com/dtstack/flinkx/util/GsonUtil.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.dtstack.flinkx.util;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.TypeAdapter;
+import com.google.gson.TypeAdapterFactory;
+import com.google.gson.internal.LinkedTreeMap;
+import com.google.gson.internal.bind.ObjectTypeAdapter;
+import com.google.gson.reflect.TypeToken;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonToken;
+import com.google.gson.stream.JsonWriter;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Date: 2020/06/12
+ * Company: www.dtstack.com
+ *
+ * Gson工具类，用于对json的序列化及反序列化，及解决int类型在map中被转换成double类型问题
+ *
+ * @author tudou
+ */
+public class GsonUtil {
+
+ public static Gson GSON = getGson();
+ public static Type gsonMapTypeToken = new TypeToken>(){}.getType();
+
+ @SuppressWarnings("unchecked")
+ private static Gson getGson() {
+ GSON = new GsonBuilder().create();
+ try {
+ Field factories = Gson.class.getDeclaredField("factories");
+ factories.setAccessible(true);
+ Object o = factories.get(GSON);
+ Class[] declaredClasses = Collections.class.getDeclaredClasses();
+ for (Class c : declaredClasses) {
+ if ("java.util.Collections$UnmodifiableList".equals(c.getName())) {
+ Field listField = c.getDeclaredField("list");
+ listField.setAccessible(true);
+ List list = (List) listField.get(o);
+ int i = list.indexOf(ObjectTypeAdapter.FACTORY);
+ list.set(i, new TypeAdapterFactory() {
+ @Override
+ public TypeAdapter create(Gson gson, TypeToken type) {
+ if (type.getRawType() == Object.class) {
+ return new TypeAdapter() {
+ @Override
+ public Object read(JsonReader in) throws IOException {
+ JsonToken token = in.peek();
+ //判断字符串的实际类型
+ switch (token) {
+ case BEGIN_ARRAY:
+ List
- com.google
- shade.db2reader.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-db2/flinkx-db2-writer/pom.xml b/flinkx-db2/flinkx-db2-writer/pom.xml
index 2ff75d22e0..b3d103b159 100644
--- a/flinkx-db2/flinkx-db2-writer/pom.xml
+++ b/flinkx-db2/flinkx-db2-writer/pom.xml
@@ -63,8 +63,12 @@
shade.db2writer.io.netty
- com.google
- shade.db2writer.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-dm/flinkx-dm-core/src/main/java/com/dtstack/flinkx/dm/DmDatabaseMeta.java b/flinkx-dm/flinkx-dm-core/src/main/java/com/dtstack/flinkx/dm/DmDatabaseMeta.java
index 3c4d2883dc..da6829b005 100644
--- a/flinkx-dm/flinkx-dm-core/src/main/java/com/dtstack/flinkx/dm/DmDatabaseMeta.java
+++ b/flinkx-dm/flinkx-dm-core/src/main/java/com/dtstack/flinkx/dm/DmDatabaseMeta.java
@@ -47,7 +47,7 @@ protected String makeValues(List column) {
public String quoteTable(String table) {
table = table.replace("\"","");
String[] part = table.split("\\.");
- if(part.length == 2) {
+ if(part.length == DB_TABLE_PART_SIZE) {
table = getStartQuote() + part[0] + getEndQuote() + "." + getStartQuote() + part[1] + getEndQuote();
} else {
table = getStartQuote() + table + getEndQuote();
diff --git a/flinkx-dm/flinkx-dm-reader/pom.xml b/flinkx-dm/flinkx-dm-reader/pom.xml
index 15ce92929b..c2feddd9c8 100644
--- a/flinkx-dm/flinkx-dm-reader/pom.xml
+++ b/flinkx-dm/flinkx-dm-reader/pom.xml
@@ -63,8 +63,12 @@
shade.dmreader.io.netty
- com.google
- shade.dmreader.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-dm/flinkx-dm-reader/src/main/java/com/dtstack/flinkx/dm/format/DmInputFormat.java b/flinkx-dm/flinkx-dm-reader/src/main/java/com/dtstack/flinkx/dm/format/DmInputFormat.java
index 8a114752f1..67e70d499d 100644
--- a/flinkx-dm/flinkx-dm-reader/src/main/java/com/dtstack/flinkx/dm/format/DmInputFormat.java
+++ b/flinkx-dm/flinkx-dm-reader/src/main/java/com/dtstack/flinkx/dm/format/DmInputFormat.java
@@ -25,8 +25,6 @@
import java.io.IOException;
import java.sql.Timestamp;
-import static com.dtstack.flinkx.rdb.util.DbUtil.clobToString;
-
/**
* Date: 2020/03/18
* Company: www.dtstack.com
@@ -49,7 +47,7 @@ public Row nextRecordInternal(Row row) throws IOException {
|| obj.getClass().getSimpleName().toUpperCase().contains("TIMESTAMP")) ) {
obj = resultSet.getTimestamp(pos + 1);
}
- obj = clobToString(obj);
+ obj = DbUtil.clobToString(obj);
}
row.setField(pos, obj);
diff --git a/flinkx-dm/flinkx-dm-writer/pom.xml b/flinkx-dm/flinkx-dm-writer/pom.xml
index abe284fc04..768be50eae 100644
--- a/flinkx-dm/flinkx-dm-writer/pom.xml
+++ b/flinkx-dm/flinkx-dm-writer/pom.xml
@@ -63,8 +63,12 @@
shade.dmwriter.io.netty
- com.google
- shade.dmwriter.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-dm/flinkx-dm-writer/src/main/java/com/dtstack/flinkx/dm/format/DmOutputFormat.java b/flinkx-dm/flinkx-dm-writer/src/main/java/com/dtstack/flinkx/dm/format/DmOutputFormat.java
index c3912a218c..9612294ec8 100644
--- a/flinkx-dm/flinkx-dm-writer/src/main/java/com/dtstack/flinkx/dm/format/DmOutputFormat.java
+++ b/flinkx-dm/flinkx-dm-writer/src/main/java/com/dtstack/flinkx/dm/format/DmOutputFormat.java
@@ -18,11 +18,16 @@
package com.dtstack.flinkx.dm.format;
import com.dtstack.flinkx.enums.ColumnType;
+import com.dtstack.flinkx.rdb.BaseDatabaseMeta;
import com.dtstack.flinkx.rdb.outputformat.JdbcOutputFormat;
import com.dtstack.flinkx.util.DateUtil;
import org.apache.flink.types.Row;
-import java.sql.*;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
@@ -61,7 +66,7 @@ protected List probeFullColumns(String table, Connection dbConn) throws
String schema =null;
String[] parts = table.split("\\.");
- if(parts.length == 2) {
+ if(parts.length == BaseDatabaseMeta.DB_TABLE_PART_SIZE) {
schema = parts[0].toUpperCase();
table = parts[1];
}
@@ -76,25 +81,27 @@ protected List probeFullColumns(String table, Connection dbConn) throws
@Override
protected Map> probePrimaryKeys(String table, Connection dbConn) throws SQLException {
- Map> map = new HashMap<>();
- PreparedStatement ps = dbConn.prepareStatement(String.format(GET_INDEX_SQL,table));
- ResultSet rs = ps.executeQuery();
+ Map> map = new HashMap<>(16);
- while(rs.next()) {
- String indexName = rs.getString("INDEX_NAME");
- if(!map.containsKey(indexName)) {
- map.put(indexName,new ArrayList<>());
+ try (PreparedStatement ps = dbConn.prepareStatement(String.format(GET_INDEX_SQL,table));
+ ResultSet rs = ps.executeQuery()) {
+ while(rs.next()) {
+ String indexName = rs.getString("INDEX_NAME");
+ if(!map.containsKey(indexName)) {
+ map.put(indexName,new ArrayList<>());
+ }
+ map.get(indexName).add(rs.getString("COLUMN_NAME"));
}
- map.get(indexName).add(rs.getString("COLUMN_NAME"));
- }
- Map> retMap = new HashMap<>();
- for(Map.Entry> entry: map.entrySet()) {
- String k = entry.getKey();
- List v = entry.getValue();
- if(v!=null && v.size() != 0 && v.get(0) != null) {
- retMap.put(k, v);
+
+ Map> retMap = new HashMap<>(16);
+ for(Map.Entry> entry: map.entrySet()) {
+ String k = entry.getKey();
+ List v = entry.getValue();
+ if(v!=null && v.size() != 0 && v.get(0) != null) {
+ retMap.put(k, v);
+ }
}
+ return retMap;
}
- return retMap;
}
}
diff --git a/flinkx-emqx/flinkx-emqx-core/src/main/java/com/dtstack/flinkx/emqx/decoder/JsonDecoder.java b/flinkx-emqx/flinkx-emqx-core/src/main/java/com/dtstack/flinkx/emqx/decoder/JsonDecoder.java
deleted file mode 100644
index ecb26a2f3e..0000000000
--- a/flinkx-emqx/flinkx-emqx-core/src/main/java/com/dtstack/flinkx/emqx/decoder/JsonDecoder.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.dtstack.flinkx.emqx.decoder;
-
-import org.codehaus.jackson.map.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * Date: 2020/02/12
- * Company: www.dtstack.com
- *
- * @author tudou
- */
-public class JsonDecoder implements IDecode {
- private static Logger LOG = LoggerFactory.getLogger(JsonDecoder.class);
-
- private static ObjectMapper objectMapper = new ObjectMapper();
-
- private static final String KEY_MESSAGE = "message";
-
- @Override
- @SuppressWarnings("unchecked")
- public Map decode(final String message) {
- try {
- Map event = objectMapper.readValue(message, Map.class);
- if (!event.containsKey(KEY_MESSAGE)) {
- event.put(KEY_MESSAGE, message);
- }
- return event;
- } catch (Exception e) {
- LOG.error(e.getMessage());
- return Collections.singletonMap(KEY_MESSAGE, message);
- }
-
- }
-
-}
diff --git a/flinkx-emqx/flinkx-emqx-reader/pom.xml b/flinkx-emqx/flinkx-emqx-reader/pom.xml
index 0e106143fd..205e8a9c5c 100644
--- a/flinkx-emqx/flinkx-emqx-reader/pom.xml
+++ b/flinkx-emqx/flinkx-emqx-reader/pom.xml
@@ -53,6 +53,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-emqx/flinkx-emqx-reader/src/main/java/com/dtstack/flinkx/emqx/format/EmqxInputFormat.java b/flinkx-emqx/flinkx-emqx-reader/src/main/java/com/dtstack/flinkx/emqx/format/EmqxInputFormat.java
index 5534601f51..7e92921dcc 100644
--- a/flinkx-emqx/flinkx-emqx-reader/src/main/java/com/dtstack/flinkx/emqx/format/EmqxInputFormat.java
+++ b/flinkx-emqx/flinkx-emqx-reader/src/main/java/com/dtstack/flinkx/emqx/format/EmqxInputFormat.java
@@ -17,16 +17,21 @@
*/
package com.dtstack.flinkx.emqx.format;
-import com.dtstack.flinkx.emqx.decoder.IDecode;
-import com.dtstack.flinkx.emqx.decoder.JsonDecoder;
-import com.dtstack.flinkx.emqx.decoder.PlainDecoder;
+import com.dtstack.flinkx.decoder.IDecode;
+import com.dtstack.flinkx.decoder.JsonDecoder;
+import com.dtstack.flinkx.decoder.PlainDecoder;
import com.dtstack.flinkx.inputformat.BaseRichInputFormat;
import com.dtstack.flinkx.util.ExceptionUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.core.io.GenericInputSplit;
import org.apache.flink.core.io.InputSplit;
import org.apache.flink.types.Row;
-import org.eclipse.paho.client.mqttv3.*;
+import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
+import org.eclipse.paho.client.mqttv3.MqttCallback;
+import org.eclipse.paho.client.mqttv3.MqttClient;
+import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
+import org.eclipse.paho.client.mqttv3.MqttException;
+import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/flinkx-emqx/flinkx-emqx-reader/src/main/java/com/dtstack/flinkx/emqx/reader/EmqxReader.java b/flinkx-emqx/flinkx-emqx-reader/src/main/java/com/dtstack/flinkx/emqx/reader/EmqxReader.java
index 83d70d80d1..32121eec6a 100644
--- a/flinkx-emqx/flinkx-emqx-reader/src/main/java/com/dtstack/flinkx/emqx/reader/EmqxReader.java
+++ b/flinkx-emqx/flinkx-emqx-reader/src/main/java/com/dtstack/flinkx/emqx/reader/EmqxReader.java
@@ -8,7 +8,13 @@
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.types.Row;
-import static com.dtstack.flinkx.emqx.EmqxConfigKeys.*;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_BROKER;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_CODEC;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_IS_CLEAN_SESSION;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_PASSWORD;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_QOS;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_TOPIC;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_USERNAME;
/**
* Date: 2020/02/12
diff --git a/flinkx-emqx/flinkx-emqx-writer/pom.xml b/flinkx-emqx/flinkx-emqx-writer/pom.xml
index a6117433f5..fcf861883a 100644
--- a/flinkx-emqx/flinkx-emqx-writer/pom.xml
+++ b/flinkx-emqx/flinkx-emqx-writer/pom.xml
@@ -53,6 +53,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-emqx/flinkx-emqx-writer/src/main/java/com/dtstack/flinkx/emqx/format/EmqxOutputFormat.java b/flinkx-emqx/flinkx-emqx-writer/src/main/java/com/dtstack/flinkx/emqx/format/EmqxOutputFormat.java
index 79dec0f172..f1e87baba3 100644
--- a/flinkx-emqx/flinkx-emqx-writer/src/main/java/com/dtstack/flinkx/emqx/format/EmqxOutputFormat.java
+++ b/flinkx-emqx/flinkx-emqx-writer/src/main/java/com/dtstack/flinkx/emqx/format/EmqxOutputFormat.java
@@ -17,7 +17,7 @@
*/
package com.dtstack.flinkx.emqx.format;
-import com.dtstack.flinkx.emqx.decoder.JsonDecoder;
+import com.dtstack.flinkx.decoder.JsonDecoder;
import com.dtstack.flinkx.exception.WriteRecordException;
import com.dtstack.flinkx.outputformat.BaseRichOutputFormat;
import com.dtstack.flinkx.util.ExceptionUtil;
diff --git a/flinkx-emqx/flinkx-emqx-writer/src/main/java/com/dtstack/flinkx/emqx/writer/EmqxWriter.java b/flinkx-emqx/flinkx-emqx-writer/src/main/java/com/dtstack/flinkx/emqx/writer/EmqxWriter.java
index 0ba2a1a12d..52a311b76e 100644
--- a/flinkx-emqx/flinkx-emqx-writer/src/main/java/com/dtstack/flinkx/emqx/writer/EmqxWriter.java
+++ b/flinkx-emqx/flinkx-emqx-writer/src/main/java/com/dtstack/flinkx/emqx/writer/EmqxWriter.java
@@ -25,7 +25,12 @@
import org.apache.flink.streaming.api.datastream.DataStreamSink;
import org.apache.flink.types.Row;
-import static com.dtstack.flinkx.emqx.EmqxConfigKeys.*;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_BROKER;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_IS_CLEAN_SESSION;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_PASSWORD;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_QOS;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_TOPIC;
+import static com.dtstack.flinkx.emqx.EmqxConfigKeys.KEY_USERNAME;
/**
* Date: 2020/02/12
diff --git a/flinkx-es/flinkx-es-reader/pom.xml b/flinkx-es/flinkx-es-reader/pom.xml
index 227dc25d10..8ccef94149 100644
--- a/flinkx-es/flinkx-es-reader/pom.xml
+++ b/flinkx-es/flinkx-es-reader/pom.xml
@@ -53,6 +53,20 @@
+
+
+ com.fasterxml.jackson.databind
+ shade.es.com.fasterxml.jackson.databind
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsInputFormat.java b/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsInputFormat.java
index 282e2ac5f8..1c54a5e115 100644
--- a/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsInputFormat.java
+++ b/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsInputFormat.java
@@ -25,7 +25,11 @@
import org.apache.flink.core.io.GenericInputSplit;
import org.apache.flink.core.io.InputSplit;
import org.apache.flink.types.Row;
-import org.elasticsearch.action.search.*;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
diff --git a/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsInputFormatBuilder.java b/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsInputFormatBuilder.java
index 33f9e85d21..57584faeb4 100644
--- a/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsInputFormatBuilder.java
+++ b/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsInputFormatBuilder.java
@@ -18,6 +18,7 @@
package com.dtstack.flinkx.es.reader;
+import com.dtstack.flinkx.constants.ConstantValue;
import com.dtstack.flinkx.inputformat.BaseRichInputFormatBuilder;
import java.util.List;
import java.util.Map;
@@ -103,5 +104,9 @@ protected void checkFormat() {
if (format.getRestoreConfig() != null && format.getRestoreConfig().isRestore()){
throw new UnsupportedOperationException("This plugin not support restore from failed state");
}
+
+ if (format.batchSize > ConstantValue.MAX_BATCH_SIZE) {
+ throw new IllegalArgumentException("批量读取数量不能大于[200000]条");
+ }
}
}
diff --git a/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsReader.java b/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsReader.java
index da95c5b0ee..e57fa881a4 100644
--- a/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsReader.java
+++ b/flinkx-es/flinkx-es-reader/src/main/java/com/dtstack/flinkx/es/reader/EsReader.java
@@ -28,6 +28,8 @@
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.types.Row;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
@@ -42,6 +44,8 @@
*/
public class EsReader extends BaseDataReader {
+ private static Logger LOG = LoggerFactory.getLogger(EsReader.class);
+
private String address;
private String username;
private String password;
@@ -87,7 +91,8 @@ public EsReader(DataTransferConfig config, StreamExecutionEnvironment env) {
columnValue.add((String) sm.get("value"));
columnName.add((String) sm.get("name"));
}
- System.out.println("init column finished");
+
+ LOG.info("init column finished");
} else if (!ConstantValue.STAR_SYMBOL.equals(columns.get(0)) || columns.size() != 1) {
throw new IllegalArgumentException("column argument error");
}
diff --git a/flinkx-es/flinkx-es-reader/src/test/java/com/dtstack/flinkx/es/reader/test/EsReadDemo.java b/flinkx-es/flinkx-es-reader/src/test/java/com/dtstack/flinkx/es/reader/test/EsReadDemo.java
deleted file mode 100644
index 06e304c46a..0000000000
--- a/flinkx-es/flinkx-es-reader/src/test/java/com/dtstack/flinkx/es/reader/test/EsReadDemo.java
+++ /dev/null
@@ -1,90 +0,0 @@
-package com.dtstack.flinkx.es.reader.test;
-
-
-import com.google.gson.Gson;
-import org.apache.http.HttpHost;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.Aggregations;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-
-public class EsReadDemo {
-
- public static void searchAll() throws IOException {
- RestHighLevelClient client = new RestHighLevelClient(
- RestClient.builder(
- new HttpHost("rdos1", 9200, "http"),
- new HttpHost("rdos2", 9200, "http")));
- SearchRequest searchRequest = new SearchRequest();
-// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-// searchSourceBuilder.query(QueryBuilders.matchAllQuery());
-// searchSourceBuilder().from(0);
-// searchSourceBuilder().size(100);
-
- SearchResponse searchResponse = client.search(searchRequest);
- SearchHits searchHits = searchResponse.getHits();
-
- for(SearchHit searchHit : searchHits) {
- Map source = searchHit.getSourceAsMap();
- System.out.println(source);
- }
-
-
- }
-
- public static void searchPart() throws IOException {
- RestHighLevelClient client = new RestHighLevelClient(
- RestClient.builder(
- new HttpHost("rdos1", 9200, "http"),
- new HttpHost("rdos2", 9200, "http")));
- SearchRequest searchRequest = new SearchRequest();
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-
- sourceBuilder.from(0);
- sourceBuilder.size(100);
-
-// QueryBuilder qb2 = QueryBuilders.wrapperQuery("{\"match_all\": {}}");
- Map map = new HashMap<>();
- Map match = new HashMap<>();
- map.put("match",match);
- match.put("col2", "hallo");
- Gson gson = new Gson();
-
- //QueryBuilder qb2 = QueryBuilders.wrapperQuery("{\"match\": {\"col2\":\"hallo\"}}");
- QueryBuilder qb2 = QueryBuilders.wrapperQuery(gson.toJson(map));
- sourceBuilder.query(qb2);
- searchRequest.source(sourceBuilder);
-
- SearchResponse searchResponse = client.search(searchRequest);
- System.out.println(searchResponse);
- SearchHits searchHits = searchResponse.getHits();
- for(SearchHit searchHit : searchHits.getHits()) {
- System.out.println(searchHit.docId() + ":" + searchHit.getSourceAsMap());
- }
- long total = searchHits.getTotalHits();
- System.out.println("total: " + total);
-
- client.close();
- }
-
-
- public static void main(String[] args) throws IOException {
- //searchAll();
- searchPart();
- //searchAll();
- }
-
-}
diff --git a/flinkx-es/flinkx-es-reader/src/test/java/com/dtstack/flinkx/es/reader/test/EsReaderTest.java b/flinkx-es/flinkx-es-reader/src/test/java/com/dtstack/flinkx/es/reader/test/EsReaderTest.java
deleted file mode 100644
index bf98581615..0000000000
--- a/flinkx-es/flinkx-es-reader/src/test/java/com/dtstack/flinkx/es/reader/test/EsReaderTest.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package com.dtstack.flinkx.es.reader.test;
-
-import com.google.gson.Gson;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Created by softfly on 18/2/8.
- */
-public class EsReaderTest {
- public static void main(String[] args) {
- Gson gson = new Gson();
- Map map = new HashMap<>();
- map.put("xxx", 111);
- map.put("yyyy", "fff");
- String json = gson.toJson(map);
- System.out.println(json);
- }
-}
diff --git a/flinkx-es/flinkx-es-writer/pom.xml b/flinkx-es/flinkx-es-writer/pom.xml
index 1cdfab73fb..8e3e15a822 100644
--- a/flinkx-es/flinkx-es-writer/pom.xml
+++ b/flinkx-es/flinkx-es-writer/pom.xml
@@ -54,6 +54,20 @@
+
+
+ com.fasterxml.jackson.databind
+ shade.es.com.fasterxml.jackson.databind
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-es/flinkx-es-writer/src/test/java/com/dtstack/flinkx/es/writer/test/EsDemo.java b/flinkx-es/flinkx-es-writer/src/test/java/com/dtstack/flinkx/es/writer/test/EsDemo.java
deleted file mode 100644
index 503c09fa7e..0000000000
--- a/flinkx-es/flinkx-es-writer/src/test/java/com/dtstack/flinkx/es/writer/test/EsDemo.java
+++ /dev/null
@@ -1,146 +0,0 @@
-package com.dtstack.flinkx.es.writer.test;
-
-import org.apache.http.HttpHost;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class EsDemo {
-
- public static void test1() throws Exception {
- RestHighLevelClient client = new RestHighLevelClient(
- RestClient.builder(
- new HttpHost("rdos1", 9200, "http"),
- new HttpHost("rdos2", 9200, "http")));
-
- IndexRequest request = new IndexRequest(
- "nani222",
- "doc222",
- "id2");
-
- String jsonString = "{" +
- "\"user\":\"user2\"," +
- "\"postDate\":\"2014-01-30\"," +
- "\"message\":\"trying out Elasticsearch\"" +
- "}";
-
-
- request.source(jsonString, XContentType.JSON);
- IndexResponse response = client.index(request);
- System.out.println(response.getResult());
- client.close();
- }
-
- public static void test3() throws Exception {
- RestHighLevelClient client = new RestHighLevelClient(
- RestClient.builder(
- new HttpHost("study", 9200, "http"),
- new HttpHost("study", 9201, "http")));
-
- IndexRequest request = new IndexRequest(
- "nani",
- "doc");
-
-// String jsonString = "{" +
-// "\"user\":\"xxxx\"," +
-// "\"postDate\":\"2013-01-30\"," +
-// "\"message\":\"trying out Elasticsearch\"" +
-// "}";
- Map jsonMap = new HashMap<>();
- jsonMap.put("xxx", "asfdasdf");
- jsonMap.put("zzz", "asdfsadf");
- request.source(jsonMap);
- IndexResponse response = client.index(request);
- System.out.println(response.getResult());
- client.close();
- }
-
- public static void test2() throws Exception {
- RestHighLevelClient client = new RestHighLevelClient(
- RestClient.builder(
- new HttpHost("localhost", 9200, "http"),
- new HttpHost("localhost", 9201, "http")));
-
- UpdateRequest request = new UpdateRequest(
- "nani250",
- "doc",
- "2");
-
- String jsonString = "{" +
- "\"user\":\"kimchy\"," +
- "\"postDate\":\"2013-01-30\"," +
- "\"message\":\"trying out Elasticsearch\"" +
- "}";
-
- request.doc(jsonString, XContentType.JSON);
- UpdateResponse response = client.update(request);
- System.out.println(response.getResult());
- client.close();
- }
-
- public static void test4() throws IOException {
-
- RestHighLevelClient client = new RestHighLevelClient(
- RestClient.builder(
- new HttpHost("study", 9200, "http"),
- new HttpHost("study", 9201, "http")));
- SearchRequest searchRequest = new SearchRequest();
- SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
- searchSourceBuilder.query(QueryBuilders.matchAllQuery());
-
- SearchResponse searchResponse = client.search(searchRequest);
- System.out.println(searchResponse.getTotalShards());
- }
-
- public static void test5() throws Exception {
- RestHighLevelClient client = new RestHighLevelClient(
- RestClient.builder(
- new HttpHost("study", 9200, "http"),
- new HttpHost("study", 9201, "http")));
- BulkRequest bulkRequest = new BulkRequest();
-
- IndexRequest request = new IndexRequest("nani", "doc1");
- Map jsonMap = new HashMap<>();
- jsonMap.put("xxx", "8888");
- jsonMap.put("yyy", "9999");
-
- bulkRequest.add(request.source(jsonMap));
- // bulkRequest.setRefreshPolicy(null);
- // WriteRequest.RefreshPolicy;
-
- BulkResponse bulkResponse = client.bulk(bulkRequest);
-
- System.out.println(bulkResponse);
- }
-
- public static void test6() throws IOException {
- RestHighLevelClient client = new RestHighLevelClient(
- RestClient.builder(
- new HttpHost("study", 9200, "http"),
- new HttpHost("study", 9201, "http")));
- SearchRequest searchRequest = new SearchRequest();
- SearchResponse resp = client.search(searchRequest);
- resp.getAggregations();
- }
-
- public static void main(String[] args) throws Exception {
- test1();
- }
-
-
-}
diff --git a/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpConfig.java b/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpConfig.java
index fc58ca41e7..afc45b081a 100644
--- a/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpConfig.java
+++ b/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpConfig.java
@@ -59,7 +59,7 @@ public class FtpConfig implements Serializable {
public int timeout = FtpConfigConstants.DEFAULT_TIMEOUT;
- public long maxFileSize = 1024 * 1024 * 1024;
+ public long maxFileSize = 1024 * 1024 * 1024L;
public String getUsername() {
return username;
diff --git a/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpHandler.java b/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpHandler.java
index d201625068..4b99e46257 100644
--- a/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpHandler.java
+++ b/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpHandler.java
@@ -89,23 +89,14 @@ public void loginFtpServer(FtpConfig ftpConfig) {
}
@Override
- public void logoutFtpServer() {
+ public void logoutFtpServer() throws IOException{
if (ftpClient.isConnected()) {
try {
ftpClient.logout();
- } catch (IOException e) {
- LOG.error(DISCONNECT_FAIL_MESSAGE);
- throw new RuntimeException(e);
- }finally {
+ } finally {
if(ftpClient.isConnected()){
- try {
- ftpClient.disconnect();
- } catch (IOException e) {
- LOG.error(DISCONNECT_FAIL_MESSAGE);
- throw new RuntimeException(e);
- }
+ ftpClient.disconnect();
}
-
}
}
}
diff --git a/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpHandlerFactory.java b/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpHandlerFactory.java
index cc83e0343a..8c775f062d 100644
--- a/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpHandlerFactory.java
+++ b/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/FtpHandlerFactory.java
@@ -19,6 +19,8 @@
package com.dtstack.flinkx.ftp;
+import org.apache.commons.lang3.StringUtils;
+
/**
* @author jiangbo
* @date 2019/11/21
@@ -28,8 +30,8 @@ public class FtpHandlerFactory {
public static IFtpHandler createFtpHandler(String protocolStr){
IFtpHandler ftpHandler;
- EProtocol protocol = EProtocol.getByName(protocolStr);
- if(EProtocol.SFTP.equals(protocol)) {
+ Protocol protocol = Protocol.getByName(protocolStr);
+ if(Protocol.SFTP.equals(protocol)) {
ftpHandler = new SftpHandler();
} else {
ftpHandler = new FtpHandler();
@@ -37,4 +39,16 @@ public static IFtpHandler createFtpHandler(String protocolStr){
return ftpHandler;
}
+
+ enum Protocol{
+ FTP, SFTP;
+
+ public static Protocol getByName(String name) {
+ if (StringUtils.isEmpty(name)) {
+ return SFTP;
+ }
+
+ return Protocol.valueOf(name.toUpperCase());
+ }
+ }
}
diff --git a/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/IFtpHandler.java b/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/IFtpHandler.java
index c8da4124b1..16faf8e6d8 100644
--- a/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/IFtpHandler.java
+++ b/flinkx-ftp/flinkx-ftp-core/src/main/java/com/dtstack/flinkx/ftp/IFtpHandler.java
@@ -19,6 +19,7 @@
package com.dtstack.flinkx.ftp;
+import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
@@ -40,8 +41,9 @@ public interface IFtpHandler {
/**
* 登出服务器
+ * @throws IOException logout error
*/
- void logoutFtpServer();
+ void logoutFtpServer() throws IOException;
/**
* 判断给定的目录是否存在
diff --git a/flinkx-ftp/flinkx-ftp-reader/pom.xml b/flinkx-ftp/flinkx-ftp-reader/pom.xml
index 1771055c60..42ae5c1123 100644
--- a/flinkx-ftp/flinkx-ftp-reader/pom.xml
+++ b/flinkx-ftp/flinkx-ftp-reader/pom.xml
@@ -71,6 +71,16 @@ under the License.
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpInputFormat.java b/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpInputFormat.java
index 6f0f4471d3..3b3b62013c 100644
--- a/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpInputFormat.java
+++ b/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpInputFormat.java
@@ -19,7 +19,12 @@
package com.dtstack.flinkx.ftp.reader;
import com.dtstack.flinkx.constants.ConstantValue;
-import com.dtstack.flinkx.ftp.*;
+import com.dtstack.flinkx.ftp.EProtocol;
+import com.dtstack.flinkx.ftp.FtpConfig;
+import com.dtstack.flinkx.ftp.FtpHandler;
+import com.dtstack.flinkx.ftp.FtpHandlerFactory;
+import com.dtstack.flinkx.ftp.IFtpHandler;
+import com.dtstack.flinkx.ftp.SftpHandler;
import com.dtstack.flinkx.inputformat.BaseRichInputFormat;
import com.dtstack.flinkx.reader.MetaColumn;
import com.dtstack.flinkx.util.StringUtil;
@@ -54,13 +59,7 @@ public class FtpInputFormat extends BaseRichInputFormat {
public void openInputFormat() throws IOException {
super.openInputFormat();
- if(EProtocol.SFTP.name().equalsIgnoreCase(ftpConfig.getProtocol())) {
- ftpHandler = new SftpHandler();
- } else if(EProtocol.FTP.name().equalsIgnoreCase(ftpConfig.getProtocol())){
- ftpHandler = new FtpHandler();
- } else {
- throw new RuntimeException("协议名称错误:" + ftpConfig.getProtocol());
- }
+ ftpHandler = FtpHandlerFactory.createFtpHandler(ftpConfig.getProtocol());
ftpHandler.loginFtpServer(ftpConfig);
}
@@ -157,5 +156,4 @@ public void closeInternal() throws IOException {
ftpHandler.logoutFtpServer();
}
}
-
}
diff --git a/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpReader.java b/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpReader.java
index 411389206d..c889b249cf 100644
--- a/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpReader.java
+++ b/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpReader.java
@@ -21,6 +21,7 @@
import com.dtstack.flinkx.config.DataTransferConfig;
import com.dtstack.flinkx.config.ReaderConfig;
import com.dtstack.flinkx.ftp.FtpConfig;
+import com.dtstack.flinkx.ftp.FtpConfigConstants;
import com.dtstack.flinkx.reader.BaseDataReader;
import com.dtstack.flinkx.reader.MetaColumn;
import com.dtstack.flinkx.util.StringUtil;
@@ -30,8 +31,6 @@
import java.util.List;
-import static com.dtstack.flinkx.ftp.FtpConfigConstants.*;
-
/**
* The reader plugin of Ftp
*
@@ -57,7 +56,7 @@ public FtpReader(DataTransferConfig config, StreamExecutionEnvironment env) {
ftpConfig.setDefaultPort();
}
- if(!DEFAULT_FIELD_DELIMITER.equals(ftpConfig.getFieldDelimiter())){
+ if(!FtpConfigConstants.DEFAULT_FIELD_DELIMITER.equals(ftpConfig.getFieldDelimiter())){
String fieldDelimiter = StringUtil.convertRegularExpr(ftpConfig.getFieldDelimiter());
ftpConfig.setFieldDelimiter(fieldDelimiter);
}
diff --git a/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpSeqBufferedReader.java b/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpSeqBufferedReader.java
index a19e566fa6..34755a4109 100644
--- a/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpSeqBufferedReader.java
+++ b/flinkx-ftp/flinkx-ftp-reader/src/main/java/com/dtstack/flinkx/ftp/reader/FtpSeqBufferedReader.java
@@ -20,8 +20,13 @@
import com.dtstack.flinkx.ftp.IFtpHandler;
import com.dtstack.flinkx.ftp.FtpHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.util.Iterator;
/**
@@ -32,6 +37,8 @@
*/
public class FtpSeqBufferedReader {
+ private static Logger LOG = LoggerFactory.getLogger(FtpSeqBufferedReader.class);
+
private IFtpHandler ftpHandler;
private Iterator iter;
@@ -76,7 +83,8 @@ private void nextStream() throws IOException{
br = new BufferedReader(new InputStreamReader(in, charsetName));
for (int i = 0; i < fromLine; i++) {
- br.readLine();
+ String skipLine = br.readLine();
+ LOG.info("Skip line:{}", skipLine);
}
} else {
br = null;
diff --git a/flinkx-ftp/flinkx-ftp-reader/src/test/java/com/dtstack/flinkx/ftp/reader/FtpServerDemo.java b/flinkx-ftp/flinkx-ftp-reader/src/test/java/com/dtstack/flinkx/ftp/reader/FtpServerDemo.java
deleted file mode 100644
index ed9177c3b7..0000000000
--- a/flinkx-ftp/flinkx-ftp-reader/src/test/java/com/dtstack/flinkx/ftp/reader/FtpServerDemo.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package com.dtstack.flinkx.ftp.reader;
-
-import org.apache.commons.net.ftp.FTPClient;
-import org.apache.commons.net.ftp.FTPFile;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Created by softfly on 17/11/22.
- */
-public class FtpServerDemo {
-
- public static void main(String[] args) throws IOException {
- FTPClient ftp = new FTPClient();
- String username = "test";
- String password = "qbI#5pNd";
- ftp.connect("node02", 21);
- ftp.login(username, password);
- FTPFile[] ftpFiles = ftp.listFiles();
- for(FTPFile ftpFile : ftpFiles) {
- System.out.println(ftpFile.getName());
- }
-
-
- String[] xxx = ftp.listNames();
-
- InputStream is1 = ftp.retrieveFileStream("hyf/ttt");
- ftp.getReply();
- InputStream is2 = ftp.retrieveFileStream("xxx");
- ftp.remoteRetrieve("/hyf/ttt");
- ftp.getReply();
-
- ftp.changeWorkingDirectory("/hyf");
- System.out.println(ftp.printWorkingDirectory());
-
- ftp.logout();
-
- }
-
-}
diff --git a/flinkx-ftp/flinkx-ftp-reader/src/test/java/com/dtstack/flinkx/ftp/reader/SftpServerDemo.java b/flinkx-ftp/flinkx-ftp-reader/src/test/java/com/dtstack/flinkx/ftp/reader/SftpServerDemo.java
deleted file mode 100644
index 6defecb105..0000000000
--- a/flinkx-ftp/flinkx-ftp-reader/src/test/java/com/dtstack/flinkx/ftp/reader/SftpServerDemo.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package com.dtstack.flinkx.ftp.reader;
-
-import com.jcraft.jsch.*;
-
-import java.util.Properties;
-import java.util.Vector;
-
-/**
- * Created by softfly on 17/11/23.
- */
-public class SftpServerDemo {
-
- public static void main(String[] args) throws JSchException, SftpException {
- JSch jsch = new JSch();
-
- Session session = jsch.getSession("mysftp", "node02");
- session.setPassword("oh1986mygod");
- session.setPort(22);
- //session.setTimeout(10);
- Properties config = new Properties();
- config.put("StrictHostKeyChecking", "no");
-
- session.setConfig(config);
- session.connect();
-
- ChannelSftp channelSftp = (ChannelSftp) session.openChannel("sftp"); // 打开SFTP通道
- channelSftp.connect(); // 建立SFTP通道的连接
-
- Vector vector = channelSftp.ls("/");
-
- for(int i = 0; i < vector.size(); ++i) {
- ChannelSftp.LsEntry le = (ChannelSftp.LsEntry) vector.get(i);
- System.out.println(le.getFilename() );
- System.out.println(le.getLongname());
- }
-
-
- //session.disconnect();
-
- }
-
-}
diff --git a/flinkx-ftp/flinkx-ftp-writer/pom.xml b/flinkx-ftp/flinkx-ftp-writer/pom.xml
index 61d8ffaede..5c5b2bb6f7 100644
--- a/flinkx-ftp/flinkx-ftp-writer/pom.xml
+++ b/flinkx-ftp/flinkx-ftp-writer/pom.xml
@@ -72,6 +72,16 @@ under the License.
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpOutputFormat.java b/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpOutputFormat.java
index aeddb31bcb..4d66e893b1 100644
--- a/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpOutputFormat.java
+++ b/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpOutputFormat.java
@@ -19,22 +19,27 @@
package com.dtstack.flinkx.ftp.writer;
import com.dtstack.flinkx.exception.WriteRecordException;
-import com.dtstack.flinkx.ftp.*;
+import com.dtstack.flinkx.ftp.FtpConfig;
+import com.dtstack.flinkx.ftp.FtpHandlerFactory;
+import com.dtstack.flinkx.ftp.IFtpHandler;
import com.dtstack.flinkx.outputformat.BaseFileOutputFormat;
+import com.dtstack.flinkx.util.ExceptionUtil;
+import com.dtstack.flinkx.util.GsonUtil;
import com.dtstack.flinkx.util.StringUtil;
import com.dtstack.flinkx.util.SysUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.flink.types.Row;
+import java.io.BufferedWriter;
import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Arrays;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
-import static com.dtstack.flinkx.ftp.FtpConfigConstants.SFTP_PROTOCOL;
-
/**
* The OutputFormat Implementation which reads data from ftp servers.
*
@@ -54,21 +59,18 @@ public class FtpOutputFormat extends BaseFileOutputFormat {
private transient IFtpHandler ftpHandler;
- private transient OutputStream os;
+ private static final int FILE_NAME_PART_SIZE = 3;
private static final String DOT = ".";
private static final String FILE_SUFFIX = ".csv";
private static final String OVERWRITE_MODE = "overwrite";
+ private transient BufferedWriter writer;
@Override
protected void openSource() throws IOException {
- if(SFTP_PROTOCOL.equalsIgnoreCase(ftpConfig.getProtocol())) {
- ftpHandler = new SftpHandler();
- } else {
- ftpHandler = new FtpHandler();
- }
+ ftpHandler = FtpHandlerFactory.createFtpHandler(ftpConfig.getProtocol());
ftpHandler.loginFtpServer(ftpConfig);
}
@@ -106,7 +108,7 @@ public boolean test(String file) {
}
String[] splits = fileName.split("\\.");
- if (splits.length == 3) {
+ if (splits.length == FILE_NAME_PART_SIZE) {
return Integer.parseInt(splits[2]) <= fileIndex;
}
@@ -125,11 +127,16 @@ public boolean test(String file) {
protected void nextBlock(){
super.nextBlock();
- if (os != null){
+ if (writer != null){
return;
}
-
- os = ftpHandler.getOutputStream(tmpPath + SP + currentBlockFileName);
+ String path = tmpPath + SP + currentBlockFileName;
+ try {
+ writer = new BufferedWriter(new OutputStreamWriter(ftpHandler.getOutputStream(path), ftpConfig.getEncoding()));
+ } catch (UnsupportedEncodingException e) {
+ LOG.error("exception when create BufferedWriter, path = {}, e = {}", path, ExceptionUtil.getErrorMessage(e));
+ throw new RuntimeException(e);
+ }
blockIndex++;
}
@@ -156,29 +163,42 @@ public void moveTemporaryDataBlockFileToDirectory(){
@Override
public void writeSingleRecordToFile(Row row) throws WriteRecordException {
- if(os == null){
- nextBlock();
- }
-
- String line = StringUtil.row2string(row, columnTypes, ftpConfig.getFieldDelimiter(), columnNames);
try {
- byte[] bytes = line.getBytes(ftpConfig.getEncoding());
- this.os.write(bytes);
- this.os.write(NEWLINE);
- this.os.flush();
+ if(writer == null){
+ nextBlock();
+ }
+
+ String line = StringUtil.row2string(row, columnTypes, ftpConfig.getFieldDelimiter());
+ this.writer.write(line);
+ this.writer.write(NEWLINE);
if(restoreConfig.isRestore()){
lastRow = row;
rowsOfCurrentBlock++;
}
- } catch(Exception ex) {
- throw new WriteRecordException(ex.getMessage(), ex);
+ } catch(Exception e) {
+ LOG.error("error happened when write single record to file, row = {}, columnTypes = {}, e = {}", row, GsonUtil.GSON.toJson(columnTypes), ExceptionUtil.getErrorMessage(e));
+ throw new WriteRecordException(e.getMessage(), e);
}
}
@Override
- protected void createFinishedTag() throws IOException {
- LOG.info("Subtask [{}] finished, create dir {}", taskNumber, finishedPath);
+ protected void createFinishedTag() {
+ LOG.info("SubTask [{}] finished, create dir {}", taskNumber, finishedPath);
+ String path = outputFilePath + SP + FINISHED_SUBDIR;
+ if(taskNumber == 0){
+ ftpHandler.mkDirRecursive(path);
+ }
+ final int maxRetryTime = 15;
+ int i = 0;
+ try {
+ while(!(ftpHandler.isDirExist(path) || i > maxRetryTime)){
+ i++;
+ TimeUnit.MILLISECONDS.sleep(10);
+ }
+ }catch (Exception e){
+ LOG.error("exception when createFinishedTag, path = {}, e = {}", path, ExceptionUtil.getErrorMessage(e));
+ }
ftpHandler.mkDirRecursive(finishedPath);
}
@@ -238,7 +258,7 @@ protected void waitForAllTasksToFinish(){
if (i == maxRetryTime) {
ftpHandler.deleteAllFilesInDir(finishedPath, null);
- throw new RuntimeException("timeout when gathering finish tags for each subtasks");
+ throw new RuntimeException("timeout when gathering finish tags for each subTasks");
}
}
@@ -246,7 +266,7 @@ protected void waitForAllTasksToFinish(){
protected void coverageData(){
boolean cleanPath = restoreConfig.isRestore() && OVERWRITE_MODE.equalsIgnoreCase(ftpConfig.getWriteMode()) && !SP.equals(ftpConfig.getPath());
if(cleanPath){
- ftpHandler.deleteAllFilesInDir(ftpConfig.getPath(), Arrays.asList(tmpPath));
+ ftpHandler.deleteAllFilesInDir(ftpConfig.getPath(), Collections.singletonList(tmpPath));
}
}
@@ -286,15 +306,15 @@ protected void moveAllTemporaryDataFileToDirectory() throws IOException {
@Override
protected void closeSource() throws IOException {
- if (os != null){
- os.flush();
- os.close();
- os = null;
+ if (writer != null){
+ writer.flush();
+ writer.close();
+ writer = null;
}
}
@Override
- protected void clearTemporaryDataFiles() throws IOException {
+ protected void clearTemporaryDataFiles() {
ftpHandler.deleteAllFilesInDir(tmpPath, null);
LOG.info("Delete .data dir:{}", tmpPath);
@@ -307,6 +327,12 @@ public void flushDataInternal() throws IOException {
closeSource();
}
+ @Override
+ public void closeInternal() throws IOException {
+ closeSource();
+ super.closeInternal();
+ }
+
@Override
public float getDeviation() {
return 1.0F;
@@ -316,4 +342,9 @@ public float getDeviation() {
protected String getExtension() {
return ".csv";
}
+
+ @Override
+ protected void writeMultipleRecordsInternal() throws Exception {
+ notSupportBatchWrite("FtpWriter");
+ }
}
diff --git a/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpOutputFormatBuilder.java b/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpOutputFormatBuilder.java
index dee19b9e1c..dc7bcae4f9 100644
--- a/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpOutputFormatBuilder.java
+++ b/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpOutputFormatBuilder.java
@@ -19,9 +19,7 @@
package com.dtstack.flinkx.ftp.writer;
import com.dtstack.flinkx.ftp.FtpConfig;
-import com.dtstack.flinkx.ftp.FtpConfigConstants;
import com.dtstack.flinkx.outputformat.FileOutputFormatBuilder;
-import org.apache.commons.lang.StringUtils;
import java.util.List;
/**
@@ -53,7 +51,7 @@ public void setFtpConfig(FtpConfig ftpConfig){
@Override
protected void checkFormat() {
-
+ notSupportBatchWrite("FtpWriter");
}
}
diff --git a/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpWriter.java b/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpWriter.java
index edfb666f4a..9588e3a3c0 100644
--- a/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpWriter.java
+++ b/flinkx-ftp/flinkx-ftp-writer/src/main/java/com/dtstack/flinkx/ftp/writer/FtpWriter.java
@@ -30,7 +30,7 @@
import java.util.List;
import java.util.Map;
-import static com.dtstack.flinkx.ftp.FtpConfigConstants.*;
+import static com.dtstack.flinkx.ftp.FtpConfigConstants.DEFAULT_FIELD_DELIMITER;
/**
* The Writer Plugin of Ftp
diff --git a/flinkx-gbase/flinkx-gbase-reader/pom.xml b/flinkx-gbase/flinkx-gbase-reader/pom.xml
index 3082049696..9a7903f7fc 100644
--- a/flinkx-gbase/flinkx-gbase-reader/pom.xml
+++ b/flinkx-gbase/flinkx-gbase-reader/pom.xml
@@ -62,8 +62,12 @@
shade.gbasereader.io.netty
- com.google
- shade.gbasereader.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-gbase/flinkx-gbase-reader/src/main/java/com/dtstack/flinkx/gbase/format/GbaseInputFormat.java b/flinkx-gbase/flinkx-gbase-reader/src/main/java/com/dtstack/flinkx/gbase/format/GbaseInputFormat.java
index 7361843030..eb1b86a8a4 100644
--- a/flinkx-gbase/flinkx-gbase-reader/src/main/java/com/dtstack/flinkx/gbase/format/GbaseInputFormat.java
+++ b/flinkx-gbase/flinkx-gbase-reader/src/main/java/com/dtstack/flinkx/gbase/format/GbaseInputFormat.java
@@ -18,6 +18,7 @@
package com.dtstack.flinkx.gbase.format;
import com.dtstack.flinkx.rdb.inputformat.JdbcInputFormat;
+import com.dtstack.flinkx.rdb.inputformat.JdbcInputSplit;
import com.dtstack.flinkx.rdb.util.DbUtil;
import com.dtstack.flinkx.util.ClassUtil;
import org.apache.commons.lang3.StringUtils;
@@ -47,7 +48,9 @@ public void openInternal(InputSplit inputSplit) throws IOException {
String startLocation = incrementConfig.getStartLocation();
if (incrementConfig.isPolling()) {
- endLocationAccumulator.add(Long.parseLong(startLocation));
+ if (StringUtils.isNotEmpty(startLocation)) {
+ endLocationAccumulator.add(Long.parseLong(startLocation));
+ }
isTimestamp = "timestamp".equalsIgnoreCase(incrementConfig.getColumnType());
} else if ((incrementConfig.isIncrement() && incrementConfig.isUseMaxFunc())) {
getMaxValue(inputSplit);
@@ -61,6 +64,10 @@ public void openInternal(InputSplit inputSplit) throws IOException {
fetchSize = Integer.MIN_VALUE;
querySql = buildQuerySql(inputSplit);
+ JdbcInputSplit jdbcInputSplit = (JdbcInputSplit) inputSplit;
+ if (null != jdbcInputSplit.getStartLocation()) {
+ startLocation = jdbcInputSplit.getStartLocation();
+ }
executeQuery(startLocation);
columnCount = resultSet.getMetaData().getColumnCount();
diff --git a/flinkx-gbase/flinkx-gbase-writer/pom.xml b/flinkx-gbase/flinkx-gbase-writer/pom.xml
index da6f21b4a2..d28e0af276 100644
--- a/flinkx-gbase/flinkx-gbase-writer/pom.xml
+++ b/flinkx-gbase/flinkx-gbase-writer/pom.xml
@@ -62,8 +62,12 @@
shade.gbasewriter.io.netty
- com.google
- shade.gbasewriter.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-greenplum/flinkx-greenplum-core/pom.xml b/flinkx-greenplum/flinkx-greenplum-core/pom.xml
new file mode 100644
index 0000000000..3a79da783d
--- /dev/null
+++ b/flinkx-greenplum/flinkx-greenplum-core/pom.xml
@@ -0,0 +1,39 @@
+
+
+
+ flinkx-greenplum
+ com.dtstack.flinkx
+ 1.6
+
+ 4.0.0
+
+ flinkx-greenplum-core
+
+
+ com.dtstack.flinkx
+ flinkx-postgresql-core
+ 1.6
+
+
+ com.pivotal
+ greenplum-jdbc
+ 5.1.4
+
+
+ org.postgresql
+ postgresql
+ 42.2.2
+
+
+
+
+
+ HandChina RDC
+ HandChina RDC
+ http://nexus.saas.hand-china.com/content/repositories/rdc/
+
+
+
+
\ No newline at end of file
diff --git a/flinkx-greenplum/flinkx-greenplum-core/src/main/java/com/dtstack/flinkx/greenplum/GreenplumDatabaseMeta.java b/flinkx-greenplum/flinkx-greenplum-core/src/main/java/com/dtstack/flinkx/greenplum/GreenplumDatabaseMeta.java
new file mode 100644
index 0000000000..95dcb1b887
--- /dev/null
+++ b/flinkx-greenplum/flinkx-greenplum-core/src/main/java/com/dtstack/flinkx/greenplum/GreenplumDatabaseMeta.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dtstack.flinkx.greenplum;
+
+import com.dtstack.flinkx.enums.EDatabaseType;
+import com.dtstack.flinkx.postgresql.PostgresqlDatabaseMeta;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The class of Greenplum database prototype
+ *
+ * @Company: www.dtstack.com
+ * @author kunni@dtstack.com
+ */
+
+public class GreenplumDatabaseMeta extends PostgresqlDatabaseMeta {
+
+ @Override
+ public String getUpsertStatement(List column, String table, Map> updateKey) {
+ throw new UnsupportedOperationException("Greenplum not support update mode");
+ }
+
+ @Override
+ public EDatabaseType getDatabaseType() {
+ return EDatabaseType.Greenplum;
+ }
+
+ @Override
+ public String getDriverClass() {
+ return "com.pivotal.jdbc.GreenplumDriver";
+ }
+}
diff --git a/flinkx-greenplum/flinkx-greenplum-core/src/main/java/com/dtstack/flinkx/greenplum/GreenplumDatabaseMetaInsert.java b/flinkx-greenplum/flinkx-greenplum-core/src/main/java/com/dtstack/flinkx/greenplum/GreenplumDatabaseMetaInsert.java
new file mode 100644
index 0000000000..cb811c6ef0
--- /dev/null
+++ b/flinkx-greenplum/flinkx-greenplum-core/src/main/java/com/dtstack/flinkx/greenplum/GreenplumDatabaseMetaInsert.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dtstack.flinkx.greenplum;
+
+/**
+ * when use insert mode, use org.postgresql.Driver
+ *
+ * @Company: www.dtstack.com
+ * @author kunni@dtstack.com
+ */
+
+public class GreenplumDatabaseMetaInsert extends GreenplumDatabaseMeta{
+ @Override
+ public String getDriverClass() {
+ return "org.postgresql.Driver";
+ }
+}
diff --git a/flinkx-greenplum/flinkx-greenplum-reader/pom.xml b/flinkx-greenplum/flinkx-greenplum-reader/pom.xml
new file mode 100644
index 0000000000..0fb695ce58
--- /dev/null
+++ b/flinkx-greenplum/flinkx-greenplum-reader/pom.xml
@@ -0,0 +1,112 @@
+
+
+
+ flinkx-greenplum
+ com.dtstack.flinkx
+ 1.6
+
+ 4.0.0
+
+ flinkx-greenplum-reader
+
+
+
+ com.dtstack.flinkx
+ flinkx-greenplum-core
+ 1.6
+
+
+ com.dtstack.flinkx
+ flinkx-postgresql-reader
+ 1.6
+
+
+ com.dtstack.flinkx
+ flinkx-rdb-reader
+ 1.6
+ provided
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.1.0
+
+
+ package
+
+ shade
+
+
+ false
+
+
+ org.slf4j:slf4j-api
+ log4j:log4j
+ ch.qos.logback:*
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+ io.netty
+ shade.greenplumreader.io.netty
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
+
+
+
+
+
+
+ maven-antrun-plugin
+ 1.2
+
+
+ copy-resources
+
+ package
+
+ run
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/flinkx-greenplum/flinkx-greenplum-reader/src/main/java/com/dtstack/flinkx/greenplum/format/GreenplumInputFormat.java b/flinkx-greenplum/flinkx-greenplum-reader/src/main/java/com/dtstack/flinkx/greenplum/format/GreenplumInputFormat.java
new file mode 100644
index 0000000000..c1fba4eb0b
--- /dev/null
+++ b/flinkx-greenplum/flinkx-greenplum-reader/src/main/java/com/dtstack/flinkx/greenplum/format/GreenplumInputFormat.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dtstack.flinkx.greenplum.format;
+
+import com.dtstack.flinkx.postgresql.format.PostgresqlInputFormat;
+
+/**Simple inheritance of PostgresqlInputFormat
+ *
+ * @Company: www.dtstack.com
+ * @author kunni@dtstack.com
+ */
+
+public class GreenplumInputFormat extends PostgresqlInputFormat {
+}
diff --git a/flinkx-greenplum/flinkx-greenplum-reader/src/main/java/com/dtstack/flinkx/greenplum/reader/GreenplumReader.java b/flinkx-greenplum/flinkx-greenplum-reader/src/main/java/com/dtstack/flinkx/greenplum/reader/GreenplumReader.java
new file mode 100644
index 0000000000..09f547469f
--- /dev/null
+++ b/flinkx-greenplum/flinkx-greenplum-reader/src/main/java/com/dtstack/flinkx/greenplum/reader/GreenplumReader.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dtstack.flinkx.greenplum.reader;
+
+import com.dtstack.flinkx.config.DataTransferConfig;
+import com.dtstack.flinkx.greenplum.format.GreenplumInputFormat;
+import com.dtstack.flinkx.inputformat.BaseRichInputFormat;
+import com.dtstack.flinkx.postgresql.PostgresqlTypeConverter;
+import com.dtstack.flinkx.postgresql.reader.PostgresqlQuerySqlBuilder;
+import com.dtstack.flinkx.rdb.datareader.JdbcDataReader;
+import com.dtstack.flinkx.rdb.datareader.QuerySqlBuilder;
+import com.dtstack.flinkx.rdb.inputformat.JdbcInputFormatBuilder;
+import com.dtstack.flinkx.greenplum.GreenplumDatabaseMeta;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.types.Row;
+
+/**
+ * The reader plugin for Greenplum database
+ *
+ * @Company: www.dtstack.com
+ * @author kunni@dtstack.com
+ */
+
+public class GreenplumReader extends JdbcDataReader {
+ public GreenplumReader(DataTransferConfig config, StreamExecutionEnvironment env) {
+ super(config, env);
+ setDatabaseInterface(new GreenplumDatabaseMeta());
+ setTypeConverterInterface(new PostgresqlTypeConverter());
+ }
+
+ @Override
+ protected JdbcInputFormatBuilder getBuilder() {
+ return new JdbcInputFormatBuilder(new GreenplumInputFormat());
+ }
+
+ @Override
+ public DataStream readData() {
+ JdbcInputFormatBuilder builder = new JdbcInputFormatBuilder(new GreenplumInputFormat());
+ builder.setDriverName(databaseInterface.getDriverClass());
+ builder.setDbUrl(dbUrl);
+ builder.setUsername(username);
+ builder.setPassword(password);
+ builder.setBytes(bytes);
+ builder.setMonitorUrls(monitorUrls);
+ builder.setTable(table);
+ builder.setDatabaseInterface(databaseInterface);
+ builder.setTypeConverter(typeConverter);
+ builder.setMetaColumn(metaColumns);
+ builder.setFetchSize(fetchSize == 0 ? databaseInterface.getFetchSize() : fetchSize);
+ builder.setQueryTimeOut(queryTimeOut == 0 ? databaseInterface.getQueryTimeout() : queryTimeOut);
+ builder.setIncrementConfig(incrementConfig);
+ builder.setSplitKey(splitKey);
+ builder.setNumPartitions(numPartitions);
+ builder.setCustomSql(customSql);
+ builder.setRestoreConfig(restoreConfig);
+ builder.setHadoopConfig(hadoopConfig);
+ builder.setTestConfig(testConfig);
+
+ QuerySqlBuilder sqlBuilder = new PostgresqlQuerySqlBuilder(this);
+ builder.setQuery(sqlBuilder.buildSql());
+
+ BaseRichInputFormat format = builder.finish();
+ return createInput(format, (databaseInterface.getDatabaseType() + "reader").toLowerCase());
+ }
+
+}
diff --git a/flinkx-greenplum/flinkx-greenplum-writer/pom.xml b/flinkx-greenplum/flinkx-greenplum-writer/pom.xml
new file mode 100644
index 0000000000..db6738d9e5
--- /dev/null
+++ b/flinkx-greenplum/flinkx-greenplum-writer/pom.xml
@@ -0,0 +1,110 @@
+
+
+
+ flinkx-greenplum
+ com.dtstack.flinkx
+ 1.6
+
+ 4.0.0
+
+ flinkx-greenplum-writer
+
+
+ com.dtstack.flinkx
+ flinkx-rdb-writer
+ 1.6
+ provided
+
+
+ com.dtstack.flinkx
+ flinkx-greenplum-core
+ 1.6
+
+
+ com.dtstack.flinkx
+ flinkx-postgresql-writer
+ 1.6
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.1.0
+
+
+ package
+
+ shade
+
+
+ false
+
+
+ org.slf4j:slf4j-api
+ log4j:log4j
+ ch.qos.logback:*
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+ io.netty
+ shade.greenplumwriter.io.netty
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
+
+
+
+
+
+
+ maven-antrun-plugin
+ 1.2
+
+
+ copy-resources
+
+ package
+
+ run
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/flinkx-greenplum/flinkx-greenplum-writer/src/main/java/com/dtstack/flinkx/greenplum/format/GreenplumOutputFormat.java b/flinkx-greenplum/flinkx-greenplum-writer/src/main/java/com/dtstack/flinkx/greenplum/format/GreenplumOutputFormat.java
new file mode 100644
index 0000000000..a1d02138d2
--- /dev/null
+++ b/flinkx-greenplum/flinkx-greenplum-writer/src/main/java/com/dtstack/flinkx/greenplum/format/GreenplumOutputFormat.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dtstack.flinkx.greenplum.format;
+
+import com.dtstack.flinkx.postgresql.format.PostgresqlOutputFormat;
+
+/**Simple inheritance of PostgresqlOutputFormat
+ *
+ * Date: 2020/6/4
+ * Company: www.dtstack.com
+ * @author kunni@dtstack.com
+ */
+
+public class GreenplumOutputFormat extends PostgresqlOutputFormat {
+}
diff --git a/flinkx-greenplum/flinkx-greenplum-writer/src/main/java/com/dtstack/flinkx/greenplum/writer/GreenplumWriter.java b/flinkx-greenplum/flinkx-greenplum-writer/src/main/java/com/dtstack/flinkx/greenplum/writer/GreenplumWriter.java
new file mode 100644
index 0000000000..8d3267968b
--- /dev/null
+++ b/flinkx-greenplum/flinkx-greenplum-writer/src/main/java/com/dtstack/flinkx/greenplum/writer/GreenplumWriter.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dtstack.flinkx.greenplum.writer;
+
+import com.dtstack.flinkx.config.DataTransferConfig;
+import com.dtstack.flinkx.greenplum.format.GreenplumOutputFormat;
+import com.dtstack.flinkx.postgresql.PostgresqlTypeConverter;
+import com.dtstack.flinkx.rdb.datawriter.JdbcDataWriter;
+import com.dtstack.flinkx.rdb.outputformat.JdbcOutputFormatBuilder;
+import com.dtstack.flinkx.greenplum.GreenplumDatabaseMetaInsert;
+import org.apache.commons.lang.StringUtils;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.types.Row;
+
+/**
+ * The writer plugin for Greenplum database
+ *
+ * @Company: www.dtstack.com
+ * @author kunni@dtstack.com
+ */
+
+public class GreenplumWriter extends JdbcDataWriter {
+
+ public static final String INSERT_SQL_MODE_TYPE = "copy";
+ public static final String DATABASE_NAME = ";DatabaseName=";
+ public static final String JDBC_POSTGRESQL_PREFIX = "jdbc:postgresql";
+ public static final String JDBC_GREENPLUM_PREFIX = "jdbc:pivotal:greenplum";
+
+ public GreenplumWriter(DataTransferConfig config) {
+ super(config);
+ //统一固定为copy模式
+ insertSqlMode = INSERT_SQL_MODE_TYPE;
+ dbUrl = changeToPostgresqlUrl();
+ setDatabaseInterface(new GreenplumDatabaseMetaInsert());
+ setTypeConverterInterface(new PostgresqlTypeConverter());
+ }
+
+ String changeToPostgresqlUrl(){
+ dbUrl = StringUtils.replaceOnce(dbUrl, JDBC_GREENPLUM_PREFIX, JDBC_POSTGRESQL_PREFIX);
+ dbUrl = StringUtils.replaceOnce(dbUrl, DATABASE_NAME, "/");
+ return dbUrl;
+ }
+
+ @Override
+ public DataStreamSink writeData(DataStream dataSet) {
+ GreenplumOutputFormat greenplumOutputFormat = new GreenplumOutputFormat();
+ JdbcOutputFormatBuilder builder = new JdbcOutputFormatBuilder(greenplumOutputFormat);
+ builder.setDriverName(databaseInterface.getDriverClass());
+ builder.setDbUrl(dbUrl);
+ builder.setUsername(username);
+ builder.setPassword(password);
+ builder.setBatchInterval(batchSize);
+ builder.setMonitorUrls(monitorUrls);
+ builder.setPreSql(preSql);
+ builder.setPostSql(postSql);
+ builder.setErrors(errors);
+ builder.setErrorRatio(errorRatio);
+ builder.setDirtyPath(dirtyPath);
+ builder.setDirtyHadoopConfig(dirtyHadoopConfig);
+ builder.setSrcCols(srcCols);
+ builder.setDatabaseInterface(databaseInterface);
+ builder.setMode(mode);
+ builder.setTable(table);
+ builder.setColumn(column);
+ builder.setFullColumn(fullColumn);
+ builder.setUpdateKey(updateKey);
+ builder.setTypeConverter(typeConverter);
+ builder.setRestoreConfig(restoreConfig);
+ builder.setInsertSqlMode(insertSqlMode);
+
+ DataStreamSink dataStreamSink = createOutput(dataSet, builder.finish());
+ String sinkName = (databaseInterface.getDatabaseType() + "writer").toLowerCase();
+ dataStreamSink.name(sinkName);
+ return dataStreamSink;
+ }
+}
diff --git a/flinkx-greenplum/pom.xml b/flinkx-greenplum/pom.xml
new file mode 100644
index 0000000000..5c3fb85de9
--- /dev/null
+++ b/flinkx-greenplum/pom.xml
@@ -0,0 +1,36 @@
+
+
+
+ flinkx-all
+ com.dtstack.flinkx
+ 1.6
+
+ 4.0.0
+
+ flinkx-greenplum
+ pom
+
+ flinkx-greenplum-reader
+ flinkx-greenplum-core
+ flinkx-greenplum-writer
+
+
+
+
+ com.dtstack.flinkx
+ flinkx-core
+ 1.6
+ provided
+
+
+
+ com.dtstack.flinkx
+ flinkx-rdb-core
+ 1.6
+ provided
+
+
+
+
\ No newline at end of file
diff --git a/flinkx-hbase/flinkx-hbase-core/src/main/java/com/dtstack/flinkx/hbase/HbaseConfigConstants.java b/flinkx-hbase/flinkx-hbase-core/src/main/java/com/dtstack/flinkx/hbase/HbaseConfigConstants.java
index d83d83b3ee..d7f0d61ba6 100644
--- a/flinkx-hbase/flinkx-hbase-core/src/main/java/com/dtstack/flinkx/hbase/HbaseConfigConstants.java
+++ b/flinkx-hbase/flinkx-hbase-core/src/main/java/com/dtstack/flinkx/hbase/HbaseConfigConstants.java
@@ -44,7 +44,7 @@ public class HbaseConfigConstants {
public static final String DEFAULT_NULL_MODE = "skip";
- public static final long DEFAULT_WRITE_BUFFER_SIZE = 8 * 1024 * 1024;
+ public static final long DEFAULT_WRITE_BUFFER_SIZE = 8 * 1024 * 1024L;
public static final boolean DEFAULT_WAL_FLAG = false;
diff --git a/flinkx-hbase/flinkx-hbase-core/src/main/java/com/dtstack/flinkx/hbase/HbaseHelper.java b/flinkx-hbase/flinkx-hbase-core/src/main/java/com/dtstack/flinkx/hbase/HbaseHelper.java
index 0bce2c0c69..3c88f828f5 100644
--- a/flinkx-hbase/flinkx-hbase-core/src/main/java/com/dtstack/flinkx/hbase/HbaseHelper.java
+++ b/flinkx-hbase/flinkx-hbase-core/src/main/java/com/dtstack/flinkx/hbase/HbaseHelper.java
@@ -27,15 +27,17 @@
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.security.PrivilegedAction;
-import java.util.Arrays;
-import java.util.List;
import java.util.Map;
/**
@@ -51,21 +53,9 @@ public class HbaseHelper {
private final static String AUTHENTICATION_TYPE = "Kerberos";
private final static String KEY_HBASE_SECURITY_AUTHENTICATION = "hbase.security.authentication";
private final static String KEY_HBASE_SECURITY_AUTHORIZATION = "hbase.security.authorization";
- private final static String KEY_HBASE_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal";
- private final static String KEY_HBASE_MASTER_KEYTAB_FILE = "hbase.master.keytab.file";
- private final static String KEY_HBASE_REGIONSERVER_KEYTAB_FILE = "hbase.regionserver.keytab.file";
- private final static String KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal";
-
- private static List KEYS_KERBEROS_REQUIRED = Arrays.asList(
- KEY_HBASE_SECURITY_AUTHENTICATION,
- KEY_HBASE_MASTER_KERBEROS_PRINCIPAL,
- KEY_HBASE_MASTER_KEYTAB_FILE,
- KEY_HBASE_REGIONSERVER_KEYTAB_FILE,
- KEY_HBASE_REGIONSERVER_KERBEROS_PRINCIPAL
- );
public static org.apache.hadoop.hbase.client.Connection getHbaseConnection(Map hbaseConfigMap) {
- Validate.isTrue(hbaseConfigMap != null && hbaseConfigMap.size() !=0, "hbaseConfig不能为空Map结构!");
+ Validate.isTrue(MapUtils.isEmpty(hbaseConfigMap), "hbaseConfig不能为空Map结构!");
if(openKerberos(hbaseConfigMap)){
return getConnectionWithKerberos(hbaseConfigMap);
@@ -81,43 +71,43 @@ public static org.apache.hadoop.hbase.client.Connection getHbaseConnection(Map hbaseConfigMap){
- for (String key : KEYS_KERBEROS_REQUIRED) {
- if(StringUtils.isEmpty(MapUtils.getString(hbaseConfigMap, key))){
- throw new IllegalArgumentException(String.format("Must provide [%s] when authentication is Kerberos", key));
- }
+ try {
+ UserGroupInformation ugi = getUgi(hbaseConfigMap);
+ return ugi.doAs(new PrivilegedAction() {
+ @Override
+ public Connection run() {
+ try {
+ Configuration hConfiguration = getConfig(hbaseConfigMap);
+ return ConnectionFactory.createConnection(hConfiguration);
+ } catch (IOException e) {
+ LOG.error("Get connection fail with config:{}", hbaseConfigMap);
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ } catch (Exception e){
+ throw new RuntimeException("Login kerberos error", e);
}
+ }
+ public static UserGroupInformation getUgi(Map hbaseConfigMap) throws IOException{
String keytabFileName = KerberosUtil.getPrincipalFileName(hbaseConfigMap);
keytabFileName = KerberosUtil.loadFile(hbaseConfigMap, keytabFileName);
- String principal = KerberosUtil.findPrincipalFromKeytab(keytabFileName);
+ String principal = KerberosUtil.getPrincipal(hbaseConfigMap, keytabFileName);
KerberosUtil.loadKrb5Conf(hbaseConfigMap);
Configuration conf = FileSystemUtil.getConfiguration(hbaseConfigMap, null);
- UserGroupInformation ugi;
- try {
- ugi = KerberosUtil.loginAndReturnUgi(conf, principal, keytabFileName);
- } catch (Exception e){
- throw new RuntimeException("Login kerberos error", e);
- }
-
- return ugi.doAs(new PrivilegedAction() {
- @Override
- public Connection run() {
- try {
- Configuration hConfiguration = getConfig(hbaseConfigMap);
- return ConnectionFactory.createConnection(hConfiguration);
- } catch (IOException e) {
- LOG.error("Get connection fail with config:{}", hbaseConfigMap);
- throw new RuntimeException(e);
- }
- }
- });
+ return KerberosUtil.loginAndReturnUgi(conf, principal, keytabFileName);
}
public static Configuration getConfig(Map hbaseConfigMap){
Configuration hConfiguration = HBaseConfiguration.create();
+ if (MapUtils.isEmpty(hbaseConfigMap)) {
+ return hConfiguration;
+ }
+
for (Map.Entry entry : hbaseConfigMap.entrySet()) {
if(entry.getValue() != null && !(entry.getValue() instanceof Map)){
hConfiguration.set(entry.getKey(), entry.getValue().toString());
diff --git a/flinkx-hbase/flinkx-hbase-core/src/test/java/com/dtstack/flinkx/hbase/test/HbaseHelperTest.java b/flinkx-hbase/flinkx-hbase-core/src/test/java/com/dtstack/flinkx/hbase/test/HbaseHelperTest.java
deleted file mode 100644
index c1a67b5750..0000000000
--- a/flinkx-hbase/flinkx-hbase-core/src/test/java/com/dtstack/flinkx/hbase/test/HbaseHelperTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package com.dtstack.flinkx.hbase.test;
-
-import com.dtstack.flinkx.hbase.HbaseHelper;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.*;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * @author jiangbo
- * @date 2019/8/29
- */
-public class HbaseHelperTest {
-
- public static void main(String[] args) throws Exception{
- Map sftpConf = new HashMap<>();
- sftpConf.put("host", "172.16.10.79");
- sftpConf.put("port", "22");
- sftpConf.put("username", "root");
- sftpConf.put("password", "abc123");
-
- Map hbaseConfig = new HashMap<>();
- hbaseConfig.put("hbase.security.authorization", "true");
- hbaseConfig.put("hbase.security.authentication", "kerberos");
- hbaseConfig.put("hbase.master.kerberos.principal", "hbase/cdh01@HADOOP.COM");
- hbaseConfig.put("hbase.master.keytab.file", "D:\\cdh_cluster\\cdh2\\hbase.keytab");
- hbaseConfig.put("hbase.regionserver.kerberos.principal", "hbase/cdh01@HADOOP.COM");
- hbaseConfig.put("hbase.regionserver.keytab.file", "D:\\cdh_cluster\\cdh2\\hbase.keytab");
- hbaseConfig.put("java.security.krb5.conf", "D:\\cdh_cluster\\cdh2\\krb5.conf");
- hbaseConfig.put("useLocalFile", "true");
-// hbaseConfig.put("sftpConf", sftpConf);
-// hbaseConfig.put("remoteDir", "/home/sftp/keytab/jiangbo");
-
-// hbaseConfig.put("hbase.zookeeper.quorum", "cdh01:2181,cdh02:2181,cdh03:2181");
- hbaseConfig.put("hbase.zookeeper.quorum", "172.16.10.201:2181");
- hbaseConfig.put("hbase.rpc.timeout", "60000");
- hbaseConfig.put("ipc.socket.timeout", "20000");
- hbaseConfig.put("hbase.client.retries.number", "3");
- hbaseConfig.put("hbase.client.pause", "100");
- hbaseConfig.put("zookeeper.recovery.retry", "3");
-
- Connection connection = HbaseHelper.getHbaseConnection(hbaseConfig);
- Table table = connection.getTable(TableName.valueOf("tb1"));
-
- ResultScanner rs = table.getScanner(new Scan());
- Result result = rs.next();
- if(result != null){
- System.out.println(result.getRow());
- }
-
- HbaseHelper.getRegionLocator(connection, "tb1");
-
- connection.close();
- }
-}
diff --git a/flinkx-hbase/flinkx-hbase-reader/pom.xml b/flinkx-hbase/flinkx-hbase-reader/pom.xml
index f77a9613ca..5cdc499262 100644
--- a/flinkx-hbase/flinkx-hbase-reader/pom.xml
+++ b/flinkx-hbase/flinkx-hbase-reader/pom.xml
@@ -65,6 +65,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-hbase/flinkx-hbase-reader/src/main/java/com/dtstack/flinkx/hbase/reader/HbaseInputFormat.java b/flinkx-hbase/flinkx-hbase-reader/src/main/java/com/dtstack/flinkx/hbase/reader/HbaseInputFormat.java
index 495a89c4e6..f17383ca87 100644
--- a/flinkx-hbase/flinkx-hbase-reader/src/main/java/com/dtstack/flinkx/hbase/reader/HbaseInputFormat.java
+++ b/flinkx-hbase/flinkx-hbase-reader/src/main/java/com/dtstack/flinkx/hbase/reader/HbaseInputFormat.java
@@ -29,16 +29,24 @@
import org.apache.flink.types.Row;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.security.UserGroupInformation;
/**
@@ -87,7 +95,17 @@ public void openInputFormat() throws IOException {
@Override
public InputSplit[] createInputSplitsInternal(int minNumSplits) throws IOException {
try (Connection connection = HbaseHelper.getHbaseConnection(hbaseConfig)) {
- return split(connection, tableName, startRowkey, endRowkey, isBinaryRowkey);
+ if(HbaseHelper.openKerberos(hbaseConfig)) {
+ UserGroupInformation ugi = HbaseHelper.getUgi(hbaseConfig);
+ return ugi.doAs(new PrivilegedAction() {
+ @Override
+ public HbaseInputSplit[] run() {
+ return split(connection, tableName, startRowkey, endRowkey, isBinaryRowkey);
+ }
+ });
+ } else {
+ return split(connection, tableName, startRowkey, endRowkey, isBinaryRowkey);
+ }
}
}
@@ -132,9 +150,10 @@ private List doSplit(byte[] startRowkeyByte,
// 当前的region为最后一个region
// 如果最后一个region的start Key大于用户指定的userEndKey,则最后一个region，应该不包含在内
// 注意如果用户指定userEndKey为"",则此判断应该不成立。userEndKey为""表示取得最大的region
- if (Bytes.compareTo(regionEndKey, HConstants.EMPTY_BYTE_ARRAY) == 0
+ boolean isSkip = Bytes.compareTo(regionEndKey, HConstants.EMPTY_BYTE_ARRAY) == 0
&& (endRowkeyByte.length != 0 && (Bytes.compareTo(
- regionStartKey, endRowkeyByte) > 0))) {
+ regionStartKey, endRowkeyByte) > 0));
+ if (isSkip) {
continue;
}
diff --git a/flinkx-hbase/flinkx-hbase-reader/src/main/java/com/dtstack/flinkx/hbase/reader/HbaseReader.java b/flinkx-hbase/flinkx-hbase-reader/src/main/java/com/dtstack/flinkx/hbase/reader/HbaseReader.java
index 04c5150990..b1d4186132 100644
--- a/flinkx-hbase/flinkx-hbase-reader/src/main/java/com/dtstack/flinkx/hbase/reader/HbaseReader.java
+++ b/flinkx-hbase/flinkx-hbase-reader/src/main/java/com/dtstack/flinkx/hbase/reader/HbaseReader.java
@@ -26,6 +26,8 @@
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.types.Row;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
@@ -39,6 +41,8 @@
*/
public class HbaseReader extends BaseDataReader {
+ private static Logger LOG = LoggerFactory.getLogger(HbaseReader.class);
+
protected List columnName;
protected List columnType;
protected List columnValue;
@@ -82,7 +86,8 @@ public HbaseReader(DataTransferConfig config, StreamExecutionEnvironment env) {
columnValue.add((String) sm.get("value"));
columnFormat.add((String) sm.get("format"));
}
- System.out.println("init column finished");
+
+ LOG.info("init column finished");
} else{
throw new IllegalArgumentException("column argument error");
}
diff --git a/flinkx-hbase/flinkx-hbase-reader/src/test/java/com/dtstack/flinkx/hbase/reader/HbaseSplitDemo.java b/flinkx-hbase/flinkx-hbase-reader/src/test/java/com/dtstack/flinkx/hbase/reader/HbaseSplitDemo.java
deleted file mode 100644
index 47f153621b..0000000000
--- a/flinkx-hbase/flinkx-hbase-reader/src/test/java/com/dtstack/flinkx/hbase/reader/HbaseSplitDemo.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package com.dtstack.flinkx.hbase.reader;
-
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.RegionLocator;
-import java.io.IOException;
-
-/**
- * Created by softfly on 17/7/25.
- */
-public class HbaseSplitDemo {
-
- private static void split() {
-
- }
-
- public static void main(String[] args) throws IOException {
-
- org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
- conf.set("hbase.zookeeper.property.clientPort", "2181");
- conf.set("hbase.zookeeper.quorum", "172.16.1.151" );
- conf.set("zookeeper.znode.parent", "/hbase2");
-
- Connection conn = ConnectionFactory.createConnection(conf);
- //Table table = conn.getTable(TableName.valueOf("tb2"));
-
- RegionLocator regionLocator = conn.getRegionLocator(TableName.valueOf("tb2"));
- regionLocator.getStartEndKeys();
-
- }
-
-}
diff --git a/flinkx-hbase/flinkx-hbase-writer/pom.xml b/flinkx-hbase/flinkx-hbase-writer/pom.xml
index d406061b54..d5dd35eba4 100644
--- a/flinkx-hbase/flinkx-hbase-writer/pom.xml
+++ b/flinkx-hbase/flinkx-hbase-writer/pom.xml
@@ -54,6 +54,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-hbase/flinkx-hbase-writer/src/main/java/com/dtstack/flinkx/hbase/writer/HbaseOutputFormat.java b/flinkx-hbase/flinkx-hbase-writer/src/main/java/com/dtstack/flinkx/hbase/writer/HbaseOutputFormat.java
index a3e75a9caa..d4db300d08 100644
--- a/flinkx-hbase/flinkx-hbase-writer/src/main/java/com/dtstack/flinkx/hbase/writer/HbaseOutputFormat.java
+++ b/flinkx-hbase/flinkx-hbase-writer/src/main/java/com/dtstack/flinkx/hbase/writer/HbaseOutputFormat.java
@@ -18,6 +18,7 @@
package com.dtstack.flinkx.hbase.writer;
+import com.dtstack.flinkx.constants.ConstantValue;
import com.dtstack.flinkx.enums.ColumnType;
import com.dtstack.flinkx.exception.WriteRecordException;
import com.dtstack.flinkx.hbase.HbaseHelper;
@@ -27,6 +28,7 @@
import com.dtstack.flinkx.util.DateUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.types.Row;
@@ -34,10 +36,12 @@
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedAction;
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -93,8 +97,40 @@ public class HbaseOutputFormat extends BaseRichOutputFormat {
private transient ThreadLocal timeMillisecondFormatThreadLocal;
+ private boolean openKerberos = false;
+
@Override
public void configure(Configuration parameters) {
+ }
+
+ @Override
+ public void openInternal(int taskNumber, int numTasks) throws IOException {
+ openKerberos = HbaseHelper.openKerberos(hbaseConfig);
+ if (openKerberos) {
+ sleepRandomTime();
+
+ UserGroupInformation ugi = HbaseHelper.getUgi(hbaseConfig);
+ ugi.doAs(new PrivilegedAction
-
- junit
- junit
- 4.12
- test
-
\ No newline at end of file
diff --git a/flinkx-kudu/flinkx-kudu-core/src/main/java/com/dtstack/flinkx/kudu/core/KuduUtil.java b/flinkx-kudu/flinkx-kudu-core/src/main/java/com/dtstack/flinkx/kudu/core/KuduUtil.java
index 928580a166..4eb6ec865a 100644
--- a/flinkx-kudu/flinkx-kudu-core/src/main/java/com/dtstack/flinkx/kudu/core/KuduUtil.java
+++ b/flinkx-kudu/flinkx-kudu-core/src/main/java/com/dtstack/flinkx/kudu/core/KuduUtil.java
@@ -26,13 +26,22 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kudu.ColumnSchema;
import org.apache.kudu.Type;
-import org.apache.kudu.client.*;
+import org.apache.kudu.client.AsyncKuduClient;
+import org.apache.kudu.client.AsyncKuduScanner;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduPredicate;
+import org.apache.kudu.client.KuduScanToken;
+import org.apache.kudu.client.KuduTable;
import java.io.IOException;
import java.math.BigDecimal;
import java.security.PrivilegedExceptionAction;
import java.sql.Timestamp;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
diff --git a/flinkx-kudu/flinkx-kudu-reader/pom.xml b/flinkx-kudu/flinkx-kudu-reader/pom.xml
index c8a99fea22..f9a1580d71 100644
--- a/flinkx-kudu/flinkx-kudu-reader/pom.xml
+++ b/flinkx-kudu/flinkx-kudu-reader/pom.xml
@@ -49,8 +49,12 @@
shade.kudureader.io.netty
- com.google
- shade.kudureader.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduInputFormat.java b/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduInputFormat.java
index f0ddce2857..6be5434a52 100644
--- a/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduInputFormat.java
+++ b/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduInputFormat.java
@@ -26,7 +26,11 @@
import org.apache.flink.core.io.InputSplit;
import org.apache.flink.types.Row;
import org.apache.kudu.Type;
-import org.apache.kudu.client.*;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduScanToken;
+import org.apache.kudu.client.KuduScanner;
+import org.apache.kudu.client.RowResult;
+import org.apache.kudu.client.RowResultIterator;
import java.io.IOException;
import java.util.List;
diff --git a/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduInputFormatBuilder.java b/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduInputFormatBuilder.java
index 48d0e43fc5..4959731c76 100644
--- a/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduInputFormatBuilder.java
+++ b/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduInputFormatBuilder.java
@@ -19,6 +19,7 @@
package com.dtstack.flinkx.kudu.reader;
+import com.dtstack.flinkx.constants.ConstantValue;
import com.dtstack.flinkx.inputformat.BaseRichInputFormatBuilder;
import com.dtstack.flinkx.kudu.core.KuduConfig;
import com.dtstack.flinkx.reader.MetaColumn;
@@ -50,5 +51,9 @@ protected void checkFormat() {
if (format.columns == null || format.columns.size() == 0){
throw new IllegalArgumentException("columns can not be empty");
}
+
+ if (format.kuduConfig.getBatchSizeBytes() > ConstantValue.STORE_SIZE_G) {
+ throw new IllegalArgumentException("批量读取字节数必须小于[1G]");
+ }
}
}
diff --git a/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduReader.java b/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduReader.java
index 7b09c9048d..604b920a6e 100644
--- a/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduReader.java
+++ b/flinkx-kudu/flinkx-kudu-reader/src/main/java/com/dtstack/flinkx/kudu/reader/KuduReader.java
@@ -32,7 +32,19 @@
import java.util.List;
-import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.*;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_ADMIN_OPERATION_TIMEOUT;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_AUTHENTICATION;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_BATCH_SIZE_BYTES;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_BOSS_COUNT;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_FILTER;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_KEYTABFILE;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_MASTER_ADDRESSES;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_OPERATION_TIMEOUT;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_PRINCIPAL;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_QUERY_TIMEOUT;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_READ_MODE;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_TABLE;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_WORKER_COUNT;
/**
* @author jiangbo
diff --git a/flinkx-kudu/flinkx-kudu-writer/pom.xml b/flinkx-kudu/flinkx-kudu-writer/pom.xml
index 742fae63ab..256505e5ab 100644
--- a/flinkx-kudu/flinkx-kudu-writer/pom.xml
+++ b/flinkx-kudu/flinkx-kudu-writer/pom.xml
@@ -49,8 +49,12 @@
shade.kuduwriter.io.netty
- com.google
- shade.kuduwriter.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-kudu/flinkx-kudu-writer/src/main/java/com/dtstack/flinkx/kudu/writer/KuduOutputFormat.java b/flinkx-kudu/flinkx-kudu-writer/src/main/java/com/dtstack/flinkx/kudu/writer/KuduOutputFormat.java
index df7a521186..69f346dcba 100644
--- a/flinkx-kudu/flinkx-kudu-writer/src/main/java/com/dtstack/flinkx/kudu/writer/KuduOutputFormat.java
+++ b/flinkx-kudu/flinkx-kudu-writer/src/main/java/com/dtstack/flinkx/kudu/writer/KuduOutputFormat.java
@@ -28,7 +28,12 @@
import com.dtstack.flinkx.util.ExceptionUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.types.Row;
-import org.apache.kudu.client.*;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduException;
+import org.apache.kudu.client.KuduSession;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.Operation;
+import org.apache.kudu.client.SessionConfiguration;
import java.io.IOException;
import java.util.List;
diff --git a/flinkx-kudu/flinkx-kudu-writer/src/main/java/com/dtstack/flinkx/kudu/writer/KuduWriter.java b/flinkx-kudu/flinkx-kudu-writer/src/main/java/com/dtstack/flinkx/kudu/writer/KuduWriter.java
index 265e5610c2..2cb5532262 100644
--- a/flinkx-kudu/flinkx-kudu-writer/src/main/java/com/dtstack/flinkx/kudu/writer/KuduWriter.java
+++ b/flinkx-kudu/flinkx-kudu-writer/src/main/java/com/dtstack/flinkx/kudu/writer/KuduWriter.java
@@ -32,7 +32,16 @@
import java.util.List;
-import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.*;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_ADMIN_OPERATION_TIMEOUT;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_AUTHENTICATION;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_BOSS_COUNT;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_FLUSH_MODE;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_KEYTABFILE;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_MASTER_ADDRESSES;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_OPERATION_TIMEOUT;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_PRINCIPAL;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_TABLE;
+import static com.dtstack.flinkx.kudu.core.KuduConfigKeys.KEY_WORKER_COUNT;
/**
* @author jiangbo
diff --git a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/ClassLoaderType.java b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/ClassLoaderType.java
index 540db38056..d287c44a70 100644
--- a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/ClassLoaderType.java
+++ b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/ClassLoaderType.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.launcher;
/**
diff --git a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/ClusterClientFactory.java b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/ClusterClientFactory.java
index 1b823c77eb..c5318c00eb 100644
--- a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/ClusterClientFactory.java
+++ b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/ClusterClientFactory.java
@@ -40,8 +40,6 @@
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.InetSocketAddress;
@@ -97,7 +95,7 @@ public static ClusterClient createYarnClient(Options launcherOptions) {
ApplicationId applicationId;
if (StringUtils.isEmpty(launcherOptions.getAppId())) {
- applicationId = getAppIdFromYarn(yarnClient);
+ applicationId = getAppIdFromYarn(yarnClient, launcherOptions);
if(applicationId != null && StringUtils.isEmpty(applicationId.toString())) {
throw new RuntimeException("No flink session found on yarn cluster.");
}
@@ -156,7 +154,7 @@ private static AbstractYarnClusterDescriptor getClusterDescriptor(Options launch
return yarnClusterDescriptor;
}
- private static ApplicationId getAppIdFromYarn(YarnClient yarnClient) throws Exception{
+ private static ApplicationId getAppIdFromYarn(YarnClient yarnClient, Options launcherOptions) throws Exception{
Set set = new HashSet<>();
set.add("Apache Flink");
EnumSet enumSet = EnumSet.noneOf(YarnApplicationState.class);
@@ -175,6 +173,10 @@ private static ApplicationId getAppIdFromYarn(YarnClient yarnClient) throws Exce
continue;
}
+ if(!report.getQueue().equals(launcherOptions.getQueue())) {
+ continue;
+ }
+
int thisMemory = report.getApplicationResourceUsageReport().getNeededResources().getMemory();
int thisCores = report.getApplicationResourceUsageReport().getNeededResources().getVirtualCores();
diff --git a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/Launcher.java b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/Launcher.java
index 91a4bd1409..c5c19fc9f4 100644
--- a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/Launcher.java
+++ b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/Launcher.java
@@ -25,6 +25,7 @@
import com.dtstack.flinkx.launcher.perjob.PerJobSubmitter;
import com.dtstack.flinkx.options.OptionParser;
import com.dtstack.flinkx.options.Options;
+import com.dtstack.flinkx.util.JsonModifyUtil;
import com.dtstack.flinkx.util.SysUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.flink.client.program.ClusterClient;
@@ -41,6 +42,7 @@
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
/**
@@ -94,8 +96,25 @@ public static void main(String[] args) throws Exception {
String mode = launcherOptions.getMode();
List argList = optionParser.getProgramExeArgList();
+
+ // 将argList转化为HashMap，方便通过参数名称来获取参数值
+ HashMap temp = new HashMap<>(16);
+ for (int i = 0; i < argList.size(); i += 2) {
+ temp.put(argList.get(i), argList.get(i + 1));
+ }
+ // 对json中的值进行修改
+ HashMap parameter = JsonModifyUtil.CommandTransform(temp.get("-p"));
+ temp.put("-job", JsonModifyUtil.JsonValueReplace(temp.get("-job"), parameter));
+
+ // 清空list，填充修改后的参数值
+ argList.clear();
+ for (int i = 0; i < temp.size(); i++) {
+ argList.add(temp.keySet().toArray()[i].toString());
+ argList.add(temp.values().toArray()[i].toString());
+ }
+
if(mode.equals(ClusterMode.local.name())) {
- String[] localArgs = argList.toArray(new String[argList.size()]);
+ String[] localArgs = argList.toArray(new String[0]);
com.dtstack.flinkx.Main.main(localArgs);
} else {
String pluginRoot = launcherOptions.getPluginRoot();
@@ -237,9 +256,8 @@ public boolean accept(File dir, String name) {
}
private static String readJob(String job) {
- try {
- File file = new File(job);
- FileInputStream in = new FileInputStream(file);
+ File file = new File(job);
+ try (FileInputStream in = new FileInputStream(file)) {
byte[] fileContent = new byte[(int) file.length()];
in.read(fileContent);
in.close();
diff --git a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/YarnConfLoader.java b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/YarnConfLoader.java
index 6707c8f138..b88eb078b8 100644
--- a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/YarnConfLoader.java
+++ b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/YarnConfLoader.java
@@ -18,6 +18,7 @@
package com.dtstack.flinkx.launcher;
+import com.dtstack.flinkx.constants.ConstantValue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -42,7 +43,7 @@ public static YarnConfiguration getYarnConf(String yarnConfDir) {
if(dir.exists() && dir.isDirectory()) {
File[] xmlFileList = new File(yarnConfDir).listFiles((dir1, name) -> {
- if(name.endsWith(".xml")){
+ if(name.endsWith(ConstantValue.FILE_SUFFIX_XML)){
return true;
}
return false;
diff --git a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/perjob/FlinkPerJobResourceUtil.java b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/perjob/FlinkPerJobResourceUtil.java
index 1d64181e81..508098067d 100644
--- a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/perjob/FlinkPerJobResourceUtil.java
+++ b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/perjob/FlinkPerJobResourceUtil.java
@@ -52,10 +52,10 @@ public static ClusterSpecification createClusterSpecification(Properties conProp
int slotsPerTaskManager = 1;
if(conProp != null){
- if(conProp.contains(JOBMANAGER_MEMORY_MB)){
+ if(conProp.containsKey(JOBMANAGER_MEMORY_MB)){
jobmanagerMemoryMb = Math.max(MIN_JM_MEMORY, ValueUtil.getInt(conProp.getProperty(JOBMANAGER_MEMORY_MB)));
}
- if(conProp.contains(TASKMANAGER_MEMORY_MB)){
+ if(conProp.containsKey(TASKMANAGER_MEMORY_MB)){
taskmanagerMemoryMb = Math.max(MIN_JM_MEMORY, ValueUtil.getInt(conProp.getProperty(TASKMANAGER_MEMORY_MB)));
}
diff --git a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/perjob/PerJobSubmitter.java b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/perjob/PerJobSubmitter.java
index d9025e2ed9..81c6387395 100644
--- a/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/perjob/PerJobSubmitter.java
+++ b/flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/perjob/PerJobSubmitter.java
@@ -21,7 +21,7 @@
import com.dtstack.flinkx.launcher.ClassLoaderType;
import com.dtstack.flinkx.launcher.YarnConfLoader;
import com.dtstack.flinkx.options.Options;
-import com.dtstack.flinkx.util.MapUtil;
+import com.dtstack.flinkx.util.GsonUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.flink.client.deployment.ClusterSpecification;
import org.apache.flink.client.program.ClusterClient;
@@ -64,7 +64,7 @@ public static String submit(Options options, File jarFile, String[] programArgs)
Configuration flinkConfig = StringUtils.isEmpty(options.getFlinkconf()) ? new Configuration() : GlobalConfiguration.loadConfiguration(options.getFlinkconf());
flinkConfig.setString("classloader.resolve-order", "child-first");
- Properties conProp = MapUtil.jsonStrToObject(options.getConfProp(), Properties.class);
+ Properties conProp = GsonUtil.GSON.fromJson(options.getConfProp(), Properties.class);
ClusterSpecification clusterSpecification = FlinkPerJobResourceUtil.createClusterSpecification(conProp);
clusterSpecification.setCreateProgramDelay(true);
clusterSpecification.setConfiguration(flinkConfig);
diff --git a/flinkx-launcher/src/test/java/bigdata/FlinkConfigTest.java b/flinkx-launcher/src/test/java/bigdata/FlinkConfigTest.java
deleted file mode 100644
index b974511769..0000000000
--- a/flinkx-launcher/src/test/java/bigdata/FlinkConfigTest.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package bigdata;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.configuration.JobManagerOptions;
-
-import java.io.File;
-
-/**
- * Created by softfly on 18/4/24.
- */
-public class FlinkConfigTest {
- public static void main(String[] args) {
- //Configuration config = GlobalConfiguration.loadConfiguration("/hadoop/flink-1.4.0/conf");
- //System.out.println(config.getString(JobManagerOptions.ADDRESS));
- String msg = "xxx" + File.separator;
- System.out.println(msg);
- }
-}
diff --git a/flinkx-mongodb/flinkx-mongodb-core/src/main/java/com/dtstack/flinkx/mongodb/MongodbClientUtil.java b/flinkx-mongodb/flinkx-mongodb-core/src/main/java/com/dtstack/flinkx/mongodb/MongodbClientUtil.java
index 26438d51fe..b5d78ffecc 100644
--- a/flinkx-mongodb/flinkx-mongodb-core/src/main/java/com/dtstack/flinkx/mongodb/MongodbClientUtil.java
+++ b/flinkx-mongodb/flinkx-mongodb-core/src/main/java/com/dtstack/flinkx/mongodb/MongodbClientUtil.java
@@ -19,7 +19,13 @@
package com.dtstack.flinkx.mongodb;
-import com.mongodb.*;
+import com.mongodb.AuthenticationMechanism;
+import com.mongodb.MongoClient;
+import com.mongodb.MongoClientOptions;
+import com.mongodb.MongoClientURI;
+import com.mongodb.MongoCredential;
+import com.mongodb.ServerAddress;
+import com.mongodb.WriteConcern;
import com.mongodb.client.MongoCursor;
import org.apache.commons.lang.StringUtils;
import org.bson.Document;
diff --git a/flinkx-mongodb/flinkx-mongodb-core/src/main/java/com/dtstack/flinkx/mongodb/MongodbUtil.java b/flinkx-mongodb/flinkx-mongodb-core/src/main/java/com/dtstack/flinkx/mongodb/MongodbUtil.java
index 425dc21aa4..c419a2672e 100644
--- a/flinkx-mongodb/flinkx-mongodb-core/src/main/java/com/dtstack/flinkx/mongodb/MongodbUtil.java
+++ b/flinkx-mongodb/flinkx-mongodb-core/src/main/java/com/dtstack/flinkx/mongodb/MongodbUtil.java
@@ -29,9 +29,11 @@
import java.math.BigDecimal;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
-import static com.dtstack.flinkx.enums.ColumnType.*;
+import static com.dtstack.flinkx.enums.ColumnType.getType;
/**
* Utilities for mongodb database connection and data format conversion
diff --git a/flinkx-mongodb/flinkx-mongodb-reader/pom.xml b/flinkx-mongodb/flinkx-mongodb-reader/pom.xml
index 09814f6c3f..a9caa60251 100644
--- a/flinkx-mongodb/flinkx-mongodb-reader/pom.xml
+++ b/flinkx-mongodb/flinkx-mongodb-reader/pom.xml
@@ -56,8 +56,12 @@
shade.mongodbreader.io.netty
- com.google
- shade.mongodbreader.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-mongodb/flinkx-mongodb-reader/src/main/java/com/dtstack/flinkx/mongodb/reader/MongodbInputFormat.java b/flinkx-mongodb/flinkx-mongodb-reader/src/main/java/com/dtstack/flinkx/mongodb/reader/MongodbInputFormat.java
index 3f3ec2d863..ac8e65d705 100644
--- a/flinkx-mongodb/flinkx-mongodb-reader/src/main/java/com/dtstack/flinkx/mongodb/reader/MongodbInputFormat.java
+++ b/flinkx-mongodb/flinkx-mongodb-reader/src/main/java/com/dtstack/flinkx/mongodb/reader/MongodbInputFormat.java
@@ -23,6 +23,7 @@
import com.dtstack.flinkx.mongodb.MongodbClientUtil;
import com.dtstack.flinkx.mongodb.MongodbConfig;
import com.dtstack.flinkx.reader.MetaColumn;
+import com.dtstack.flinkx.util.ExceptionUtil;
import com.dtstack.flinkx.util.StringUtil;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient;
@@ -37,7 +38,8 @@
import org.bson.conversions.Bson;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
/**
* Read plugin for reading static data
@@ -149,6 +151,9 @@ public InputSplit[] createInputSplitsInternal(int minNumSplits) throws IOExcepti
if(size * minNumSplits < docNum){
splits.add(new MongodbInputSplit((int)(size * minNumSplits), (int)(docNum - size * minNumSplits)));
}
+ } catch (Exception e){
+ LOG.error("error to create inputSplits, e = {}", ExceptionUtil.getErrorMessage(e));
+ throw e;
} finally {
MongodbClientUtil.close(client, null);
}
diff --git a/flinkx-mongodb/flinkx-mongodb-reader/src/main/java/com/dtstack/flinkx/mongodb/reader/MongodbInputFormatBuilder.java b/flinkx-mongodb/flinkx-mongodb-reader/src/main/java/com/dtstack/flinkx/mongodb/reader/MongodbInputFormatBuilder.java
index 918fb6b8f3..e1661838c2 100644
--- a/flinkx-mongodb/flinkx-mongodb-reader/src/main/java/com/dtstack/flinkx/mongodb/reader/MongodbInputFormatBuilder.java
+++ b/flinkx-mongodb/flinkx-mongodb-reader/src/main/java/com/dtstack/flinkx/mongodb/reader/MongodbInputFormatBuilder.java
@@ -18,6 +18,7 @@
package com.dtstack.flinkx.mongodb.reader;
+import com.dtstack.flinkx.constants.ConstantValue;
import com.dtstack.flinkx.inputformat.BaseRichInputFormatBuilder;
import com.dtstack.flinkx.mongodb.MongodbConfig;
import com.dtstack.flinkx.reader.MetaColumn;
@@ -55,5 +56,9 @@ protected void checkFormat() {
if (format.getRestoreConfig() != null && format.getRestoreConfig().isRestore()){
throw new UnsupportedOperationException("This plugin not support restore from failed state");
}
+
+ if (format.mongodbConfig.getFetchSize() > ConstantValue.MAX_BATCH_SIZE) {
+ throw new IllegalArgumentException("批量读取条数必须小于[200000]条");
+ }
}
}
diff --git a/flinkx-mongodb/flinkx-mongodb-writer/pom.xml b/flinkx-mongodb/flinkx-mongodb-writer/pom.xml
index 1d81ef36ef..a411920d71 100644
--- a/flinkx-mongodb/flinkx-mongodb-writer/pom.xml
+++ b/flinkx-mongodb/flinkx-mongodb-writer/pom.xml
@@ -56,8 +56,12 @@
shade.mongodbwriter.io.netty
- com.google
- shade.mongodbwriter.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-mysql/flinkx-mysql-dreader/pom.xml b/flinkx-mysql/flinkx-mysql-dreader/pom.xml
index 047e273350..21e5e39d4f 100644
--- a/flinkx-mysql/flinkx-mysql-dreader/pom.xml
+++ b/flinkx-mysql/flinkx-mysql-dreader/pom.xml
@@ -62,8 +62,12 @@
shade.mysqldreader.io.netty
- com.google
- shade.mysqldreader.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-mysql/flinkx-mysql-dreader/src/main/java/com/dtstack/flinkx/mysqld/reader/MysqldReader.java b/flinkx-mysql/flinkx-mysql-dreader/src/main/java/com/dtstack/flinkx/mysqld/reader/MysqldReader.java
index 9eae9b3dee..60181af17c 100644
--- a/flinkx-mysql/flinkx-mysql-dreader/src/main/java/com/dtstack/flinkx/mysqld/reader/MysqldReader.java
+++ b/flinkx-mysql/flinkx-mysql-dreader/src/main/java/com/dtstack/flinkx/mysqld/reader/MysqldReader.java
@@ -29,7 +29,6 @@
import java.util.ArrayList;
import java.util.Collections;
-import java.util.List;
/**
* @author toutian
@@ -47,8 +46,8 @@ protected DistributedJdbcInputFormatBuilder getBuilder(){
}
@Override
- protected List buildConnections(){
- List sourceList = new ArrayList<>(connectionConfigs.size());
+ protected ArrayList buildConnections(){
+ ArrayList sourceList = new ArrayList<>(connectionConfigs.size());
for (ReaderConfig.ParameterConfig.ConnectionConfig connectionConfig : connectionConfigs) {
String curUsername = (connectionConfig.getUsername() == null || connectionConfig.getUsername().length() == 0)
? username : connectionConfig.getUsername();
diff --git a/flinkx-mysql/flinkx-mysql-reader/pom.xml b/flinkx-mysql/flinkx-mysql-reader/pom.xml
index 8856cdf063..a52536b7ec 100644
--- a/flinkx-mysql/flinkx-mysql-reader/pom.xml
+++ b/flinkx-mysql/flinkx-mysql-reader/pom.xml
@@ -62,9 +62,13 @@
shade.mysqlreader.io.netty
- com.google
- shade.mysqlreader.com.google
-
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
diff --git a/flinkx-mysql/flinkx-mysql-reader/src/main/java/com/dtstack/flinkx/mysql/format/MysqlInputFormat.java b/flinkx-mysql/flinkx-mysql-reader/src/main/java/com/dtstack/flinkx/mysql/format/MysqlInputFormat.java
index 9836df273a..148d4bbb95 100644
--- a/flinkx-mysql/flinkx-mysql-reader/src/main/java/com/dtstack/flinkx/mysql/format/MysqlInputFormat.java
+++ b/flinkx-mysql/flinkx-mysql-reader/src/main/java/com/dtstack/flinkx/mysql/format/MysqlInputFormat.java
@@ -18,11 +18,10 @@
package com.dtstack.flinkx.mysql.format;
import com.dtstack.flinkx.rdb.inputformat.JdbcInputFormat;
+import com.dtstack.flinkx.rdb.inputformat.JdbcInputSplit;
import com.dtstack.flinkx.rdb.util.DbUtil;
import com.dtstack.flinkx.util.ClassUtil;
import com.dtstack.flinkx.util.DateUtil;
-import com.dtstack.flinkx.util.ExceptionUtil;
-import com.google.gson.Gson;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.core.io.InputSplit;
@@ -51,7 +50,9 @@ public void openInternal(InputSplit inputSplit) throws IOException {
String startLocation = incrementConfig.getStartLocation();
if (incrementConfig.isPolling()) {
- endLocationAccumulator.add(Long.parseLong(startLocation));
+ if (StringUtils.isNotEmpty(startLocation)) {
+ endLocationAccumulator.add(Long.parseLong(startLocation));
+ }
isTimestamp = "timestamp".equalsIgnoreCase(incrementConfig.getColumnType());
} else if ((incrementConfig.isIncrement() && incrementConfig.isUseMaxFunc())) {
getMaxValue(inputSplit);
@@ -64,8 +65,10 @@ public void openInternal(InputSplit inputSplit) throws IOException {
}
querySql = buildQuerySql(inputSplit);
- //MySQL流式读取
- fetchSize = Integer.MIN_VALUE;
+ JdbcInputSplit jdbcInputSplit = (JdbcInputSplit) inputSplit;
+ if (null != jdbcInputSplit.getStartLocation()) {
+ startLocation = jdbcInputSplit.getStartLocation();
+ }
executeQuery(startLocation);
columnCount = resultSet.getMetaData().getColumnCount();
boolean splitWithRowCol = numPartitions > 1 && StringUtils.isNotEmpty(splitKey) && splitKey.contains("(");
@@ -112,7 +115,6 @@ public Row nextRecordInternal(Row row) throws IOException {
}
return super.nextRecordInternal(row);
}catch (Exception e) {
- LOG.error("error to get next record, row = {}, descColumnTypeList = {}, e = {}", row, new Gson().toJson(descColumnTypeList), ExceptionUtil.getErrorMessage(e));
throw new IOException("Couldn't read data - " + e.getMessage(), e);
}
}
diff --git a/flinkx-mysql/flinkx-mysql-reader/src/test/java/bigdata/TestMysqlPk.java b/flinkx-mysql/flinkx-mysql-reader/src/test/java/bigdata/TestMysqlPk.java
deleted file mode 100644
index 50dff8f343..0000000000
--- a/flinkx-mysql/flinkx-mysql-reader/src/test/java/bigdata/TestMysqlPk.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package bigdata;
-
-import com.dtstack.flinkx.mysql.MySqlDatabaseMeta;
-import com.dtstack.flinkx.rdb.util.DbUtil;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.Map;
-
-
-public class TestMysqlPk {
- public static void main(String[] args) throws ClassNotFoundException, SQLException {
- MySqlDatabaseMeta databaseMeta = new MySqlDatabaseMeta();
- Class.forName(databaseMeta.getDriverClass());
- Connection conn = DriverManager.getConnection("jdbc:mysql://172.16.8.104:3306/test?useCursorFetch=true", "dtstack", "abc123");
- //List list = databaseMeta.listUniqueKeys("sb250", conn);
- //System.out.println(list);
- Map map = DbUtil.getPrimaryOrUniqueKeys("sb252", conn);
- System.out.println(map);
- }
-}
diff --git a/flinkx-mysql/flinkx-mysql-writer/pom.xml b/flinkx-mysql/flinkx-mysql-writer/pom.xml
index 82e3aa55c4..94e5d95a50 100644
--- a/flinkx-mysql/flinkx-mysql-writer/pom.xml
+++ b/flinkx-mysql/flinkx-mysql-writer/pom.xml
@@ -62,8 +62,12 @@
shade.mysqlwriter.io.netty
- com.google
- shade.mysqlwriter.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-odps/flinkx-odps-core/src/main/java/com/dtstack/flinkx/odps/OdpsUtil.java b/flinkx-odps/flinkx-odps-core/src/main/java/com/dtstack/flinkx/odps/OdpsUtil.java
index dd54b0ba18..1cb857e3ec 100644
--- a/flinkx-odps/flinkx-odps-core/src/main/java/com/dtstack/flinkx/odps/OdpsUtil.java
+++ b/flinkx-odps/flinkx-odps-core/src/main/java/com/dtstack/flinkx/odps/OdpsUtil.java
@@ -56,7 +56,7 @@ public class OdpsUtil {
public static int MAX_RETRY_TIME = 3;
- public static final long BUFFER_SIZE_DEFAULT = 64 * 1024 * 1024;
+ public static final long BUFFER_SIZE_DEFAULT = 64 * 1024 * 1024L;
public static Odps initOdps(Map odpsConfig) {
String odpsServer = odpsConfig.get(OdpsConfigKeys.KEY_ODPS_SERVER);
diff --git a/flinkx-odps/flinkx-odps-reader/pom.xml b/flinkx-odps/flinkx-odps-reader/pom.xml
index 866fd622fe..8007e473c0 100644
--- a/flinkx-odps/flinkx-odps-reader/pom.xml
+++ b/flinkx-odps/flinkx-odps-reader/pom.xml
@@ -50,6 +50,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-odps/flinkx-odps-reader/src/main/java/com/dtstack/flinkx/odps/reader/OdpsInputFormatBuilder.java b/flinkx-odps/flinkx-odps-reader/src/main/java/com/dtstack/flinkx/odps/reader/OdpsInputFormatBuilder.java
index 1bd4db402b..eba6fc7175 100644
--- a/flinkx-odps/flinkx-odps-reader/src/main/java/com/dtstack/flinkx/odps/reader/OdpsInputFormatBuilder.java
+++ b/flinkx-odps/flinkx-odps-reader/src/main/java/com/dtstack/flinkx/odps/reader/OdpsInputFormatBuilder.java
@@ -19,11 +19,11 @@
package com.dtstack.flinkx.odps.reader;
import com.dtstack.flinkx.inputformat.BaseRichInputFormatBuilder;
+import com.dtstack.flinkx.odps.OdpsConfigKeys;
import com.dtstack.flinkx.reader.MetaColumn;
import java.util.List;
import java.util.Map;
-import static com.dtstack.flinkx.odps.OdpsConfigKeys.*;
/**
* The Builder of OdpsInputFormat
@@ -41,7 +41,7 @@ public OdpsInputFormatBuilder() {
public void setOdpsConfig(Map odpsConfig) {
format.odpsConfig = odpsConfig;
- format.projectName = odpsConfig.get(KEY_PROJECT);
+ format.projectName = odpsConfig.get(OdpsConfigKeys.KEY_PROJECT);
}
public void setTableName(String tableName) {
diff --git a/flinkx-odps/flinkx-odps-reader/src/main/java/com/dtstack/flinkx/odps/reader/OdpsReader.java b/flinkx-odps/flinkx-odps-reader/src/main/java/com/dtstack/flinkx/odps/reader/OdpsReader.java
index 4a076ce122..f45ee2d76a 100644
--- a/flinkx-odps/flinkx-odps-reader/src/main/java/com/dtstack/flinkx/odps/reader/OdpsReader.java
+++ b/flinkx-odps/flinkx-odps-reader/src/main/java/com/dtstack/flinkx/odps/reader/OdpsReader.java
@@ -29,7 +29,9 @@
import java.util.List;
import java.util.Map;
-import static com.dtstack.flinkx.odps.OdpsConfigKeys.*;
+import static com.dtstack.flinkx.odps.OdpsConfigKeys.KEY_ODPS_CONFIG;
+import static com.dtstack.flinkx.odps.OdpsConfigKeys.KEY_PARTITION;
+import static com.dtstack.flinkx.odps.OdpsConfigKeys.KEY_TABLE;
/**
* The reader plugin of Odps
diff --git a/flinkx-odps/flinkx-odps-writer/pom.xml b/flinkx-odps/flinkx-odps-writer/pom.xml
index b268bf43b1..f1622adc0b 100644
--- a/flinkx-odps/flinkx-odps-writer/pom.xml
+++ b/flinkx-odps/flinkx-odps-writer/pom.xml
@@ -55,6 +55,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-odps/flinkx-odps-writer/src/main/java/com/dtstack/flinkx/odps/writer/OdpsOutputFormat.java b/flinkx-odps/flinkx-odps-writer/src/main/java/com/dtstack/flinkx/odps/writer/OdpsOutputFormat.java
index 7e6510104d..cdf8b9c6e8 100644
--- a/flinkx-odps/flinkx-odps-writer/src/main/java/com/dtstack/flinkx/odps/writer/OdpsOutputFormat.java
+++ b/flinkx-odps/flinkx-odps-writer/src/main/java/com/dtstack/flinkx/odps/writer/OdpsOutputFormat.java
@@ -116,7 +116,7 @@ public void writeSingleRecordInternal(Row row) throws WriteRecordException{
@Override
protected void writeMultipleRecordsInternal() throws Exception {
- throw new UnsupportedOperationException();
+ notSupportBatchWrite("OdpsWriter");
}
private Record row2record(Row row, String[] columnTypes) throws WriteRecordException {
diff --git a/flinkx-odps/flinkx-odps-writer/src/main/java/com/dtstack/flinkx/odps/writer/OdpsOutputFormatBuilder.java b/flinkx-odps/flinkx-odps-writer/src/main/java/com/dtstack/flinkx/odps/writer/OdpsOutputFormatBuilder.java
index 95c7dad246..ee3d21d10a 100644
--- a/flinkx-odps/flinkx-odps-writer/src/main/java/com/dtstack/flinkx/odps/writer/OdpsOutputFormatBuilder.java
+++ b/flinkx-odps/flinkx-odps-writer/src/main/java/com/dtstack/flinkx/odps/writer/OdpsOutputFormatBuilder.java
@@ -76,5 +76,7 @@ protected void checkFormat() {
if (format.getRestoreConfig() != null && format.getRestoreConfig().isRestore()){
throw new UnsupportedOperationException("This plugin not support restore from failed state");
}
+
+ notSupportBatchWrite("OdpsWriter");
}
}
diff --git a/flinkx-oracle/flinkx-oracle-core/src/main/java/com/dtstack/flinkx/oracle/OracleDatabaseMeta.java b/flinkx-oracle/flinkx-oracle-core/src/main/java/com/dtstack/flinkx/oracle/OracleDatabaseMeta.java
index 5033bf64af..bc4b88f3ba 100644
--- a/flinkx-oracle/flinkx-oracle-core/src/main/java/com/dtstack/flinkx/oracle/OracleDatabaseMeta.java
+++ b/flinkx-oracle/flinkx-oracle-core/src/main/java/com/dtstack/flinkx/oracle/OracleDatabaseMeta.java
@@ -36,7 +36,7 @@ public class OracleDatabaseMeta extends BaseDatabaseMeta {
public String quoteTable(String table) {
table = table.replace("\"","");
String[] part = table.split("\\.");
- if(part.length == 2) {
+ if(part.length == DB_TABLE_PART_SIZE) {
table = getStartQuote() + part[0] + getEndQuote() + "." + getStartQuote() + part[1] + getEndQuote();
} else {
table = getStartQuote() + table + getEndQuote();
diff --git a/flinkx-oracle/flinkx-oracle-reader/pom.xml b/flinkx-oracle/flinkx-oracle-reader/pom.xml
index 52ff1880be..225d8e2554 100644
--- a/flinkx-oracle/flinkx-oracle-reader/pom.xml
+++ b/flinkx-oracle/flinkx-oracle-reader/pom.xml
@@ -62,8 +62,12 @@
shade.oracleReader.io.netty
- com.google.guava
- shade.oracleReader.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-oracle/flinkx-oracle-reader/src/test/java/com/dtstack/flinkx/oracle/reader/test/OracleLocalTest.java b/flinkx-oracle/flinkx-oracle-reader/src/test/java/com/dtstack/flinkx/oracle/reader/test/OracleLocalTest.java
deleted file mode 100644
index d4227ff774..0000000000
--- a/flinkx-oracle/flinkx-oracle-reader/src/test/java/com/dtstack/flinkx/oracle/reader/test/OracleLocalTest.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package com.dtstack.flinkx.oracle.reader.test;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-
-/**
- * Created by softfly on 18/2/1.
- */
-public class OracleLocalTest {
- public static void main(String[] args) throws ClassNotFoundException, SQLException {
- final String DRIVER = "oracle.jdbc.OracleDriver";
- final String URL = "jdbc:oracle:thin:dev/pass1234@172.16.8.121:1521:dtstack";
- final String USER = "dev";
- final String PASSWORD = "pass1234";
- Connection connection = null;
- Class.forName(DRIVER);
- connection = DriverManager.getConnection(URL);
-
-
- }
-}
diff --git a/flinkx-oracle/flinkx-oracle-writer/pom.xml b/flinkx-oracle/flinkx-oracle-writer/pom.xml
index c6fc7b4e2b..6e9c1850e7 100644
--- a/flinkx-oracle/flinkx-oracle-writer/pom.xml
+++ b/flinkx-oracle/flinkx-oracle-writer/pom.xml
@@ -63,8 +63,12 @@
shade.oracleWriter.io.netty
- com.google.guava
- shade.oracleWriter.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-oracle/flinkx-oracle-writer/src/main/java/com/dtstack/flinkx/oracle/format/OracleOutputFormat.java b/flinkx-oracle/flinkx-oracle-writer/src/main/java/com/dtstack/flinkx/oracle/format/OracleOutputFormat.java
index ed77a77ea4..98840d5b10 100644
--- a/flinkx-oracle/flinkx-oracle-writer/src/main/java/com/dtstack/flinkx/oracle/format/OracleOutputFormat.java
+++ b/flinkx-oracle/flinkx-oracle-writer/src/main/java/com/dtstack/flinkx/oracle/format/OracleOutputFormat.java
@@ -18,11 +18,16 @@
package com.dtstack.flinkx.oracle.format;
import com.dtstack.flinkx.enums.ColumnType;
+import com.dtstack.flinkx.oracle.OracleDatabaseMeta;
import com.dtstack.flinkx.rdb.outputformat.JdbcOutputFormat;
import com.dtstack.flinkx.util.DateUtil;
import org.apache.flink.types.Row;
-import java.sql.*;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
@@ -63,7 +68,7 @@ protected List probeFullColumns(String table, Connection dbConn) throws
String schema =null;
String[] parts = table.split("\\.");
- if(parts.length == 2) {
+ if(parts.length == OracleDatabaseMeta.DB_TABLE_PART_SIZE) {
schema = parts[0].toUpperCase();
table = parts[1];
}
@@ -79,24 +84,26 @@ protected List probeFullColumns(String table, Connection dbConn) throws
@Override
protected Map> probePrimaryKeys(String table, Connection dbConn) throws SQLException {
Map> map = new HashMap<>(16);
- PreparedStatement ps = dbConn.prepareStatement(String.format(GET_INDEX_SQL,table));
- ResultSet rs = ps.executeQuery();
- while(rs.next()) {
- String indexName = rs.getString("INDEX_NAME");
- if(!map.containsKey(indexName)) {
- map.put(indexName,new ArrayList<>());
+ try (PreparedStatement ps = dbConn.prepareStatement(String.format(GET_INDEX_SQL, table));
+ ResultSet rs = ps.executeQuery()) {
+ while(rs.next()) {
+ String indexName = rs.getString("INDEX_NAME");
+ if(!map.containsKey(indexName)) {
+ map.put(indexName,new ArrayList<>());
+ }
+ map.get(indexName).add(rs.getString("COLUMN_NAME"));
}
- map.get(indexName).add(rs.getString("COLUMN_NAME"));
- }
- Map> retMap = new HashMap<>((map.size()<<2)/3);
- for(Map.Entry> entry: map.entrySet()) {
- String k = entry.getKey();
- List v = entry.getValue();
- if(v!=null && v.size() != 0 && v.get(0) != null) {
- retMap.put(k, v);
+
+ Map> retMap = new HashMap<>((map.size()<<2)/3);
+ for(Map.Entry> entry: map.entrySet()) {
+ String k = entry.getKey();
+ List v = entry.getValue();
+ if(v!=null && v.size() != 0 && v.get(0) != null) {
+ retMap.put(k, v);
+ }
}
+ return retMap;
}
- return retMap;
}
}
diff --git a/flinkx-pgwal/flinkx-pgwal-reader/pom.xml b/flinkx-pgwal/flinkx-pgwal-reader/pom.xml
index 182ea07067..04f327d64e 100644
--- a/flinkx-pgwal/flinkx-pgwal-reader/pom.xml
+++ b/flinkx-pgwal/flinkx-pgwal-reader/pom.xml
@@ -33,6 +33,16 @@
false
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-phoenix/flinkx-phoenix-reader/pom.xml b/flinkx-phoenix/flinkx-phoenix-reader/pom.xml
index 180385dc01..b067a02a7e 100644
--- a/flinkx-phoenix/flinkx-phoenix-reader/pom.xml
+++ b/flinkx-phoenix/flinkx-phoenix-reader/pom.xml
@@ -56,6 +56,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-phoenix/flinkx-phoenix-reader/src/main/java/com/dtstack/flinkx/phoenix/format/PhoenixInputFormat.java b/flinkx-phoenix/flinkx-phoenix-reader/src/main/java/com/dtstack/flinkx/phoenix/format/PhoenixInputFormat.java
index 6d6b416559..89bdc80968 100644
--- a/flinkx-phoenix/flinkx-phoenix-reader/src/main/java/com/dtstack/flinkx/phoenix/format/PhoenixInputFormat.java
+++ b/flinkx-phoenix/flinkx-phoenix-reader/src/main/java/com/dtstack/flinkx/phoenix/format/PhoenixInputFormat.java
@@ -29,7 +29,6 @@
import java.io.IOException;
import java.sql.SQLException;
-import java.sql.Statement;
import static com.dtstack.flinkx.rdb.util.DbUtil.clobToString;
@@ -65,7 +64,7 @@ public void openInternal(InputSplit inputSplit) throws IOException {
// 部分驱动需要关闭事务自动提交，fetchSize参数才会起作用
dbConn.setAutoCommit(false);
- Statement statement = dbConn.createStatement(resultSetType, resultSetConcurrency);
+ statement = dbConn.createStatement(resultSetType, resultSetConcurrency);
statement.setFetchSize(0);
diff --git a/flinkx-phoenix/flinkx-phoenix-writer/pom.xml b/flinkx-phoenix/flinkx-phoenix-writer/pom.xml
index b97dab27c8..1dc16435a8 100644
--- a/flinkx-phoenix/flinkx-phoenix-writer/pom.xml
+++ b/flinkx-phoenix/flinkx-phoenix-writer/pom.xml
@@ -56,6 +56,16 @@
+
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
+
diff --git a/flinkx-polardb/flinkx-polardb-dreader/pom.xml b/flinkx-polardb/flinkx-polardb-dreader/pom.xml
index d477787cac..26060ebcf6 100644
--- a/flinkx-polardb/flinkx-polardb-dreader/pom.xml
+++ b/flinkx-polardb/flinkx-polardb-dreader/pom.xml
@@ -56,6 +56,14 @@
com.mysql
shade.polardb.com.mysql
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
diff --git a/flinkx-polardb/flinkx-polardb-dreader/src/main/java/com/dtstack/flinkx/polardbd/reader/PolardbdReader.java b/flinkx-polardb/flinkx-polardb-dreader/src/main/java/com/dtstack/flinkx/polardbd/reader/PolardbdReader.java
index b7b3e95855..5e21aec025 100644
--- a/flinkx-polardb/flinkx-polardb-dreader/src/main/java/com/dtstack/flinkx/polardbd/reader/PolardbdReader.java
+++ b/flinkx-polardb/flinkx-polardb-dreader/src/main/java/com/dtstack/flinkx/polardbd/reader/PolardbdReader.java
@@ -21,7 +21,6 @@
import com.dtstack.flinkx.mysqld.reader.MysqldReader;
import com.dtstack.flinkx.polardbd.PolardbDatabaseMeta;
import com.dtstack.flinkx.polardbd.format.PolardbdInputFormat;
-import com.dtstack.flinkx.rdb.inputformat.DistributedJdbcInputFormat;
import com.dtstack.flinkx.rdb.inputformat.DistributedJdbcInputFormatBuilder;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
diff --git a/flinkx-polardb/flinkx-polardb-reader/pom.xml b/flinkx-polardb/flinkx-polardb-reader/pom.xml
index 093bdb5152..77ca955ae3 100644
--- a/flinkx-polardb/flinkx-polardb-reader/pom.xml
+++ b/flinkx-polardb/flinkx-polardb-reader/pom.xml
@@ -55,6 +55,14 @@
com.mysql
shade.polardb.com.mysql
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
diff --git a/flinkx-polardb/flinkx-polardb-writer/pom.xml b/flinkx-polardb/flinkx-polardb-writer/pom.xml
index d088128085..4ca3fb7511 100644
--- a/flinkx-polardb/flinkx-polardb-writer/pom.xml
+++ b/flinkx-polardb/flinkx-polardb-writer/pom.xml
@@ -56,6 +56,14 @@
com.mysql
shade.polardb.com.mysql
+
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
+
diff --git a/flinkx-postgresql/flinkx-postgresql-core/src/main/java/com/dtstack/flinkx/postgresql/PostgresqlDatabaseMeta.java b/flinkx-postgresql/flinkx-postgresql-core/src/main/java/com/dtstack/flinkx/postgresql/PostgresqlDatabaseMeta.java
index 985b99efaa..4af93b9480 100644
--- a/flinkx-postgresql/flinkx-postgresql-core/src/main/java/com/dtstack/flinkx/postgresql/PostgresqlDatabaseMeta.java
+++ b/flinkx-postgresql/flinkx-postgresql-core/src/main/java/com/dtstack/flinkx/postgresql/PostgresqlDatabaseMeta.java
@@ -20,10 +20,7 @@
import com.dtstack.flinkx.enums.EDatabaseType;
import com.dtstack.flinkx.rdb.BaseDatabaseMeta;
-import org.apache.commons.lang3.StringUtils;
-import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
diff --git a/flinkx-postgresql/flinkx-postgresql-core/src/main/java/com/dtstack/flinkx/postgresql/PostgresqlTypeConverter.java b/flinkx-postgresql/flinkx-postgresql-core/src/main/java/com/dtstack/flinkx/postgresql/PostgresqlTypeConverter.java
index 190ff818b1..04f5616f23 100644
--- a/flinkx-postgresql/flinkx-postgresql-core/src/main/java/com/dtstack/flinkx/postgresql/PostgresqlTypeConverter.java
+++ b/flinkx-postgresql/flinkx-postgresql-core/src/main/java/com/dtstack/flinkx/postgresql/PostgresqlTypeConverter.java
@@ -19,8 +19,10 @@
package com.dtstack.flinkx.postgresql;
import com.dtstack.flinkx.rdb.type.TypeConverterInterface;
+import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
/**
@@ -35,9 +37,9 @@ public class PostgresqlTypeConverter implements TypeConverterInterface {
private List byteTypes = Arrays.asList("bytea","bit varying");
- private List bitTypes = Arrays.asList("bit");
+ private List bitTypes = Collections.singletonList("bit");
- private List doubleTypes = Arrays.asList("money");
+ private List doubleTypes = Collections.singletonList("money");
private List intTypes = Arrays.asList("int","int2","int4","int8");
@@ -46,19 +48,24 @@ public Object convert(Object data,String typeName) {
if (data == null){
return null;
}
-
+ String dataValue = data.toString();
+ if(stringTypes.contains(typeName)){
+ return dataValue;
+ }
+ if(StringUtils.isBlank(dataValue)){
+ return null;
+ }
if(doubleTypes.contains(typeName)){
- data = Double.parseDouble(String.valueOf(data));
+ if(StringUtils.startsWith(dataValue, "$")){
+ dataValue = StringUtils.substring(dataValue, 1);
+ }
+ data = Double.parseDouble(dataValue);
} else if(bitTypes.contains(typeName)){
//
- } else if(stringTypes.contains(typeName)){
- data = String.valueOf(data);
- } else if(byteTypes.contains(typeName)){
- data = Byte.valueOf(String.valueOf(data));
+ }else if(byteTypes.contains(typeName)){
+ data = Byte.valueOf(dataValue);
} else if(intTypes.contains(typeName)){
- if(data instanceof String){
- data = Integer.parseInt(data.toString());
- }
+ data = Integer.parseInt(dataValue);
}
return data;
diff --git a/flinkx-postgresql/flinkx-postgresql-reader/pom.xml b/flinkx-postgresql/flinkx-postgresql-reader/pom.xml
index f103f55d85..6e3b8612d6 100644
--- a/flinkx-postgresql/flinkx-postgresql-reader/pom.xml
+++ b/flinkx-postgresql/flinkx-postgresql-reader/pom.xml
@@ -63,8 +63,12 @@
shade.postgresqlreader.io.netty
- com.google
- shade.postgresqlreader.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-postgresql/flinkx-postgresql-reader/src/main/java/com/dtstack/flinkx/postgresql/format/PostgresqlInputFormat.java b/flinkx-postgresql/flinkx-postgresql-reader/src/main/java/com/dtstack/flinkx/postgresql/format/PostgresqlInputFormat.java
index 5ba10faad7..790f66f7e9 100644
--- a/flinkx-postgresql/flinkx-postgresql-reader/src/main/java/com/dtstack/flinkx/postgresql/format/PostgresqlInputFormat.java
+++ b/flinkx-postgresql/flinkx-postgresql-reader/src/main/java/com/dtstack/flinkx/postgresql/format/PostgresqlInputFormat.java
@@ -19,6 +19,7 @@
package com.dtstack.flinkx.postgresql.format;
import com.dtstack.flinkx.rdb.inputformat.JdbcInputFormat;
+import com.dtstack.flinkx.rdb.inputformat.JdbcInputSplit;
import com.dtstack.flinkx.rdb.util.DbUtil;
import com.dtstack.flinkx.util.ClassUtil;
import org.apache.commons.collections.CollectionUtils;
@@ -49,7 +50,9 @@ public void openInternal(InputSplit inputSplit) throws IOException {
String startLocation = incrementConfig.getStartLocation();
if (incrementConfig.isPolling()) {
- endLocationAccumulator.add(Long.parseLong(startLocation));
+ if (StringUtils.isNotEmpty(startLocation)) {
+ endLocationAccumulator.add(Long.parseLong(startLocation));
+ }
isTimestamp = "timestamp".equalsIgnoreCase(incrementConfig.getColumnType());
} else if ((incrementConfig.isIncrement() && incrementConfig.isUseMaxFunc())) {
getMaxValue(inputSplit);
@@ -62,6 +65,10 @@ public void openInternal(InputSplit inputSplit) throws IOException {
}
querySql = buildQuerySql(inputSplit);
+ JdbcInputSplit jdbcInputSplit = (JdbcInputSplit) inputSplit;
+ if (null != jdbcInputSplit.getStartLocation()) {
+ startLocation = jdbcInputSplit.getStartLocation();
+ }
executeQuery(startLocation);
columnCount = resultSet.getMetaData().getColumnCount();
boolean splitWithRowCol = numPartitions > 1 && StringUtils.isNotEmpty(splitKey) && splitKey.contains("(");
diff --git a/flinkx-postgresql/flinkx-postgresql-writer/pom.xml b/flinkx-postgresql/flinkx-postgresql-writer/pom.xml
index 2264b0dddb..7f6857ea04 100644
--- a/flinkx-postgresql/flinkx-postgresql-writer/pom.xml
+++ b/flinkx-postgresql/flinkx-postgresql-writer/pom.xml
@@ -62,8 +62,12 @@
shade.postgresqlwriter.io.netty
- com.google
- shade.postgresqlwriter.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-pulsar/flinkx-pulsar-writer/pom.xml b/flinkx-pulsar/flinkx-pulsar-writer/pom.xml
index 72c8493094..88396482f4 100644
--- a/flinkx-pulsar/flinkx-pulsar-writer/pom.xml
+++ b/flinkx-pulsar/flinkx-pulsar-writer/pom.xml
@@ -11,11 +11,8 @@
flinkx-pulsar-writer
-
-
-
org.apache.maven.plugins
maven-shade-plugin
diff --git a/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/Constants.java b/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/Constants.java
index b12d1f8ece..28cd09eaad 100644
--- a/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/Constants.java
+++ b/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/Constants.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.pulsar.writer;
/**
diff --git a/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarOutputFormat.java b/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarOutputFormat.java
index 55aeee7dda..fdfab032da 100644
--- a/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarOutputFormat.java
+++ b/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarOutputFormat.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.pulsar.writer;
import com.dtstack.flinkx.exception.WriteRecordException;
@@ -92,7 +109,7 @@ protected void emit(Map event) throws IOException {
}
@Override
- protected void writeMultipleRecordsInternal() throws Exception {
+ protected void writeMultipleRecordsInternal() {
throw new UnsupportedOperationException();
}
diff --git a/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarOutputFormatBuilder.java b/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarOutputFormatBuilder.java
index 637a97e67a..c6e35cc79c 100644
--- a/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarOutputFormatBuilder.java
+++ b/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarOutputFormatBuilder.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.pulsar.writer;
import com.dtstack.flinkx.outputformat.BaseRichOutputFormatBuilder;
diff --git a/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarWriter.java b/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarWriter.java
index ae84b857f1..2068128992 100644
--- a/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarWriter.java
+++ b/flinkx-pulsar/flinkx-pulsar-writer/src/main/java/com/dtstack/flinkx/pulsar/writer/PulsarWriter.java
@@ -1,8 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.dtstack.flinkx.pulsar.writer;
import com.dtstack.flinkx.config.DataTransferConfig;
import com.dtstack.flinkx.writer.BaseDataWriter;
-import static com.dtstack.flinkx.pulsar.writer.Constants.*;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSink;
import org.apache.flink.types.Row;
@@ -10,6 +26,8 @@
import java.util.List;
import java.util.Map;
+import static com.dtstack.flinkx.pulsar.writer.Constants.*;
+
/**
* @author: pierre
@@ -22,6 +40,7 @@ public class PulsarWriter extends BaseDataWriter {
protected List tableFields;
protected Map producerSettings;
+ @SuppressWarnings("unchecked")
public PulsarWriter(DataTransferConfig config){
super(config);
topic = config.getJob().getContent().get(0).getWriter().getParameter().getStringVal(KEY_TOPIC);
diff --git a/flinkx-rdb/flinkx-rdb-core/src/main/java/com/dtstack/flinkx/rdb/BaseDatabaseMeta.java b/flinkx-rdb/flinkx-rdb-core/src/main/java/com/dtstack/flinkx/rdb/BaseDatabaseMeta.java
index ad667ceb13..758be23ace 100644
--- a/flinkx-rdb/flinkx-rdb-core/src/main/java/com/dtstack/flinkx/rdb/BaseDatabaseMeta.java
+++ b/flinkx-rdb/flinkx-rdb-core/src/main/java/com/dtstack/flinkx/rdb/BaseDatabaseMeta.java
@@ -21,7 +21,11 @@
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* Abstract base parent class of other database prototype implementations
@@ -31,6 +35,8 @@
*/
public abstract class BaseDatabaseMeta implements DatabaseInterface, Serializable {
+ public static final int DB_TABLE_PART_SIZE = 2;
+
@Override
public String getStartQuote() {
return "\"";
diff --git a/flinkx-rdb/flinkx-rdb-core/src/main/java/com/dtstack/flinkx/rdb/util/DbUtil.java b/flinkx-rdb/flinkx-rdb-core/src/main/java/com/dtstack/flinkx/rdb/util/DbUtil.java
index 26269c7c34..88477b16b3 100644
--- a/flinkx-rdb/flinkx-rdb-core/src/main/java/com/dtstack/flinkx/rdb/util/DbUtil.java
+++ b/flinkx-rdb/flinkx-rdb-core/src/main/java/com/dtstack/flinkx/rdb/util/DbUtil.java
@@ -18,21 +18,17 @@
package com.dtstack.flinkx.rdb.util;
import com.dtstack.flinkx.constants.ConstantValue;
-import com.dtstack.flinkx.rdb.DatabaseInterface;
import com.dtstack.flinkx.rdb.ParameterValuesProvider;
-import com.dtstack.flinkx.reader.MetaColumn;
import com.dtstack.flinkx.util.ClassUtil;
import com.dtstack.flinkx.util.ExceptionUtil;
import com.dtstack.flinkx.util.SysUtil;
import com.dtstack.flinkx.util.TelnetUtil;
-import com.google.gson.Gson;
import org.apache.commons.lang.StringUtils;
import org.apache.flink.util.CollectionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
-import java.math.BigDecimal;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
@@ -41,9 +37,9 @@
import java.util.regex.Pattern;
/**
+ *
* Utilities for relational database connection and sql execution
* company: www.dtstack.com
- *
* @author huyifan_zju@
*/
public class DbUtil {
@@ -72,6 +68,9 @@ public class DbUtil {
*/
private static int NANOS_LENGTH = 19;
+ public static int NANOS_PART_LENGTH = 9;
+ private static int FORMAT_TIME_NANOS_LENGTH = 29;
+
/**
* jdbc连接URL的分割正则，用于获取URL?后的连接参数
*/
@@ -93,16 +92,15 @@ public class DbUtil {
/**
* 获取jdbc连接(超时10S)
- *
- * @param url url
- * @param username 账号
- * @param password 密码
+ * @param url url
+ * @param username 账号
+ * @param password 密码
* @return
* @throws SQLException
*/
private static Connection getConnectionInternal(String url, String username, String password) throws SQLException {
Connection dbConn;
- synchronized (ClassUtil.LOCK_STR) {
+ synchronized (ClassUtil.LOCK_STR){
DriverManager.setLoginTimeout(10);
// telnet
@@ -120,10 +118,9 @@ private static Connection getConnectionInternal(String url, String username, Str
/**
* 获取jdbc连接(重试3次)
- *
- * @param url url
- * @param username 账号
- * @param password 密码
+ * @param url url
+ * @param username 账号
+ * @param password 密码
* @return
* @throws SQLException
*/
@@ -136,8 +133,10 @@ public static Connection getConnection(String url, String username, String passw
for (int i = 0; i < MAX_RETRY_TIMES && failed; ++i) {
try {
dbConn = getConnectionInternal(url, username, password);
- dbConn.createStatement().execute("select 111");
- failed = false;
+ try (Statement statement = dbConn.createStatement()){
+ statement.execute("select 111");
+ failed = false;
+ }
} catch (Exception e) {
if (dbConn != null) {
dbConn.close();
@@ -157,10 +156,9 @@ public static Connection getConnection(String url, String username, String passw
/**
* 关闭连接资源
- *
- * @param rs ResultSet
- * @param stmt Statement
- * @param conn Connection
+ * @param rs ResultSet
+ * @param stmt Statement
+ * @param conn Connection
* @param commit
*/
public static void closeDbResources(ResultSet rs, Statement stmt, Connection conn, boolean commit) {
@@ -182,7 +180,7 @@ public static void closeDbResources(ResultSet rs, Statement stmt, Connection con
if (null != conn) {
try {
- if (commit) {
+ if(commit){
commit(conn);
}
@@ -195,76 +193,49 @@ public static void closeDbResources(ResultSet rs, Statement stmt, Connection con
/**
* 手动提交事物
- *
* @param conn Connection
*/
- public static void commit(Connection conn) {
+ public static void commit(Connection conn){
try {
- if (!conn.isClosed() && !conn.getAutoCommit()) {
+ if (!conn.isClosed() && !conn.getAutoCommit()){
conn.commit();
}
- } catch (SQLException e) {
+ } catch (SQLException e){
LOG.warn("commit error:{}", ExceptionUtil.getErrorMessage(e));
}
}
/**
* 批量执行sql
- *
* @param dbConn Connection
* @param sqls sql列表
*/
public static void executeBatch(Connection dbConn, List sqls) {
- if (sqls == null || sqls.size() == 0) {
+ if(sqls == null || sqls.size() == 0) {
return;
}
- try {
- Statement stmt = dbConn.createStatement();
- for (String sql : sqls) {
+ try (Statement stmt = dbConn.createStatement()) {
+ for(String sql : sqls) {
stmt.addBatch(sql);
}
stmt.executeBatch();
} catch (SQLException e) {
- throw new RuntimeException("execute batch sql error:{}", e);
+ throw new RuntimeException("execute batch sql error:{}",e);
} finally {
commit(dbConn);
}
}
- /**
- * 获取某数据库某表的主键和唯一索引
- *
- * @param table 表名
- * @param dbConn 数据库连接
- * @return
- * @throws SQLException
- */
- public static Map> getPrimaryOrUniqueKeys(String table, Connection dbConn) throws SQLException {
- Map> keyMap = new HashMap<>(16);
- DatabaseMetaData meta = dbConn.getMetaData();
- ResultSet rs = meta.getIndexInfo(null, null, table, true, false);
- while (rs.next()) {
- String pkName = rs.getString(6);
- String columnName = rs.getString(9);
- if (!keyMap.containsKey(pkName)) {
- keyMap.put(pkName, new ArrayList<>());
- }
- keyMap.get(pkName).add(columnName);
- }
- return keyMap;
- }
-
/**
* 封装channel通道顺序
- *
* @param channels
* @return
*/
- public static Object[][] getParameterValues(final int channels) {
+ public static Object[][] getParameterValues(final int channels){
ParameterValuesProvider provider = () -> {
Integer[][] parameters = new Integer[channels][];
- for (int i = 0; i < channels; ++i) {
+ for(int i = 0; i < channels; ++i) {
parameters[i] = new Integer[2];
parameters[i][0] = channels;
parameters[i][1] = i;
@@ -295,62 +266,20 @@ public static List analyzeColumnType(ResultSet resultSet){
return columnTypeList;
}
- /**
- * 占位符设值
- *
- * @param param 参数
- * @param statement PreparedStatement
- * @param i 占位符位置
- * @throws SQLException
- */
- public static void setParameterValue(Object param, PreparedStatement statement, int i) throws SQLException {
- if (param instanceof String) {
- statement.setString(i + 1, (String) param);
- } else if (param instanceof Long) {
- statement.setLong(i + 1, (Long) param);
- } else if (param instanceof Integer) {
- statement.setInt(i + 1, (Integer) param);
- } else if (param instanceof Double) {
- statement.setDouble(i + 1, (Double) param);
- } else if (param instanceof Boolean) {
- statement.setBoolean(i + 1, (Boolean) param);
- } else if (param instanceof Float) {
- statement.setFloat(i + 1, (Float) param);
- } else if (param instanceof BigDecimal) {
- statement.setBigDecimal(i + 1, (BigDecimal) param);
- } else if (param instanceof Byte) {
- statement.setByte(i + 1, (Byte) param);
- } else if (param instanceof Short) {
- statement.setShort(i + 1, (Short) param);
- } else if (param instanceof Date) {
- statement.setDate(i + 1, (Date) param);
- } else if (param instanceof Time) {
- statement.setTime(i + 1, (Time) param);
- } else if (param instanceof Timestamp) {
- statement.setTimestamp(i + 1, (Timestamp) param);
- } else if (param instanceof Array) {
- statement.setArray(i + 1, (Array) param);
- } else {
- //extends with other types if needed
- throw new IllegalArgumentException("open() failed. Parameter " + i + " of type " + param.getClass() + " is not handled (yet).");
- }
- }
-
/**
* clob转string
- *
- * @param obj clob
+ * @param obj clob
* @return
* @throws Exception
*/
- public static Object clobToString(Object obj) throws Exception {
+ public static Object clobToString(Object obj) throws Exception{
String dataStr;
- if (obj instanceof Clob) {
- Clob clob = (Clob) obj;
+ if(obj instanceof Clob){
+ Clob clob = (Clob)obj;
BufferedReader bf = new BufferedReader(clob.getCharacterStream());
StringBuilder stringBuilder = new StringBuilder();
String line;
- while ((line = bf.readLine()) != null) {
+ while ((line = bf.readLine()) != null){
stringBuilder.append(line);
}
dataStr = stringBuilder.toString();
@@ -363,34 +292,32 @@ public static Object clobToString(Object obj) throws Exception {
/**
* 获取纳秒字符串
- *
- * @param timeStr
+ * @param timeStr 2020-03-23 11:03:22.000000000
* @return
*/
- public static String getNanosTimeStr(String timeStr) {
- if (timeStr.length() < 29) {
- timeStr += StringUtils.repeat("0", 29 - timeStr.length());
+ public static String getNanosTimeStr(String timeStr){
+ if(timeStr.length() < FORMAT_TIME_NANOS_LENGTH){
+ timeStr += StringUtils.repeat("0",FORMAT_TIME_NANOS_LENGTH - timeStr.length());
}
return timeStr;
}
/**
* 将边界位置时间转换成对应饿的纳秒时间
- *
* @param startLocation 边界位置(起始/结束)
* @return
*/
- public static int getNanos(long startLocation) {
+ public static int getNanos(long startLocation){
String timeStr = String.valueOf(startLocation);
int nanos;
- if (timeStr.length() == SECOND_LENGTH) {
+ if (timeStr.length() == SECOND_LENGTH){
nanos = 0;
- } else if (timeStr.length() == MILLIS_LENGTH) {
- nanos = Integer.parseInt(timeStr.substring(SECOND_LENGTH, MILLIS_LENGTH)) * 1000000;
- } else if (timeStr.length() == MICRO_LENGTH) {
- nanos = Integer.parseInt(timeStr.substring(SECOND_LENGTH, MICRO_LENGTH)) * 1000;
- } else if (timeStr.length() == NANOS_LENGTH) {
- nanos = Integer.parseInt(timeStr.substring(SECOND_LENGTH, NANOS_LENGTH));
+ } else if (timeStr.length() == MILLIS_LENGTH){
+ nanos = Integer.parseInt(timeStr.substring(SECOND_LENGTH,MILLIS_LENGTH)) * 1000000;
+ } else if (timeStr.length() == MICRO_LENGTH){
+ nanos = Integer.parseInt(timeStr.substring(SECOND_LENGTH,MICRO_LENGTH)) * 1000;
+ } else if (timeStr.length() == NANOS_LENGTH){
+ nanos = Integer.parseInt(timeStr.substring(SECOND_LENGTH,NANOS_LENGTH));
} else {
throw new IllegalArgumentException("Unknown time unit:startLocation=" + startLocation);
}
@@ -400,20 +327,19 @@ public static int getNanos(long startLocation) {
/**
* 将边界位置时间转换成对应饿的毫秒时间
- *
- * @param startLocation 边界位置(起始/结束)
+ * @param startLocation 边界位置(起始/结束)
* @return
*/
- public static long getMillis(long startLocation) {
+ public static long getMillis(long startLocation){
String timeStr = String.valueOf(startLocation);
long millisSecond;
- if (timeStr.length() == SECOND_LENGTH) {
+ if (timeStr.length() == SECOND_LENGTH){
millisSecond = startLocation * 1000;
- } else if (timeStr.length() == MILLIS_LENGTH) {
+ } else if (timeStr.length() == MILLIS_LENGTH){
millisSecond = startLocation;
- } else if (timeStr.length() == MICRO_LENGTH) {
+ } else if (timeStr.length() == MICRO_LENGTH){
millisSecond = startLocation / 1000;
- } else if (timeStr.length() == NANOS_LENGTH) {
+ } else if (timeStr.length() == NANOS_LENGTH){
millisSecond = startLocation / 1000000;
} else {
throw new IllegalArgumentException("Unknown time unit:startLocation=" + startLocation);
@@ -424,24 +350,23 @@ public static long getMillis(long startLocation) {
/**
* 格式化jdbc连接
- *
- * @param dbUrl 原jdbc连接
- * @param extParamMap 需要额外添加的参数
- * @return 格式化后jdbc连接URL字符串
+ * @param dbUrl 原jdbc连接
+ * @param extParamMap 需要额外添加的参数
+ * @return 格式化后jdbc连接URL字符串
*/
- public static String formatJdbcUrl(String dbUrl, Map extParamMap) {
+ public static String formatJdbcUrl(String dbUrl, Map extParamMap){
String[] splits = DB_PATTERN.split(dbUrl);
- Map paramMap = new HashMap<>(16);
- if (splits.length > 1) {
+ Map paramMap = new HashMap<>(16);
+ if(splits.length > 1) {
String[] pairs = splits[1].split("&");
- for (String pair : pairs) {
+ for(String pair : pairs) {
String[] leftRight = pair.split("=");
paramMap.put(leftRight[0], leftRight[1]);
}
}
- if (!CollectionUtil.isNullOrEmpty(extParamMap)) {
+ if(!CollectionUtil.isNullOrEmpty(extParamMap)){
paramMap.putAll(extParamMap);
}
paramMap.put("useCursorFetch", "true");
@@ -450,8 +375,8 @@ public static String formatJdbcUrl(String dbUrl, Map extParamMap
StringBuffer sb = new StringBuffer(dbUrl.length() + 128);
sb.append(splits[0]).append("?");
int index = 0;
- for (Map.Entry entry : paramMap.entrySet()) {
- if (index != 0) {
+ for(Map.Entry entry : paramMap.entrySet()) {
+ if(index != 0) {
sb.append("&");
}
sb.append(entry.getKey()).append("=").append(entry.getValue());
diff --git a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.datareader/DistributedJdbcDataReader.java b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.datareader/DistributedJdbcDataReader.java
index 4faa32e0ce..f4442111d7 100644
--- a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.datareader/DistributedJdbcDataReader.java
+++ b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.datareader/DistributedJdbcDataReader.java
@@ -107,8 +107,8 @@ protected DistributedJdbcInputFormatBuilder getBuilder(){
throw new RuntimeException("子类必须覆盖getBuilder方法");
}
- protected List buildConnections(){
- List sourceList = new ArrayList<>(connectionConfigs.size());
+ protected ArrayList buildConnections(){
+ ArrayList sourceList = new ArrayList<>(connectionConfigs.size());
for (ReaderConfig.ParameterConfig.ConnectionConfig connectionConfig : connectionConfigs) {
String curUsername = (StringUtils.isBlank(connectionConfig.getUsername())) ? username : connectionConfig.getUsername();
String curPassword = (StringUtils.isBlank(connectionConfig.getPassword())) ? password : connectionConfig.getPassword();
diff --git a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputFormat.java b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputFormat.java
index 2d33a29161..6e9534b554 100644
--- a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputFormat.java
+++ b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputFormat.java
@@ -30,7 +30,11 @@
import org.apache.flink.core.io.InputSplit;
import org.apache.flink.types.Row;
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -65,7 +69,7 @@ public class DistributedJdbcInputFormat extends BaseRichInputFormat {
protected List descColumnTypeList;
- protected List sourceList;
+ protected ArrayList sourceList;
protected transient int sourceIndex;
@@ -197,7 +201,7 @@ protected void closeCurrentSource(){
@Override
protected void closeInternal() throws IOException {
-
+ closeCurrentSource();
}
@Override
@@ -208,7 +212,7 @@ public InputSplit[] createInputSplitsInternal(int minPart) throws IOException {
Object[][] parmeter = DbUtil.getParameterValues(numPartitions);
for (int j = 0; j < numPartitions; j++) {
DistributedJdbcInputSplit split = new DistributedJdbcInputSplit(j,numPartitions);
- List sourceCopy = deepCopyList(sourceList);
+ ArrayList sourceCopy = deepCopyList(sourceList);
for (int i = 0; i < sourceCopy.size(); i++) {
sourceCopy.get(i).setSplitByKey(true);
sourceCopy.get(i).setParameterValues(parmeter[j]);
@@ -221,7 +225,9 @@ public InputSplit[] createInputSplitsInternal(int minPart) throws IOException {
if (partNum == 0){
for (int i = 0; i < sourceList.size(); i++) {
DistributedJdbcInputSplit split = new DistributedJdbcInputSplit(i,numPartitions);
- split.setSourceList(Arrays.asList(sourceList.get(i)));
+ ArrayList arrayList = new ArrayList<>();
+ arrayList.add(sourceList.get(i));
+ split.setSourceList(arrayList);
inputSplits[i] = split;
}
} else {
@@ -250,7 +256,7 @@ public boolean reachedEnd() throws IOException {
return readNextRecord();
}
- public List deepCopyList(List src) throws IOException{
+ public ArrayList deepCopyList(ArrayList src) throws IOException{
try {
ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
ObjectOutputStream out = new ObjectOutputStream(byteOut);
@@ -258,7 +264,7 @@ public List deepCopyList(List src) throws IOException{
ByteArrayInputStream byteIn = new ByteArrayInputStream(byteOut.toByteArray());
ObjectInputStream in = new ObjectInputStream(byteIn);
- List dest = (List) in.readObject();
+ ArrayList dest = (ArrayList) in.readObject();
return dest;
} catch (Exception e){
diff --git a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputFormatBuilder.java b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputFormatBuilder.java
index 28f2c2c5f1..0c8b21c07f 100644
--- a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputFormatBuilder.java
+++ b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputFormatBuilder.java
@@ -24,6 +24,7 @@
import com.dtstack.flinkx.reader.MetaColumn;
import org.apache.commons.lang.StringUtils;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -65,7 +66,7 @@ public void setSplitKey(String splitKey){
format.splitKey = splitKey;
}
- public void setSourceList(List sourceList){
+ public void setSourceList(ArrayList sourceList){
format.sourceList = sourceList;
}
diff --git a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputSplit.java b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputSplit.java
index 724f9532d5..43df7f5c88 100644
--- a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputSplit.java
+++ b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/DistributedJdbcInputSplit.java
@@ -32,7 +32,7 @@
*/
public class DistributedJdbcInputSplit extends GenericInputSplit {
- private List sourceList;
+ private ArrayList sourceList;
public DistributedJdbcInputSplit(int partitionNumber, int totalNumberOfPartitions) {
super(partitionNumber, totalNumberOfPartitions);
@@ -54,11 +54,11 @@ public void addSource(DataSource source){
this.sourceList.add(source);
}
- public List getSourceList() {
+ public ArrayList getSourceList() {
return sourceList;
}
- public void setSourceList(List sourceList) {
+ public void setSourceList(ArrayList sourceList) {
this.sourceList = sourceList;
}
}
diff --git a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/JdbcInputFormat.java b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/JdbcInputFormat.java
index 706600d712..d71fa41d16 100644
--- a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/JdbcInputFormat.java
+++ b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/JdbcInputFormat.java
@@ -27,12 +27,8 @@
import com.dtstack.flinkx.rdb.type.TypeConverterInterface;
import com.dtstack.flinkx.rdb.util.DbUtil;
import com.dtstack.flinkx.reader.MetaColumn;
-import com.dtstack.flinkx.util.*;
import com.dtstack.flinkx.restore.FormatState;
-import com.dtstack.flinkx.util.ClassUtil;
-import com.dtstack.flinkx.util.DateUtil;
-import com.dtstack.flinkx.util.StringUtil;
-import com.dtstack.flinkx.util.UrlUtil;
+import com.dtstack.flinkx.util.*;
import com.google.gson.Gson;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.accumulators.LongMaximum;
@@ -48,7 +44,9 @@
import java.io.IOException;
import java.sql.*;
import java.util.Date;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
@@ -161,7 +159,9 @@ public void openInternal(InputSplit inputSplit) throws IOException {
initMetric(inputSplit);
String startLocation = incrementConfig.getStartLocation();
if (incrementConfig.isPolling()) {
- endLocationAccumulator.add(Long.parseLong(startLocation));
+ if (StringUtils.isNotEmpty(startLocation)) {
+ endLocationAccumulator.add(Long.parseLong(startLocation));
+ }
isTimestamp = "timestamp".equalsIgnoreCase(incrementConfig.getColumnType());
} else if ((incrementConfig.isIncrement() && incrementConfig.isUseMaxFunc())) {
getMaxValue(inputSplit);
@@ -252,6 +252,12 @@ public Row nextRecordInternal(Row row) throws IOException {
boolean isUpdateLocation = incrementConfig.isPolling() || (incrementConfig.isIncrement() && !incrementConfig.isUseMaxFunc());
if (isUpdateLocation) {
Object incrementVal = resultSet.getObject(incrementConfig.getColumnName());
+ if(incrementVal != null) {
+ if((incrementVal instanceof java.util.Date
+ || incrementVal.getClass().getSimpleName().toUpperCase().contains("TIMESTAMP")) ) {
+ incrementVal = resultSet.getTimestamp(incrementConfig.getColumnName());
+ }
+ }
String location;
if(incrementConfig.isPolling()){
location = String.valueOf(incrementVal);
@@ -371,7 +377,7 @@ public String getMaxValueFromApi(){
}
String url = monitorUrls;
- if (monitorUrls.startsWith("http")) {
+ if (monitorUrls.startsWith(ConstantValue.PROTOCOL_HTTP)) {
url = String.format("%s/jobs/%s/accumulators", monitorUrls, jobId);
}
@@ -707,10 +713,10 @@ private String getLocation(String columnType, Object columnVal) {
long time = ((Timestamp) columnVal).getTime() / 1000;
String nanosStr = String.valueOf(((Timestamp) columnVal).getNanos());
- if (nanosStr.length() == 9) {
+ if (nanosStr.length() == DbUtil.NANOS_PART_LENGTH) {
location = time + nanosStr;
} else {
- String fillZeroStr = StringUtils.repeat("0", 9 - nanosStr.length());
+ String fillZeroStr = StringUtils.repeat("0", DbUtil.NANOS_PART_LENGTH - nanosStr.length());
location = time + fillZeroStr + nanosStr;
}
} else {
@@ -808,7 +814,7 @@ protected void executeQuery(String startLocation) throws SQLException {
queryForPolling(startLocation);
}
} else {
- Statement statement = dbConn.createStatement(resultSetType, resultSetConcurrency);
+ statement = dbConn.createStatement(resultSetType, resultSetConcurrency);
statement.setFetchSize(fetchSize);
statement.setQueryTimeout(queryTimeOut);
resultSet = statement.executeQuery(querySql);
diff --git a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/JdbcInputFormatBuilder.java b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/JdbcInputFormatBuilder.java
index cb48f98b59..ae788328f2 100644
--- a/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/JdbcInputFormatBuilder.java
+++ b/flinkx-rdb/flinkx-rdb-reader/src/main/java/com.dtstack.flinkx.rdb.inputformat/JdbcInputFormatBuilder.java
@@ -18,6 +18,7 @@
package com.dtstack.flinkx.rdb.inputformat;
+import com.dtstack.flinkx.constants.ConstantValue;
import com.dtstack.flinkx.inputformat.BaseRichInputFormatBuilder;
import com.dtstack.flinkx.rdb.DatabaseInterface;
import com.dtstack.flinkx.rdb.datareader.IncrementConfig;
@@ -128,6 +129,10 @@ protected void checkFormat() {
if (StringUtils.isEmpty(format.splitKey) && format.numPartitions > 1){
throw new IllegalArgumentException("Must specify the split column when the channel is greater than 1");
}
+
+ if (format.fetchSize > ConstantValue.MAX_BATCH_SIZE) {
+ throw new IllegalArgumentException("批量读取条数必须小于[200000]条");
+ }
}
}
diff --git a/flinkx-rdb/flinkx-rdb-writer/src/main/java/com.dtstack.flinkx.rdb.datawriter/JdbcDataWriter.java b/flinkx-rdb/flinkx-rdb-writer/src/main/java/com.dtstack.flinkx.rdb.datawriter/JdbcDataWriter.java
index 2a94afb8ff..a07eb3473c 100644
--- a/flinkx-rdb/flinkx-rdb-writer/src/main/java/com.dtstack.flinkx.rdb.datawriter/JdbcDataWriter.java
+++ b/flinkx-rdb/flinkx-rdb-writer/src/main/java/com.dtstack.flinkx.rdb.datawriter/JdbcDataWriter.java
@@ -32,7 +32,15 @@
import java.util.List;
import java.util.Map;
-import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.*;
+import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.KEY_BATCH_SIZE;
+import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.KEY_FULL_COLUMN;
+import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.KEY_INSERT_SQL_MODE;
+import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.KEY_PASSWORD;
+import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.KEY_POST_SQL;
+import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.KEY_PRE_SQL;
+import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.KEY_UPDATE_KEY;
+import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.KEY_USERNAME;
+import static com.dtstack.flinkx.rdb.datawriter.JdbcConfigKeys.KEY_WRITE_MODE;
/**
* The Writer plugin for any database that can be connected via JDBC.
diff --git a/flinkx-rdb/flinkx-rdb-writer/src/main/java/com/dtstack/flinkx/rdb/outputformat/JdbcOutputFormat.java b/flinkx-rdb/flinkx-rdb-writer/src/main/java/com/dtstack/flinkx/rdb/outputformat/JdbcOutputFormat.java
index bd2dc5afe1..47ab767ef3 100644
--- a/flinkx-rdb/flinkx-rdb-writer/src/main/java/com/dtstack/flinkx/rdb/outputformat/JdbcOutputFormat.java
+++ b/flinkx-rdb/flinkx-rdb-writer/src/main/java/com/dtstack/flinkx/rdb/outputformat/JdbcOutputFormat.java
@@ -30,12 +30,19 @@
import com.google.gson.Gson;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ObjectUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.flink.types.Row;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.sql.*;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -107,6 +114,7 @@ public class JdbcOutputFormat extends BaseRichOutputFormat {
"AND t.table_name = '%s'";
protected final static String CONN_CLOSE_ERROR_MSG = "No operations allowed";
+ protected static List STRING_TYPES = Arrays.asList("CHAR", "VARCHAR","TINYBLOB","TINYTEXT","BLOB","TEXT", "MEDIUMBLOB", "MEDIUMTEXT", "LONGBLOB", "LONGTEXT");
protected PreparedStatement prepareTemplates() throws SQLException {
if(CollectionUtils.isEmpty(fullColumn)) {
@@ -202,7 +210,13 @@ protected void writeSingleRecordInternal(Row row) throws WriteRecordException {
int index = 0;
try {
for (; index < row.getArity(); index++) {
- preparedStatement.setObject(index+1,getField(row,index));
+ Object object = row.getField(index);
+ if( object instanceof String && StringUtils.isBlank((String) object)){
+ if(!STRING_TYPES.contains(columnType.get(index))){
+ object = null;
+ }
+ }
+ preparedStatement.setObject(index+1, object);
}
preparedStatement.execute();
@@ -233,8 +247,14 @@ protected String recordConvertDetailErrorMessage(int pos, Row row) {
protected void writeMultipleRecordsInternal() throws Exception {
try {
for (Row row : rows) {
- for (int j = 0; j < row.getArity(); ++j) {
- preparedStatement.setObject(j + 1, getField(row, j));
+ for (int index = 0; index < row.getArity(); index++) {
+ Object object = row.getField(index);
+ if( object instanceof String && StringUtils.isBlank((String) object)){
+ if(!STRING_TYPES.contains(columnType.get(index))){
+ object = null;
+ }
+ }
+ preparedStatement.setObject(index+1, object);
}
preparedStatement.addBatch();
diff --git a/flinkx-redis/flinkx-redis-core/src/main/java/com/dtstack/flinkx/redis/JedisUtil.java b/flinkx-redis/flinkx-redis-core/src/main/java/com/dtstack/flinkx/redis/JedisUtil.java
index 31f40ecb64..9aed6f527b 100644
--- a/flinkx-redis/flinkx-redis-core/src/main/java/com/dtstack/flinkx/redis/JedisUtil.java
+++ b/flinkx-redis/flinkx-redis-core/src/main/java/com/dtstack/flinkx/redis/JedisUtil.java
@@ -19,13 +19,18 @@
package com.dtstack.flinkx.redis;
import com.dtstack.flinkx.util.TelnetUtil;
-import redis.clients.jedis.*;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.JedisPoolConfig;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import static com.dtstack.flinkx.redis.RedisConfigKeys.*;
+import static com.dtstack.flinkx.redis.RedisConfigKeys.KEY_DB;
+import static com.dtstack.flinkx.redis.RedisConfigKeys.KEY_HOST_PORT;
+import static com.dtstack.flinkx.redis.RedisConfigKeys.KEY_PASSWORD;
+import static com.dtstack.flinkx.redis.RedisConfigKeys.KEY_TIMEOUT;
/**
* Utilities for redis database connection
diff --git a/flinkx-redis/flinkx-redis-writer/pom.xml b/flinkx-redis/flinkx-redis-writer/pom.xml
index 4a92436c6b..891f05acf5 100644
--- a/flinkx-redis/flinkx-redis-writer/pom.xml
+++ b/flinkx-redis/flinkx-redis-writer/pom.xml
@@ -56,8 +56,12 @@
shade.rediswriter.io.netty
- com.google
- shade.rediswriter.com.google
+ com.google.common
+ shade.core.com.google.common
+
+
+ com.google.thirdparty
+ shade.core.com.google.thirdparty
diff --git a/flinkx-redis/flinkx-redis-writer/src/main/java/com/dtstack/flinkx/redis/writer/RedisOutputFormat.java b/flinkx-redis/flinkx-redis-writer/src/main/java/com/dtstack/flinkx/redis/writer/RedisOutputFormat.java
index 7fe2fe9358..2fc85d1fd3 100644
--- a/flinkx-redis/flinkx-redis-writer/src/main/java/com/dtstack/flinkx/redis/writer/RedisOutputFormat.java
+++ b/flinkx-redis/flinkx-redis-writer/src/main/java/com/dtstack/flinkx/redis/writer/RedisOutputFormat.java
@@ -30,9 +30,15 @@
import java.io.IOException;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
-import static com.dtstack.flinkx.redis.RedisConfigKeys.*;
+import static com.dtstack.flinkx.redis.RedisConfigKeys.KEY_DB;
+import static com.dtstack.flinkx.redis.RedisConfigKeys.KEY_HOST_PORT;
+import static com.dtstack.flinkx.redis.RedisConfigKeys.KEY_PASSWORD;
+import static com.dtstack.flinkx.redis.RedisConfigKeys.KEY_TIMEOUT;
/**
* OutputFormat for writing data to redis database.
@@ -70,6 +76,8 @@ public class RedisOutputFormat extends BaseRichOutputFormat {
private static final int CRITICAL_TIME = 60 * 60 * 24 * 30;
+ private static final int KEY_VALUE_SIZE = 2;
+
@Override
public void configure(Configuration parameters) {
super.configure(parameters);
@@ -138,7 +146,7 @@ private void processTimeFormat(Row row){
}
private List
+
+ org.jacoco
+ jacoco-maven-plugin
+ 0.7.8
+
+
+
+ prepare-agent
+ report
+
+
+
+
+
org.apache.maven.plugins
maven-source-plugin
+ 3.2.1
attach-sources
@@ -168,4 +190,20 @@
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+
+
+
+ report
+
+
+
+
+
+
+
\ No newline at end of file