Skip to content

Commit 4537147

Browse files
imperio-wxmimperio-wxm
committed
update: spark version 3.x
1 parent 48afb33 commit 4537147

File tree

6 files changed

+141
-60
lines changed

6 files changed

+141
-60
lines changed

‎spark-best-practice/.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
.idea
2+
*.iml
3+
target
4+
*.iml
5+
*.log*

‎spark-best-practice/pom.xml

Lines changed: 21 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@
1111
<modules>
1212
<module>simple-demo</module>
1313
<module>spark-phoenix</module>
14-
<module>spark-springboot</module>
15-
<module>structured-streaming</module>
1614
</modules>
1715

1816
<parent>
@@ -32,6 +30,11 @@
3230
</properties>
3331

3432
<dependencies>
33+
<dependency>
34+
<groupId>com.thoughtworks.paranamer</groupId>
35+
<artifactId>paranamer</artifactId>
36+
<version>2.8</version>
37+
</dependency>
3538

3639
<dependency>
3740
<groupId>io.delta</groupId>
@@ -66,69 +69,59 @@
6669
<dependency>
6770
<groupId>org.scala-lang</groupId>
6871
<artifactId>scala-library</artifactId>
69-
<version>2.11.12</version>
72+
<version>2.12.15</version>
7073
</dependency>
7174

7275
<dependency>
7376
<groupId>org.apache.spark</groupId>
74-
<artifactId>spark-sql_2.11</artifactId>
75-
<!--<version>2.2.0.cloudera1</version>-->
76-
<version>2.4.0.cloudera2</version>
77+
<artifactId>spark-sql_2.12</artifactId>
78+
<version>3.2.1</version>
7779
</dependency>
7880

7981
<dependency>
8082
<groupId>org.apache.spark</groupId>
81-
<artifactId>spark-core_2.11</artifactId>
82-
<!-- <version>2.2.0.cloudera1</version>-->
83-
<version>2.4.0.cloudera2</version>
83+
<artifactId>spark-core_2.12</artifactId>
84+
<version>3.2.1</version>
8485
</dependency>
8586

8687

8788
<dependency>
8889
<groupId>org.apache.spark</groupId>
89-
<artifactId>spark-avro_2.11</artifactId>
90-
<version>2.4.0.cloudera2</version>
90+
<artifactId>spark-avro_2.12</artifactId>
91+
<version>3.2.1</version>
9192
</dependency>
9293

9394
<dependency>
9495
<groupId>org.apache.spark</groupId>
95-
<artifactId>spark-sql-kafka-0-10_2.11</artifactId>
96-
<version>2.4.0.cloudera2</version>
96+
<artifactId>spark-sql-kafka-0-10_2.12</artifactId>
97+
<version>3.2.1</version>
9798
</dependency>
9899

99-
100100
<dependency>
101101
<groupId>org.apache.hadoop</groupId>
102102
<artifactId>hadoop-client</artifactId>
103-
<version>2.6.0-cdh5.11.1</version>
103+
<version>3.3.3</version>
104104
<exclusions>
105105
<exclusion>
106-
<artifactId>guava</artifactId>
107-
<groupId>com.google.guava</groupId>
108-
</exclusion>
109-
<exclusion>
110-
<artifactId>servlet-api</artifactId>
111-
<groupId>javax.servlet</groupId>
106+
<artifactId>slf4j-reload4j</artifactId>
107+
<groupId>org.slf4j</groupId>
112108
</exclusion>
113109
</exclusions>
114110
</dependency>
115111

116112
<dependency>
117113
<groupId>org.apache.hadoop</groupId>
118114
<artifactId>hadoop-common</artifactId>
119-
<version>2.6.0-cdh5.11.1</version>
115+
<version>3.3.3</version>
120116
<exclusions>
121117
<exclusion>
122-
<artifactId>guava</artifactId>
123-
<groupId>com.google.guava</groupId>
124-
</exclusion>
125-
<exclusion>
126-
<artifactId>servlet-api</artifactId>
127-
<groupId>javax.servlet</groupId>
118+
<artifactId>slf4j-reload4j</artifactId>
119+
<groupId>org.slf4j</groupId>
128120
</exclusion>
129121
</exclusions>
130122
</dependency>
131123

124+
132125
<dependency>
133126
<groupId>junit</groupId>
134127
<artifactId>junit</artifactId>
@@ -143,21 +136,6 @@
143136
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
144137
</repository>
145138

146-
<repository>
147-
<id>nexus-public</id>
148-
<url>http://nexus.dw.sdo.com/content/groups/public/</url>
149-
</repository>
150-
151-
<repository>
152-
<id>dw-releases</id>
153-
<url>http://nexus.dw.sdo.com/content/repositories/dw-releases/</url>
154-
</repository>
155-
156-
<repository>
157-
<id>dw-snapshots</id>
158-
<url>http://nexus.dw.sdo.com/content/repositories/dw-snapshots/</url>
159-
</repository>
160-
161139
<repository>
162140
<id>SparkPackagesRepo</id>
163141
<url>http://dl.bintray.com/spark-packages/maven</url>

‎spark-best-practice/simple-demo/pom.xml

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@
6868
</build>
6969

7070

71-
7271
<dependencies>
7372
<dependency>
7473
<groupId>junit</groupId>
@@ -77,12 +76,29 @@
7776
<scope>test</scope>
7877
</dependency>
7978

79+
<dependency>
80+
<groupId>org.apache.commons</groupId>
81+
<artifactId>commons-lang3</artifactId>
82+
<version>3.12.0</version>
83+
</dependency>
84+
8085
<dependency>
8186
<groupId>org.apache.hadoop</groupId>
8287
<artifactId>hadoop-common</artifactId>
83-
<version>2.6.0-cdh5.11.1</version>
88+
<version>3.3.3</version>
89+
<exclusions>
90+
<exclusion>
91+
<artifactId>slf4j-reload4j</artifactId>
92+
<groupId>org.slf4j</groupId>
93+
</exclusion>
94+
<exclusion>
95+
<groupId>io.netty</groupId>
96+
<artifactId>netty-all</artifactId>
97+
</exclusion>
98+
</exclusions>
8499
</dependency>
85100

101+
86102
<dependency>
87103
<groupId>mysql</groupId>
88104
<artifactId>mysql-connector-java</artifactId>
@@ -91,6 +107,17 @@
91107

92108
</dependencies>
93109

110+
111+
<dependencyManagement>
112+
<dependencies>
113+
<dependency>
114+
<groupId>io.netty</groupId>
115+
<artifactId>netty-all</artifactId>
116+
<version>4.1.77.Final</version>
117+
</dependency>
118+
</dependencies>
119+
</dependencyManagement>
120+
94121
<repositories>
95122
<repository>
96123
<id>cloudera</id>

‎spark-best-practice/simple-demo/src/main/java/com/wxmimperio/spark/SparkSequenceFileRead.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package com.wxmimperio.spark;
22

3-
import org.apache.commons.lang.StringUtils;
3+
import org.apache.commons.lang3.StringUtils;
44
import org.apache.hadoop.conf.Configuration;
55
import org.apache.hadoop.io.Text;
66
import org.apache.hadoop.mapred.SequenceFileOutputFormat;

‎spark-best-practice/simple-demo/src/test/java/com/wxmimperio/spark/Test.java

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,16 @@
44

55
import java.sql.Timestamp;
66
import java.text.SimpleDateFormat;
7+
import java.util.Arrays;
8+
import java.util.HashMap;
9+
import java.util.Map;
10+
import java.util.Set;
11+
import java.util.function.IntFunction;
12+
import java.util.stream.Collectors;
13+
import java.util.stream.IntStream;
14+
import java.util.stream.Stream;
15+
16+
import static java.lang.Integer.parseInt;
717

818
public class Test {
919

@@ -17,5 +27,80 @@ public void test001() throws Exception {
1727

1828
System.out.println(System.currentTimeMillis());
1929

30+
Integer a = Integer.valueOf(1);
31+
a.intValue();
32+
33+
int i = 45;
34+
if (i >= IntegerCache.low && i <= IntegerCache.high) {
35+
System.out.println(IntegerCache.cache[i + (-IntegerCache.low)]);
36+
}
37+
38+
}
39+
40+
private static class IntegerCache {
41+
static final int low = -128;
42+
static final int high;
43+
static final Integer cache[];
44+
45+
static {
46+
// high value may be configured by property
47+
int h = 127;
48+
String integerCacheHighPropValue =
49+
sun.misc.VM.getSavedProperty("java.lang.Integer.IntegerCache.high");
50+
if (integerCacheHighPropValue != null) {
51+
try {
52+
int i = parseInt(integerCacheHighPropValue);
53+
i = Math.max(i, 127);
54+
// Maximum array size is Integer.MAX_VALUE
55+
h = Math.min(i, Integer.MAX_VALUE - (-low) - 1);
56+
} catch (NumberFormatException nfe) {
57+
// If the property cannot be parsed into an int, ignore it.
58+
}
59+
}
60+
high = h;
61+
62+
cache = new Integer[(high - low) + 1];
63+
int j = low;
64+
for (int k = 0; k < cache.length; k++)
65+
cache[k] = new Integer(j++);
66+
67+
// range [-128, 127] must be interned (JLS7 5.1.7)
68+
assert IntegerCache.high >= 127;
69+
}
70+
71+
private IntegerCache() {
72+
}
73+
}
74+
75+
@org.junit.Test
76+
public void testLeecode() {
77+
int[] nums = new int[]{2, 7, 11, 15};
78+
System.out.println(Arrays.toString(twoSum(nums, 9)));
79+
80+
System.out.println(Arrays.toString(twoSum2(nums, 9)));
81+
}
82+
83+
public int[] twoSum(int[] nums, int target) {
84+
int[] result = new int[2];
85+
for (int i = 0; i < nums.length; i++) {
86+
for (int j = i + 1; j < nums.length; j++) {
87+
if (nums[i] + nums[j] == target) {
88+
result = new int[]{i, j};
89+
}
90+
}
91+
}
92+
return result;
93+
}
94+
95+
public int[] twoSum2(int[] nums, int target) {
96+
Map<Integer, Integer> map = new HashMap<>();
97+
for (int i = 0; i < nums.length; i++) {
98+
int sub = target - nums[i];
99+
if (map.containsKey(sub)) {
100+
return new int[]{map.get(sub), i};
101+
}
102+
map.put(nums[i], i);
103+
}
104+
return nums;
20105
}
21106
}

‎spark-best-practice/spark-phoenix/pom.xml

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,20 +35,6 @@
3535
<artifactId>phoenix-spark</artifactId>
3636
<version>4.14.0-cdh5.11.2</version>
3737
</dependency>-->
38-
39-
40-
<dependency>
41-
<groupId>org.apache.hadoop</groupId>
42-
<artifactId>hadoop-common</artifactId>
43-
<version>2.6.0-cdh5.11.1</version>
44-
</dependency>
45-
46-
<dependency>
47-
<groupId>org.apache.phoenix</groupId>
48-
<artifactId>phoenix-client</artifactId>
49-
<version>4.14.0-cdh5.11.2-sdg-1.0.2-RELEASE</version>
50-
<classifier>client</classifier>
51-
</dependency>
5238
</dependencies>
5339

5440
<!-- <build>

0 commit comments

Comments
 (0)