Skip to content

Commit 987ab40

Browse files
samxbrsamxbr
authored
Set Spark 3.x default to Scala 2.13 (#2328)
1 parent 5d2ff74 commit 987ab40

File tree

51 files changed

+45
-44
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+45
-44
lines changed

‎buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy

+2-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ class SparkYarnServiceDescriptor implements ServiceDescriptor {
7878
@Override
7979
String artifactName(ServiceConfiguration configuration) {
8080
Version version = configuration.getVersion()
81-
return "spark-$version-bin-hadoop${hadoopVersionCompatibility()}"
81+
// artifact name: https://spark.apache.org/downloads.html
82+
return "spark-$version-bin-hadoop${hadoopVersionCompatibility()}-scala2.13"
8283
}
8384

8485
@Override

‎dist/build.gradle

+8-8
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ base {
1212
archivesName = 'elasticsearch-hadoop'
1313
}
1414

15-
def sparkVariantIncluded = 'spark30scala212'
15+
def sparkVariantIncluded = 'spark30scala213'
1616

1717
configurations {
1818
embedded {
@@ -86,25 +86,25 @@ dependencies {
8686
}
8787
implementation("org.apache.hive:hive-exec:$hiveVersion")
8888
implementation("org.apache.hive:hive-metastore:$hiveVersion")
89-
implementation("org.apache.spark:spark-core_${project.ext.scala212MajorVersion}:$spark30Version") {
89+
implementation("org.apache.spark:spark-core_${project.ext.scala213MajorVersion}:$spark30Version") {
9090
exclude group: 'javax.servlet'
9191
exclude group: 'org.apache.hadoop'
9292
}
93-
implementation("org.apache.spark:spark-yarn_${project.ext.scala212MajorVersion}:$spark30Version") {
93+
implementation("org.apache.spark:spark-yarn_${project.ext.scala213MajorVersion}:$spark30Version") {
9494
exclude group: 'org.apache.hadoop'
9595
}
96-
implementation("org.apache.spark:spark-sql_${project.ext.scala212MajorVersion}:$spark30Version") {
96+
implementation("org.apache.spark:spark-sql_${project.ext.scala213MajorVersion}:$spark30Version") {
9797
exclude group: 'org.apache.hadoop'
9898
}
99-
implementation("org.apache.spark:spark-streaming_${project.ext.scala212MajorVersion}:$spark30Version") {
99+
implementation("org.apache.spark:spark-streaming_${project.ext.scala213MajorVersion}:$spark30Version") {
100100
exclude group: 'org.apache.hadoop'
101101
}
102-
implementation("org.scala-lang:scala-library:$scala211Version")
103-
implementation("org.scala-lang:scala-reflect:$scala211Version")
102+
implementation("org.scala-lang:scala-library:$scala213Version")
103+
implementation("org.scala-lang:scala-reflect:$scala213Version")
104104
implementation(project.ext.hadoopClient)
105105
implementation("org.apache.hadoop:hadoop-common:${project.ext.hadoopVersion}")
106106
implementation("org.apache.hadoop:hadoop-mapreduce-client-core:${project.ext.hadoopVersion}")
107-
compileOnly("org.apache.spark:spark-catalyst_${project.ext.scala212MajorVersion}:$spark30Version")
107+
compileOnly("org.apache.spark:spark-catalyst_${project.ext.scala213MajorVersion}:$spark30Version")
108108
}
109109

110110
// Configure uber jar

‎dist/licenses/scala-library-2.11.12.jar.sha1

-1
This file was deleted.

‎dist/licenses/scala-library-2.13.13.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3b1434403ddb147b8030f98f246153e4ce622c78

‎dist/licenses/scala-reflect-2.11.12.jar.sha1

-1
This file was deleted.

‎dist/licenses/scala-reflect-2.13.13.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
46866f0e2cf6168322dc11f0f721c2db78d52ac7

‎dist/licenses/spark-core_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎dist/licenses/spark-core_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
a8055d61a13e9af6871e05ecdca141134d102f1f

‎dist/licenses/spark-sql_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎dist/licenses/spark-sql_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
b1744080f5e2f81b9a1a088eb375d33b85132b6d

‎dist/licenses/spark-streaming_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎dist/licenses/spark-streaming_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
64c4e38f3a89428926383f4d9f8a8636a81d499b

‎dist/licenses/spark-yarn_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎dist/licenses/spark-yarn_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
d8fd5251bb23d3bed9d9ef1f19c4402211c78fe4

‎licenses/scala-library-2.12.19.jar.sha1

-1
This file was deleted.

‎licenses/scala-library-2.13.13.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3b1434403ddb147b8030f98f246153e4ce622c78

‎licenses/scala-reflect-2.12.19.jar.sha1

-1
This file was deleted.

‎licenses/scala-reflect-2.13.13.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
46866f0e2cf6168322dc11f0f721c2db78d52ac7

‎licenses/spark-catalyst_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎licenses/spark-catalyst_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
6a9e9c5e149bd019dd5d781888e1e81ca4d32875

‎licenses/spark-core_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎licenses/spark-core_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
a8055d61a13e9af6871e05ecdca141134d102f1f

‎licenses/spark-sql_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎licenses/spark-sql_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
b1744080f5e2f81b9a1a088eb375d33b85132b6d

‎licenses/spark-streaming_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎licenses/spark-streaming_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
64c4e38f3a89428926383f4d9f8a8636a81d499b

‎licenses/spark-yarn_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎licenses/spark-yarn_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
d8fd5251bb23d3bed9d9ef1f19c4402211c78fe4

‎qa/kerberos/build.gradle

+8-8
Original file line numberDiff line numberDiff line change
@@ -63,22 +63,22 @@ configurations {
6363
dependencies {
6464
implementation(project(":elasticsearch-hadoop-mr"))
6565

66-
implementation("org.scala-lang:scala-library:${scala212Version}")
67-
implementation("org.scala-lang:scala-reflect:${scala212Version}")
66+
implementation("org.scala-lang:scala-library:${scala213Version}")
67+
implementation("org.scala-lang:scala-reflect:${scala213Version}")
6868
implementation(project(":elasticsearch-spark-30")) {
6969
capabilities {
70-
// Spark 3.x on Scala 2.12
71-
requireCapability("org.elasticsearch.spark.sql.variant:spark30scala212:$project.version")
70+
// Spark 3.x on Scala 2.13
71+
requireCapability("org.elasticsearch.spark.sql.variant:spark30scala213:$project.version")
7272
}
7373
}
7474

75-
compileOnly("com.fasterxml.jackson.module:jackson-module-scala_2.12:2.6.7.1")
76-
compileOnly("com.fasterxml.jackson.core:jackson-annotations:2.6.7")
77-
compileOnly("org.json4s:json4s-jackson_2.12:3.2.11")
75+
compileOnly("com.fasterxml.jackson.module:jackson-module-scala_${scala213MajorVersion}:2.9.10")
76+
compileOnly("com.fasterxml.jackson.core:jackson-annotations:2.9.10")
77+
compileOnly("org.json4s:json4s-jackson_${scala213MajorVersion}:3.6.7")
7878
compileOnly("org.slf4j:slf4j-api:${project.ext.slf4jVersion}")
7979

8080
implementation("org.apache.hadoop:hadoop-client:${HadoopClusterConfiguration.HADOOP.defaultVersion()}")
81-
implementation("org.apache.spark:spark-sql_2.12:$project.ext.spark30Version")
81+
implementation("org.apache.spark:spark-sql_${scala213MajorVersion}:$project.ext.spark30Version")
8282

8383
implementation( project(path: ':elasticsearch-hadoop-mr', configuration: 'itestJarConfig'))
8484

‎spark/core/build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ sparkVariants {
1212

1313
// Changing the formatting of these lines could break .buildkite/pipeline.py, it uses regex to parse the `spark30scala212` part
1414
// We should maybe move these to a separate config file that can be read from both this file and the pipeline script in the future if it creates issues
15-
setCoreDefaultVariant "spark30scala212", spark30Version, scala212Version
16-
addCoreFeatureVariant "spark30scala213", spark30Version, scala213Version
15+
setCoreDefaultVariant "spark30scala213", spark30Version, scala213Version
16+
addCoreFeatureVariant "spark30scala212", spark30Version, scala212Version
1717

1818
all { SparkVariantPlugin.SparkVariant variant ->
1919

‎spark/core/licenses/scala-library-2.12.19.jar.sha1

-1
This file was deleted.

‎spark/core/licenses/scala-library-2.13.13.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3b1434403ddb147b8030f98f246153e4ce622c78

‎spark/core/licenses/scala-reflect-2.12.19.jar.sha1

-1
This file was deleted.

‎spark/core/licenses/scala-reflect-2.13.13.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
46866f0e2cf6168322dc11f0f721c2db78d52ac7

‎spark/core/licenses/spark-core_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎spark/core/licenses/spark-core_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
a8055d61a13e9af6871e05ecdca141134d102f1f

‎spark/sql-30/build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ apply plugin: 'spark.variants'
99

1010
sparkVariants {
1111
capabilityGroup 'org.elasticsearch.spark.sql.variant'
12-
setDefaultVariant "spark30scala212", spark30Version, scala212Version
13-
addFeatureVariant "spark30scala213", spark30Version, scala213Version
12+
setDefaultVariant "spark30scala213", spark30Version, scala213Version
13+
addFeatureVariant "spark30scala212", spark30Version, scala212Version
1414

1515
all { SparkVariantPlugin.SparkVariant variant ->
1616
String scalaCompileTaskName = project.sourceSets

‎spark/sql-30/licenses/scala-library-2.12.19.jar.sha1

-1
This file was deleted.

‎spark/sql-30/licenses/scala-library-2.13.13.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3b1434403ddb147b8030f98f246153e4ce622c78

‎spark/sql-30/licenses/scala-reflect-2.12.19.jar.sha1

-1
This file was deleted.

‎spark/sql-30/licenses/scala-reflect-2.13.13.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
46866f0e2cf6168322dc11f0f721c2db78d52ac7

‎spark/sql-30/licenses/spark-catalyst_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎spark/sql-30/licenses/spark-catalyst_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
6a9e9c5e149bd019dd5d781888e1e81ca4d32875

‎spark/sql-30/licenses/spark-core_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎spark/sql-30/licenses/spark-core_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
a8055d61a13e9af6871e05ecdca141134d102f1f

‎spark/sql-30/licenses/spark-sql_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎spark/sql-30/licenses/spark-sql_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
b1744080f5e2f81b9a1a088eb375d33b85132b6d

‎spark/sql-30/licenses/spark-streaming_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎spark/sql-30/licenses/spark-streaming_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
64c4e38f3a89428926383f4d9f8a8636a81d499b

‎spark/sql-30/licenses/spark-yarn_2.12-3.4.3.jar.sha1

-1
This file was deleted.

‎spark/sql-30/licenses/spark-yarn_2.13-3.4.3.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
d8fd5251bb23d3bed9d9ef1f19c4402211c78fe4

0 commit comments

Comments
 (0)