Upgrade Spark to 3.4.3 and Hive to 3.1.3 (#2231)

* Upgrade Spark to 3.4.3 and Hive to 3.1.3

* Update shas

Author: jbaiera

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/HiveServiceDescriptor.groovy

@@ -60,7 +60,7 @@ class HiveServiceDescriptor implements ServiceDescriptor {
 
     @Override
     Version defaultVersion() {
-        return new Version(3, 1, 2)
+        return new Version(3, 1, 3)
     }
 
     @Override

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy

@@ -56,7 +56,7 @@ class SparkYarnServiceDescriptor implements ServiceDescriptor {
 
     @Override
     Version defaultVersion() {
-        return new Version(3, 4, 2)
+        return new Version(3, 4, 3)
     }
 
     String hadoopVersionCompatibility() {

dist/licenses/hive-exec-3.1.2.jar.sha1

@@ -0,0 +1 @@
+6cf7f97e1438f94f196b7fb480d5d985a04336a9

dist/licenses/hive-exec-3.1.3.jar.sha1

@@ -0,0 +1 @@
+99a1348743f3550dd4524408725efab8eb319960

dist/licenses/hive-metastore-3.1.2.jar.sha1

@@ -0,0 +1 @@
+49fd7ad46801a895d2b185706637a8cb302fc044

dist/licenses/hive-metastore-3.1.3.jar.sha1

@@ -20,7 +20,7 @@ hadoop2Version  = 2.7.6
 hadoop22Version = 2.2.0
 
 # Common libraries
-hiveVersion = 3.1.2
+hiveVersion = 3.1.3
 # note the versions here are tied to the ones in Hadoop distro - 1.8.8
 jacksonVersion = 1.8.8
 
@@ -29,7 +29,7 @@ spark13Version = 1.6.2
 spark20Version = 2.3.0
 spark22Version = 2.2.3
 spark24Version = 2.4.4
-spark30Version = 3.4.2
+spark30Version = 3.4.3
 
 # same as Spark's
 scala210Version = 2.10.7
@@ -45,7 +45,7 @@ scala213MajorVersion = 2.13
 junitVersion = 4.11
 mockitoVersion = 1.8.5
 hamcrestVersion = 1.3
-minikdcVersion = 2.7.7
+minikdcVersion = 3.4.0
 antlrVersion = 3.4
 thriftVersion = 0.5.0
 

dist/licenses/hive-service-3.1.2.jar.sha1

@@ -0,0 +1 @@
+6cf7f97e1438f94f196b7fb480d5d985a04336a9

dist/licenses/hive-service-3.1.3.jar.sha1

@@ -0,0 +1 @@
+99a1348743f3550dd4524408725efab8eb319960

gradle.properties

@@ -0,0 +1 @@
+49fd7ad46801a895d2b185706637a8cb302fc044

hive/licenses/hive-exec-3.1.2.jar.sha1

@@ -0,0 +1 @@
+6cf7f97e1438f94f196b7fb480d5d985a04336a9

hive/licenses/hive-exec-3.1.3.jar.sha1

@@ -0,0 +1 @@
+99a1348743f3550dd4524408725efab8eb319960

hive/licenses/hive-metastore-3.1.2.jar.sha1

@@ -0,0 +1 @@
+49fd7ad46801a895d2b185706637a8cb302fc044

hive/licenses/hive-metastore-3.1.3.jar.sha1

@@ -0,0 +1 @@
+86d1ed1f60a0e5855aeca82e18d04b3acd410439

hive/licenses/hive-service-3.1.2.jar.sha1

@@ -0,0 +1 @@
+6f7105e792cbb650eca44890f0444720bd6a8204

hive/licenses/hive-service-3.1.3.jar.sha1

@@ -0,0 +1 @@
+a28ead1aa19899654296a6f8a458f8f207f89a73

licenses/hive-exec-3.1.2.jar.sha1

@@ -0,0 +1 @@
+e4a9f76eff1db09c67f855cdf60cd83bbfad69ab

licenses/hive-exec-3.1.3.jar.sha1

@@ -0,0 +1 @@
+0c8dd1e45fbc589d2438a889c7fe98f0e9fd77ec

licenses/hive-metastore-3.1.2.jar.sha1

@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Notice:
+# Modified for keytab login - Changes listed between ++Change and ==Change comments.
+# Previous lines commented out.
+
+# Need arguments [host [port [db]]]
+THISSERVICE=beeline
+export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
+
+beeline () {
+  # ++Change the class to SecureBeeline wrapper
+  # CLASS=org.apache.hive.beeline.BeeLine;
+  CLASS=org.elasticsearch.hadoop.qa.kerberos.hive.SecureBeeline;
+  BEELINECLASS=org.apache.hive.beeline.BeeLine;
+
+  # add our test jar path for ES-Hadoop
+  testJarPath=`ls ${TEST_LIB}`
+  # ==Change
+
+  # include only the beeline client jar and its dependencies
+  beelineJarPath=`ls ${HIVE_LIB}/hive-beeline-*.jar`
+  superCsvJarPath=`ls ${HIVE_LIB}/super-csv-*.jar`
+  jlineJarPath=`ls ${HIVE_LIB}/jline-*.jar`
+  hadoopClasspath=""
+  if [[ -n "${HADOOP_CLASSPATH}" ]]
+  then
+    hadoopClasspath="${HADOOP_CLASSPATH}:"
+  fi
+  # ++Change the classpath to include our test jar
+  # export HADOOP_CLASSPATH="${hadoopClasspath}${HIVE_CONF_DIR}:${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}"
+  export HADOOP_CLASSPATH="${hadoopClasspath}${HIVE_CONF_DIR}:${testJarPath}:${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}"
+  # ==Change
+  export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.properties "
+
+  # ++Change the execution to run our class which will in turn run the Beeline class
+  # exec $HADOOP jar ${beelineJarPath} $CLASS $HIVE_OPTS "$@"
+  exec $HADOOP jar ${testJarPath} $CLASS $BEELINECLASS $HIVE_OPTS "$@"
+  # ==Change
+}
+
+beeline_help () {
+  beeline "--help"
+}

licenses/hive-metastore-3.1.3.jar.sha1

@@ -0,0 +1 @@
+86d1ed1f60a0e5855aeca82e18d04b3acd410439

licenses/hive-service-3.1.2.jar.sha1

@@ -0,0 +1 @@
+6f7105e792cbb650eca44890f0444720bd6a8204

licenses/hive-service-3.1.3.jar.sha1

@@ -0,0 +1 @@
+a28ead1aa19899654296a6f8a458f8f207f89a73

licenses/spark-catalyst_2.12-3.4.2.jar.sha1

@@ -0,0 +1 @@
+e4a9f76eff1db09c67f855cdf60cd83bbfad69ab

licenses/spark-catalyst_2.12-3.4.3.jar.sha1

@@ -0,0 +1 @@
+0c8dd1e45fbc589d2438a889c7fe98f0e9fd77ec