Support Spark 3.0.2 (#833)

Author: suhsteve

azure-pipelines.yml

@@ -33,12 +33,16 @@ variables:
 
   # Filter DataFrameTests.TestDataFrameGroupedMapUdf and DataFrameTests.TestGroupedMapUdf backwardCompatible
   # tests due to https://github.com/dotnet/spark/pull/711
-  backwardCompatibleTestOptions_Windows_3: "--filter \
+  backwardCompatibleTestOptions_Windows_3_0: "--filter \
   (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\
   (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestGroupedMapUdf)"
-  forwardCompatibleTestOptions_Windows_3: ""
-  backwardCompatibleTestOptions_Linux_3: $(backwardCompatibleTestOptions_Windows_3)
-  forwardCompatibleTestOptions_Linux_3: $(forwardCompatibleTestOptions_Linux_2_4)
+  forwardCompatibleTestOptions_Windows_3_0: ""
+  backwardCompatibleTestOptions_Linux_3_0: $(backwardCompatibleTestOptions_Windows_3_0)
+  forwardCompatibleTestOptions_Linux_3_0: $(forwardCompatibleTestOptions_Linux_2_4)
+
+  # Skip all forward compatible tests since Spark 3.0.2 is not supported in microsoft-spark-3-0_2.12-1.0.0.jar
+  forwardCompatibleTestOptions_Windows_3_0_2: "--filter FullyQualifiedName=NONE"
+  forwardCompatibleTestOptions_Linux_3_0_2: $(forwardCompatibleTestOptions_Windows_3_0_2)
 
   # Azure DevOps variables are transformed into environment variables, with these variables we
   # avoid the first time experience and telemetry to speed up the build.
@@ -331,19 +335,29 @@ stages:
       jobOptions:
       - pool: 'Hosted VS2017'
         testOptions: ""
-        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3)
-        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3)
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0)
       - pool: 'Hosted Ubuntu 1604'
         testOptions: ""
-        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3)
-        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3)
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
     - version: '3.0.1'
       jobOptions:
       - pool: 'Hosted VS2017'
         testOptions: ""
-        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3)
-        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3)
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
+    - version: '3.0.2'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0_2)
       - pool: 'Hosted Ubuntu 1604'
         testOptions: ""
-        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3)
-        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3)
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0_2)

src/scala/microsoft-spark-2-3/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala

@@ -166,7 +166,7 @@ object DotnetRunner extends Logging {
 
   private def validateSparkVersions: Unit = {
     if (!supportedSparkVersions(SPARK_VERSION)) {
-      val supportedVersions = supportedSparkVersions.mkString(", ")
+      val supportedVersions = supportedSparkVersions.toSeq.sorted.mkString(", ")
       throw new IllegalArgumentException(
         s"Unsupported spark version used: ${spark.SPARK_VERSION}. Normalized spark version used: ${SPARK_VERSION}." +
           s" Supported versions: ${supportedVersions}")

src/scala/microsoft-spark-2-4/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala

@@ -167,7 +167,7 @@ object DotnetRunner extends Logging {
 
   private def validateSparkVersions: Unit = {
     if (!supportedSparkVersions(SPARK_VERSION)) {
-      val supportedVersions = supportedSparkVersions.mkString(", ")
+      val supportedVersions = supportedSparkVersions.toSeq.sorted.mkString(", ")
       throw new IllegalArgumentException(
         s"Unsupported spark version used: ${spark.SPARK_VERSION}. Normalized spark version used: ${SPARK_VERSION}." +
           s" Supported versions: ${supportedVersions}")

src/scala/microsoft-spark-3-0/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala

@@ -34,7 +34,7 @@ import scala.util.Try
  */
 object DotnetRunner extends Logging {
   private val DEBUG_PORT = 5567
-  private val supportedSparkVersions = Set[String]("3.0.0", "3.0.1")
+  private val supportedSparkVersions = Set[String]("3.0.0", "3.0.1", "3.0.2")
 
   val SPARK_VERSION = DotnetUtils.normalizeSparkVersion(spark.SPARK_VERSION)
 
@@ -166,7 +166,7 @@ object DotnetRunner extends Logging {
 
   private def validateSparkVersions: Unit = {
     if (!supportedSparkVersions(SPARK_VERSION)) {
-      val supportedVersions = supportedSparkVersions.mkString(", ")
+      val supportedVersions = supportedSparkVersions.toSeq.sorted.mkString(", ")
       throw new IllegalArgumentException(
         s"Unsupported spark version used: ${spark.SPARK_VERSION}. Normalized spark version used: ${SPARK_VERSION}." +
           s" Supported versions: ${supportedVersions}")