Set spark master with spark.task.maxFailures in test

vpipkt · vpipkt · commit 8f9e7463e892 · 2020-11-02T09:58:02.000-05:00
This should reduce spurious failures on CI builds

Signed-off-by: Jason T. Brown &lt;jason@astraea.earth&gt;
diff --git a/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala b/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala
@@ -50,7 +50,8 @@ trait TestEnvironment extends FunSpec
     outputDir
   }
 
-  def sparkMaster: String = "local[*]"
+  // allow 2 retries, should stabilize CI builds. https://spark.apache.org/docs/2.4.7/submitting-applications.html#master-urls
+  def sparkMaster: String = "local[*, 2]"
 
   def additionalConf = new SparkConf(false)
 
diff --git a/pyrasterframes/src/main/python/tests/__init__.py b/pyrasterframes/src/main/python/tests/__init__.py
@@ -61,6 +61,7 @@ def pdir(curr):
 
 def spark_test_session():
     spark = create_rf_spark_session(**{
+        'spark.master': 'local[*, 2]',
         'spark.ui.enabled': 'false',
         'spark.app.name': app_name
     })

Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,8 @@ trait TestEnvironment extends FunSpec`
`50`	`50`	`outputDir`
`51`	`51`	`}`
`52`	`52`
`53`		`- def sparkMaster: String = "local[*]"`
	`53`	`+ // allow 2 retries, should stabilize CI builds. https://spark.apache.org/docs/2.4.7/submitting-applications.html#master-urls`
	`54`	`+ def sparkMaster: String = "local[*, 2]"`
`54`	`55`
`55`	`56`	`def additionalConf = new SparkConf(false)`
`56`	`57`