Skip to content

Commit 6fba9e2

Browse files
AngersZhuuuucloud-fan
authored andcommitted
[SPARK-37951][MLLIB][K8S] Move test file from ../data/ to corresponding module's resource folder
### What changes were proposed in this pull request? Move test file form `data/` dir to corresponding module's resource folder 1. move `../data/mllib/images/partitioned` to mllib's `resources/inages/partitioned` 2. move `../data/mllib/iris_libsvm.txt` to mllib's `resources/iris_libsvm.txt` 3. copy `data/mllib/pagerank_data.txt` to kubenets-integration-test's `resources/pagerank_data.txt` ### Why are the changes needed? Refactor code to avoid test failure ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existed UT Closes apache#35237 from AngersZhuuuu/SPARK-37951. Authored-by: Angerszhuuuu <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
1 parent 71af5b9 commit 6fba9e2

File tree

15 files changed

+13
-8
lines changed

15 files changed

+13
-8
lines changed

mllib/src/main/scala/org/apache/spark/ml/source/image/ImageDataSource.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,12 @@ package org.apache.spark.ml.source.image
3737
* // Scala
3838
* val df = spark.read.format("image")
3939
* .option("dropInvalid", true)
40-
* .load("data/mllib/images/partitioned")
40+
* .load("/path/to/images")
4141
*
4242
* // Java
4343
* Dataset<Row> df = spark.read().format("image")
4444
* .option("dropInvalid", true)
45-
* .load("data/mllib/images/partitioned");
45+
* .load("/path/to/images");
4646
* }}}
4747
*
4848
* Image data source supports the following options:
File renamed without changes.

mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class ClusteringEvaluatorSuite
4040

4141
override def beforeAll(): Unit = {
4242
super.beforeAll()
43-
irisDataset = spark.read.format("libsvm").load("../data/mllib/iris_libsvm.txt")
43+
irisDataset = spark.read.format("libsvm").load(getTestResourcePath("iris_libsvm.txt"))
4444
val datasets = MLTestingUtils.generateArrayFeatureDataset(irisDataset)
4545
newIrisDataset = datasets._1
4646
newIrisDatasetD = datasets._2

mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala

+1-2
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,7 @@ import org.apache.spark.sql.functions.{col, substring_index}
2929
class ImageFileFormatSuite extends SparkFunSuite with MLlibTestSparkContext {
3030

3131
// Single column of images named "image"
32-
private lazy val imagePath = "../data/mllib/images/partitioned"
33-
private lazy val recursiveImagePath = "../data/mllib/images"
32+
private lazy val imagePath = getTestResourcePath("images/partitioned")
3433

3534
test("Smoke test: create basic ImageSchema dataframe") {
3635
val origin = "path"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
1 2
2+
1 3
3+
1 4
4+
2 1
5+
3 1
6+
4 1

resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala

+3-3
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import io.fabric8.kubernetes.api.model.Pod
2222
import org.scalatest.concurrent.Eventually
2323
import org.scalatest.matchers.should.Matchers._
2424

25-
import org.apache.spark.TestUtils
25+
import org.apache.spark.{SparkFunSuite, TestUtils}
2626
import org.apache.spark.launcher.SparkLauncher
2727

2828
private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
@@ -126,11 +126,11 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
126126
}
127127
}
128128

129-
private[spark] object BasicTestsSuite {
129+
private[spark] object BasicTestsSuite extends SparkFunSuite {
130130
val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank"
131131
val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files"
132132
val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE =
133133
s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt"
134-
val REMOTE_PAGE_RANK_DATA_FILE = "data/mllib/pagerank_data.txt"
134+
val REMOTE_PAGE_RANK_DATA_FILE = getTestResourcePath("pagerank_data.txt")
135135
val REMOTE_PAGE_RANK_FILE_NAME = "pagerank_data.txt"
136136
}

0 commit comments

Comments
 (0)