Update SparkSesssion.builder to remove appName and master spark-examples#2

bekisz · bekisz · commit 0bbceb982b0c · 2024-10-04T16:11:52.000+02:00
diff --git a/src/main/scala/com/sparkbyexamples/spark/SQLContextExample.scala b/src/main/scala/com/sparkbyexamples/spark/SQLContextExample.scala
@@ -5,8 +5,6 @@ import org.apache.spark.sql.{SQLContext, SparkSession}
 object SQLContextExample extends App {
 
   val spark = SparkSession.builder()
-    .master("local[1]")
-    .appName("SparkByExamples.com")
     .getOrCreate();
 
   spark.sparkContext.setLogLevel("ERROR")
diff --git a/src/main/scala/com/sparkbyexamples/spark/SparkSessionWrapper.scala b/src/main/scala/com/sparkbyexamples/spark/SparkSessionWrapper.scala
@@ -6,8 +6,6 @@ trait SparkSessionWrapper {
   lazy val spark: SparkSession = {
     SparkSession
       .builder()
-      .master("local")
-      .appName("spark session")
       .config("spark.sql.shuffle.partitions", "1")
       .getOrCreate()
   }
diff --git a/src/main/scala/com/sparkbyexamples/spark/dataframe/AvroToJson.scala b/src/main/scala/com/sparkbyexamples/spark/dataframe/AvroToJson.scala
@@ -5,8 +5,6 @@ import org.apache.spark.sql.{SaveMode, SparkSession}
 object AvroToJson extends App {
 
   val spark: SparkSession = SparkSession.builder()
-    .master("local[1]")
-    .appName("SparkByExample")
     .getOrCreate()
 
   spark.sparkContext.setLogLevel("ERROR")
diff --git a/src/main/scala/com/sparkbyexamples/spark/dataframe/AvroToParquet.scala b/src/main/scala/com/sparkbyexamples/spark/dataframe/AvroToParquet.scala
@@ -5,8 +5,6 @@ import org.apache.spark.sql.{SaveMode, SparkSession}
 object AvroToParquet extends App {
 
   val spark: SparkSession = SparkSession.builder()
-    .master("local[1]")
-    .appName("SparkByExample")
     .getOrCreate()
 
   spark.sparkContext.setLogLevel("ERROR")
diff --git a/src/main/scala/com/sparkbyexamples/spark/dataframe/CsvToAvroParquetJson.scala b/src/main/scala/com/sparkbyexamples/spark/dataframe/CsvToAvroParquetJson.scala
@@ -5,8 +5,6 @@ import org.apache.spark.sql.{SaveMode, SparkSession}
 object CsvToAvroParquetJson extends App {
 
   val spark: SparkSession = SparkSession.builder()
-    .master("local[1]")
-    .appName("SparkByExample")
     .getOrCreate()
 
   spark.sparkContext.setLogLevel("ERROR")
diff --git a/src/main/scala/com/sparkbyexamples/spark/dataframe/JsonToAvroCsvParquet.scala b/src/main/scala/com/sparkbyexamples/spark/dataframe/JsonToAvroCsvParquet.scala
@@ -5,8 +5,6 @@ import org.apache.spark.sql.SparkSession
 object JsonToAvroCsvParquet extends App {
 
   val spark: SparkSession = SparkSession.builder()
-    .master("local[1]")
-    .appName("SparkByExample")
     .getOrCreate()
 
   spark.sparkContext.setLogLevel("ERROR")
diff --git a/src/main/scala/com/sparkbyexamples/spark/dataframe/ParquetToAvro.scala b/src/main/scala/com/sparkbyexamples/spark/dataframe/ParquetToAvro.scala
@@ -5,8 +5,6 @@ import org.apache.spark.sql.{SaveMode, SparkSession}
 object ParquetToAvro extends App {
 
   val spark: SparkSession = SparkSession.builder()
-    .master("local[1]")
-    .appName("SparkByExample")
     .getOrCreate()
 
   spark.sparkContext.setLogLevel("ERROR")
diff --git a/src/main/scala/com/sparkbyexamples/spark/dataframe/examples/DataFrameEmptyCheck.scala b/src/main/scala/com/sparkbyexamples/spark/dataframe/examples/DataFrameEmptyCheck.scala
@@ -5,8 +5,6 @@ import org.apache.spark.sql.SparkSession
 object DataFrameEmptyCheck extends App {
 
   val spark:SparkSession = SparkSession.builder()
-    .master("local[1]")
-    .appName("SparkByExample")
     .getOrCreate()
 
   val df = spark.emptyDataFrame
diff --git a/src/main/scala/com/sparkbyexamples/spark/rdd/OperationOnPairRDDComplex.scala b/src/main/scala/com/sparkbyexamples/spark/rdd/OperationOnPairRDDComplex.scala
@@ -8,8 +8,6 @@ object OperationOnPairRDDComplex extends App{
 
 
   val spark = SparkSession.builder()
-    .appName("SparkByExample")
-    .master("local")
     .getOrCreate()
 
   spark.sparkContext.setLogLevel("ERROR")
diff --git a/src/main/scala/com/sparkbyexamples/spark/rdd/RDDAccumulator.scala b/src/main/scala/com/sparkbyexamples/spark/rdd/RDDAccumulator.scala
@@ -6,8 +6,6 @@ import org.apache.spark.sql.SparkSession
 object RDDAccumulator extends App {
 
   val spark = SparkSession.builder()
-    .appName("SparkByExample")
-    .master("local")
     .getOrCreate()
 
   val longAcc = spark.sparkContext.longAccumulator("SumAccumulator")
diff --git a/src/main/scala/com/sparkbyexamples/spark/rdd/RDDActions.scala b/src/main/scala/com/sparkbyexamples/spark/rdd/RDDActions.scala
@@ -8,8 +8,6 @@ import scala.collection.mutable
 object RDDActions extends App {
 
   val spark = SparkSession.builder()
-    .appName("SparkByExample")
-    .master("local")
     .getOrCreate()
 
   spark.sparkContext.setLogLevel("ERROR")
diff --git a/src/main/scala/com/sparkbyexamples/spark/rdd/RDDPrint.scala b/src/main/scala/com/sparkbyexamples/spark/rdd/RDDPrint.scala
@@ -5,8 +5,6 @@ import org.apache.spark.sql.SparkSession
 object RDDPrint extends App{
 
   val spark:SparkSession = SparkSession.builder()
-    .master("local[1]")
-    .appName("SparkByExample")
     .getOrCreate()
   val dept = List(("Finance",10),("Marketing",20),
       ("Sales",30), ("IT",40))

Original file line number	Diff line number	Diff line change
`@@ -6,8 +6,6 @@ trait SparkSessionWrapper {`
`6`	`6`	`lazy val spark: SparkSession = {`
`7`	`7`	`SparkSession`
`8`	`8`	`.builder()`
`9`		`- .master("local")`
`10`		`- .appName("spark session")`
`11`	`9`	`.config("spark.sql.shuffle.partitions", "1")`
`12`	`10`	`.getOrCreate()`
`13`	`11`	`}`