spark-examples · kapil90k · Feb 12, 2023 · May 6, 2023 · Sep 12, 2024
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+.idea/
+.DS_Store
+*.pyc
diff --git a/.idea/.gitignore b/.idea/.gitignore
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/modules.xml b/.idea/modules.xml
diff --git a/.idea/pyspark-examples.iml b/.idea/pyspark-examples.iml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/aws-jars-needed-for-s3/aws-java-sdk-1.12.183.jar b/aws-jars-needed-for-s3/aws-java-sdk-1.12.183.jar
diff --git a/aws-jars-needed-for-s3/aws-java-sdk-core-1.12.183.jar b/aws-jars-needed-for-s3/aws-java-sdk-core-1.12.183.jar
diff --git a/aws-jars-needed-for-s3/aws-java-sdk-dynamodb-1.12.183.jar b/aws-jars-needed-for-s3/aws-java-sdk-dynamodb-1.12.183.jar
diff --git a/aws-jars-needed-for-s3/aws-java-sdk-s3-1.12.183.jar b/aws-jars-needed-for-s3/aws-java-sdk-s3-1.12.183.jar
diff --git a/aws-jars-needed-for-s3/hadoop-aws-2.7.4.jar b/aws-jars-needed-for-s3/hadoop-aws-2.7.4.jar
diff --git a/kapil-practice/CreateSpark.py b/kapil-practice/CreateSpark.py
diff --git a/kapil-practice/S3Df.py b/kapil-practice/S3Df.py
@@ -0,0 +1,25 @@
+from pyspark.sql.session import SparkSession
+from pyspark.sql.functions import *
+from pyspark.sql.window import *
+from pyspark.sql.types import *
+
+spark = SparkSession.builder.\
+master("local").\
+appName("prac").\
+getOrCreate()
+
+sc = spark.sparkContext
+sc._jsc.hadoopConfiguration().set("com.amazonaws.services.s3.enableV4", "true")
+sc._jsc.hadoopConfiguration().set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
+sc._jsc.hadoopConfiguration().set("fs.s3a.aws.credentials.provider", "com.amazonaws.auth.InstanceProfileCredentialsProvider,com.amazonaws.auth.DefaultAWSCredentialsProviderChain")
+sc._jsc.hadoopConfiguration().set("fs.AbstractFileSystem.s3a.impl", "org.apache.hadoop.fs.s3a.S3A")
+sc._jsc.hadoopConfiguration().set("fs.s3a.endpoint", "s3.us-east-1.amazonaws.com")
+
+sc._jsc.hadoopConfiguration().set('fs.s3a.access.key', 'A')
+sc._jsc.hadoopConfiguration().set('fs.s3a.secret.key', 'VracelDcLui')
+sc._jsc.hadoopConfiguration().set("fs.s3a.endpoint", "s3.amazonaws.com")
+
+df = spark.read.option("inferSchema","true").option("header","true").csv("s3a://ind-north-up-agra-kapil-test-1/datasets/input-ds/movies/")
+df.show()
+
+spark.stop()
diff --git a/kapil-practice/__init__.py b/kapil-practice/__init__.py
diff --git a/kapil-practice/__pycache__/CreateSpark.cpython-38.pyc b/kapil-practice/__pycache__/CreateSpark.cpython-38.pyc
diff --git a/kapil-practice/kp-convert-column-python-list.py b/kapil-practice/kp-convert-column-python-list.py
@@ -0,0 +1,15 @@
+from pyspark.sql.functions import *
+from pyspark.sql import SparkSession
+
+spark = SparkSession.builder.master("local[1]") \
+    .appName('SparkByExamples.com') \
+    .getOrCreate()
+
+nasa_df = spark.read.json("G:\\datasets\\input-datasets\\json-datasets\\nasa-singleline-zipcodes.json")
+nasa_df.printSchema()
+
+nasa_df.show()
+
+# all_cities = nasa_df.select("City").distinct().rdd.map(lambda x: x.City).collect()
+# print(all_cities)
+spark.stop()
diff --git a/spark-by-examples/__init__.py b/spark-by-examples/__init__.py
diff --git a/convert-column-python-list.py → ...by-examples/convert-column-python-list.py b/convert-column-python-list.py → ...by-examples/convert-column-python-list.py
diff --git a/currentdate.py → spark-by-examples/currentdate.py b/currentdate.py → spark-by-examples/currentdate.py
diff --git a/data.txt → spark-by-examples/data.txt b/data.txt → spark-by-examples/data.txt
diff --git a/pandas-pyspark-dataframe.py → ...k-by-examples/pandas-pyspark-dataframe.py b/pandas-pyspark-dataframe.py → ...k-by-examples/pandas-pyspark-dataframe.py
diff --git a/pyspark-add-month.py → spark-by-examples/pyspark-add-month.py b/pyspark-add-month.py → spark-by-examples/pyspark-add-month.py
diff --git a/pyspark-add-new-column.py → spark-by-examples/pyspark-add-new-column.py b/pyspark-add-new-column.py → spark-by-examples/pyspark-add-new-column.py
diff --git a/pyspark-aggregate.py → spark-by-examples/pyspark-aggregate.py b/pyspark-aggregate.py → spark-by-examples/pyspark-aggregate.py
diff --git a/pyspark-array-string.py → spark-by-examples/pyspark-array-string.py b/pyspark-array-string.py → spark-by-examples/pyspark-array-string.py
diff --git a/pyspark-arraytype.py → spark-by-examples/pyspark-arraytype.py b/pyspark-arraytype.py → spark-by-examples/pyspark-arraytype.py
diff --git a/pyspark-broadcast-dataframe.py → ...y-examples/pyspark-broadcast-dataframe.py b/pyspark-broadcast-dataframe.py → ...y-examples/pyspark-broadcast-dataframe.py
diff --git a/pyspark-cast-column.py → spark-by-examples/pyspark-cast-column.py b/pyspark-cast-column.py → spark-by-examples/pyspark-cast-column.py
diff --git a/pyspark-change-string-double.py → ...-examples/pyspark-change-string-double.py b/pyspark-change-string-double.py → ...-examples/pyspark-change-string-double.py
diff --git a/pyspark-collect.py → spark-by-examples/pyspark-collect.py b/pyspark-collect.py → spark-by-examples/pyspark-collect.py
diff --git a/pyspark-column-functions.py → ...k-by-examples/pyspark-column-functions.py b/pyspark-column-functions.py → ...k-by-examples/pyspark-column-functions.py
diff --git a/pyspark-column-operations.py → ...-by-examples/pyspark-column-operations.py b/pyspark-column-operations.py → ...-by-examples/pyspark-column-operations.py
diff --git a/pyspark-convert-map-to-columns.py → ...xamples/pyspark-convert-map-to-columns.py b/pyspark-convert-map-to-columns.py → ...xamples/pyspark-convert-map-to-columns.py
diff --git a/pyspark-convert_columns-to-map.py → ...xamples/pyspark-convert_columns-to-map.py b/pyspark-convert_columns-to-map.py → ...xamples/pyspark-convert_columns-to-map.py
diff --git a/pyspark-count-distinct.py → spark-by-examples/pyspark-count-distinct.py b/pyspark-count-distinct.py → spark-by-examples/pyspark-count-distinct.py
diff --git a/pyspark-create-dataframe-dictionary.py → ...es/pyspark-create-dataframe-dictionary.py b/pyspark-create-dataframe-dictionary.py → ...es/pyspark-create-dataframe-dictionary.py
diff --git a/pyspark-create-dataframe.py → ...k-by-examples/pyspark-create-dataframe.py b/pyspark-create-dataframe.py → ...k-by-examples/pyspark-create-dataframe.py
diff --git a/pyspark-create-list.py → spark-by-examples/pyspark-create-list.py b/pyspark-create-list.py → spark-by-examples/pyspark-create-list.py
diff --git a/pyspark-current-date-timestamp.py → ...xamples/pyspark-current-date-timestamp.py b/pyspark-current-date-timestamp.py → ...xamples/pyspark-current-date-timestamp.py
diff --git a/pyspark-dataframe-flatMap.py → ...-by-examples/pyspark-dataframe-flatMap.py b/pyspark-dataframe-flatMap.py → ...-by-examples/pyspark-dataframe-flatMap.py
diff --git a/pyspark-dataframe-repartition.py → ...examples/pyspark-dataframe-repartition.py b/pyspark-dataframe-repartition.py → ...examples/pyspark-dataframe-repartition.py
diff --git a/pyspark-dataframe.py → spark-by-examples/pyspark-dataframe.py b/pyspark-dataframe.py → spark-by-examples/pyspark-dataframe.py
diff --git a/pyspark-date-string.py → spark-by-examples/pyspark-date-string.py b/pyspark-date-string.py → spark-by-examples/pyspark-date-string.py
diff --git a/pyspark-date-timestamp-functions.py → ...mples/pyspark-date-timestamp-functions.py b/pyspark-date-timestamp-functions.py → ...mples/pyspark-date-timestamp-functions.py
diff --git a/pyspark-datediff.py → spark-by-examples/pyspark-datediff.py b/pyspark-datediff.py → spark-by-examples/pyspark-datediff.py
diff --git a/pyspark-distinct.py → spark-by-examples/pyspark-distinct.py b/pyspark-distinct.py → spark-by-examples/pyspark-distinct.py
diff --git a/pyspark-drop-column.py → spark-by-examples/pyspark-drop-column.py b/pyspark-drop-column.py → spark-by-examples/pyspark-drop-column.py
diff --git a/pyspark-drop-null.py → spark-by-examples/pyspark-drop-null.py b/pyspark-drop-null.py → spark-by-examples/pyspark-drop-null.py
@@ -9,7 +9,7 @@
     .appName("SparkByExamples.com") \
     .getOrCreate()
 
-filePath="resources/small_zipcode.csv"
+filePath= "../resources/small_zipcode.csv"
 df = spark.read.options(header='true', inferSchema='true') \
           .csv(filePath)
 

diff --git a/pyspark-empty-data-frame.py → ...k-by-examples/pyspark-empty-data-frame.py b/pyspark-empty-data-frame.py → ...k-by-examples/pyspark-empty-data-frame.py
diff --git a/pyspark-explode-array-map.py → ...-by-examples/pyspark-explode-array-map.py b/pyspark-explode-array-map.py → ...-by-examples/pyspark-explode-array-map.py
diff --git a/pyspark-explode-nested-array.py → ...-examples/pyspark-explode-nested-array.py b/pyspark-explode-nested-array.py → ...-examples/pyspark-explode-nested-array.py
diff --git a/pyspark-expr.py → spark-by-examples/pyspark-expr.py b/pyspark-expr.py → spark-by-examples/pyspark-expr.py
diff --git a/pyspark-filter-null.py → spark-by-examples/pyspark-filter-null.py b/pyspark-filter-null.py → spark-by-examples/pyspark-filter-null.py
diff --git a/pyspark-filter.py → spark-by-examples/pyspark-filter.py b/pyspark-filter.py → spark-by-examples/pyspark-filter.py
diff --git a/pyspark-filter2.py → spark-by-examples/pyspark-filter2.py b/pyspark-filter2.py → spark-by-examples/pyspark-filter2.py
diff --git a/pyspark-fulter-null.py → spark-by-examples/pyspark-fulter-null.py b/pyspark-fulter-null.py → spark-by-examples/pyspark-fulter-null.py
diff --git a/pyspark-groupby-sort.py → spark-by-examples/pyspark-groupby-sort.py b/pyspark-groupby-sort.py → spark-by-examples/pyspark-groupby-sort.py
diff --git a/pyspark-groupby.py → spark-by-examples/pyspark-groupby.py b/pyspark-groupby.py → spark-by-examples/pyspark-groupby.py
diff --git a/pyspark-join-two-dataframes.py → ...y-examples/pyspark-join-two-dataframes.py b/pyspark-join-two-dataframes.py → ...y-examples/pyspark-join-two-dataframes.py
diff --git a/pyspark-join.py → spark-by-examples/pyspark-join.py b/pyspark-join.py → spark-by-examples/pyspark-join.py
diff --git a/pyspark-left-anti-join.py → spark-by-examples/pyspark-left-anti-join.py b/pyspark-left-anti-join.py → spark-by-examples/pyspark-left-anti-join.py
diff --git a/pyspark-lit.py → spark-by-examples/pyspark-lit.py b/pyspark-lit.py → spark-by-examples/pyspark-lit.py
diff --git a/pyspark-loop.py → spark-by-examples/pyspark-loop.py b/pyspark-loop.py → spark-by-examples/pyspark-loop.py
diff --git a/pyspark-mappartitions.py → spark-by-examples/pyspark-mappartitions.py b/pyspark-mappartitions.py → spark-by-examples/pyspark-mappartitions.py
diff --git a/pyspark-maptype-dataframe-column.py → ...mples/pyspark-maptype-dataframe-column.py b/pyspark-maptype-dataframe-column.py → ...mples/pyspark-maptype-dataframe-column.py
diff --git a/pyspark-orderby-groupby.py → spark-by-examples/pyspark-orderby-groupby.py b/pyspark-orderby-groupby.py → spark-by-examples/pyspark-orderby-groupby.py
diff --git a/pyspark-orderby.py → spark-by-examples/pyspark-orderby.py b/pyspark-orderby.py → spark-by-examples/pyspark-orderby.py
diff --git a/pyspark-parallelize.py → spark-by-examples/pyspark-parallelize.py b/pyspark-parallelize.py → spark-by-examples/pyspark-parallelize.py
diff --git a/pyspark-partitionby.py → spark-by-examples/pyspark-partitionby.py b/pyspark-partitionby.py → spark-by-examples/pyspark-partitionby.py
diff --git a/pyspark-pivot.py → spark-by-examples/pyspark-pivot.py b/pyspark-pivot.py → spark-by-examples/pyspark-pivot.py
diff --git a/pyspark-print-contents.py → spark-by-examples/pyspark-print-contents.py b/pyspark-print-contents.py → spark-by-examples/pyspark-print-contents.py
diff --git a/pyspark-python-dataframe.py → ...k-by-examples/pyspark-python-dataframe.py b/pyspark-python-dataframe.py → ...k-by-examples/pyspark-python-dataframe.py
diff --git a/pyspark-range-partition.py → spark-by-examples/pyspark-range-partition.py b/pyspark-range-partition.py → spark-by-examples/pyspark-range-partition.py
diff --git a/pyspark-rdd-actions.py → spark-by-examples/pyspark-rdd-actions.py b/pyspark-rdd-actions.py → spark-by-examples/pyspark-rdd-actions.py
diff --git a/pyspark-rdd-broadcast.py → spark-by-examples/pyspark-rdd-broadcast.py b/pyspark-rdd-broadcast.py → spark-by-examples/pyspark-rdd-broadcast.py
diff --git a/pyspark-rdd-flatMap.py → spark-by-examples/pyspark-rdd-flatMap.py b/pyspark-rdd-flatMap.py → spark-by-examples/pyspark-rdd-flatMap.py
diff --git a/pyspark-rdd-map.py → spark-by-examples/pyspark-rdd-map.py b/pyspark-rdd-map.py → spark-by-examples/pyspark-rdd-map.py
diff --git a/pyspark-rdd-reduceByKey.py → spark-by-examples/pyspark-rdd-reduceByKey.py b/pyspark-rdd-reduceByKey.py → spark-by-examples/pyspark-rdd-reduceByKey.py
diff --git a/pyspark-rdd-to-dataframe.py → ...k-by-examples/pyspark-rdd-to-dataframe.py b/pyspark-rdd-to-dataframe.py → ...k-by-examples/pyspark-rdd-to-dataframe.py
diff --git a/pyspark-rdd-wordcount-2.py → spark-by-examples/pyspark-rdd-wordcount-2.py b/pyspark-rdd-wordcount-2.py → spark-by-examples/pyspark-rdd-wordcount-2.py
diff --git a/pyspark-rdd-wordcount.py → spark-by-examples/pyspark-rdd-wordcount.py b/pyspark-rdd-wordcount.py → spark-by-examples/pyspark-rdd-wordcount.py
diff --git a/pyspark-rdd.py → spark-by-examples/pyspark-rdd.py b/pyspark-rdd.py → spark-by-examples/pyspark-rdd.py
diff --git a/pyspark-read-csv.py → spark-by-examples/pyspark-read-csv.py b/pyspark-read-csv.py → spark-by-examples/pyspark-read-csv.py
diff --git a/pyspark-read-json.py → spark-by-examples/pyspark-read-json.py b/pyspark-read-json.py → spark-by-examples/pyspark-read-json.py
diff --git a/pyspark-rename-column.py → spark-by-examples/pyspark-rename-column.py b/pyspark-rename-column.py → spark-by-examples/pyspark-rename-column.py
diff --git a/pyspark-repace-null.py → spark-by-examples/pyspark-repace-null.py b/pyspark-repace-null.py → spark-by-examples/pyspark-repace-null.py
@@ -8,7 +8,7 @@
     .appName("SparkByExamples.com") \
     .getOrCreate()
 
-filePath="resources/small_zipcode.csv"
+filePath= "../resources/small_zipcode.csv"
 df = spark.read.options(header='true', inferSchema='true') \
           .csv(filePath)
 

diff --git a/pyspark-repartition-2.py → spark-by-examples/pyspark-repartition-2.py b/pyspark-repartition-2.py → spark-by-examples/pyspark-repartition-2.py
diff --git a/pyspark-repartition.py → spark-by-examples/pyspark-repartition.py b/pyspark-repartition.py → spark-by-examples/pyspark-repartition.py
diff --git a/pyspark-row.py → spark-by-examples/pyspark-row.py b/pyspark-row.py → spark-by-examples/pyspark-row.py
diff --git a/pyspark-sampling.py → spark-by-examples/pyspark-sampling.py b/pyspark-sampling.py → spark-by-examples/pyspark-sampling.py
diff --git a/pyspark-select-columns.py → spark-by-examples/pyspark-select-columns.py b/pyspark-select-columns.py → spark-by-examples/pyspark-select-columns.py
diff --git a/pyspark-shape-dataframe.py → spark-by-examples/pyspark-shape-dataframe.py b/pyspark-shape-dataframe.py → spark-by-examples/pyspark-shape-dataframe.py
diff --git a/pyspark-show-top-n-rows.py → spark-by-examples/pyspark-show-top-n-rows.py b/pyspark-show-top-n-rows.py → spark-by-examples/pyspark-show-top-n-rows.py
diff --git a/pyspark-sparksession.py → spark-by-examples/pyspark-sparksession.py b/pyspark-sparksession.py → spark-by-examples/pyspark-sparksession.py
diff --git a/pyspark-split-function.py → spark-by-examples/pyspark-split-function.py b/pyspark-split-function.py → spark-by-examples/pyspark-split-function.py
diff --git a/pyspark-sql-case-when.py → spark-by-examples/pyspark-sql-case-when.py b/pyspark-sql-case-when.py → spark-by-examples/pyspark-sql-case-when.py
diff --git a/pyspark-string-date.py → spark-by-examples/pyspark-string-date.py b/pyspark-string-date.py → spark-by-examples/pyspark-string-date.py
diff --git a/pyspark-string-timestamp.py → ...k-by-examples/pyspark-string-timestamp.py b/pyspark-string-timestamp.py → ...k-by-examples/pyspark-string-timestamp.py
diff --git a/pyspark-string-to-array.py → spark-by-examples/pyspark-string-to-array.py b/pyspark-string-to-array.py → spark-by-examples/pyspark-string-to-array.py
diff --git a/pyspark-struct-to-map.py → spark-by-examples/pyspark-struct-to-map.py b/pyspark-struct-to-map.py → spark-by-examples/pyspark-struct-to-map.py
diff --git a/pyspark-structtype.py → spark-by-examples/pyspark-structtype.py b/pyspark-structtype.py → spark-by-examples/pyspark-structtype.py
diff --git a/pyspark-time-diff.py → spark-by-examples/pyspark-time-diff.py b/pyspark-time-diff.py → spark-by-examples/pyspark-time-diff.py
diff --git a/pyspark-timestamp-date.py → spark-by-examples/pyspark-timestamp-date.py b/pyspark-timestamp-date.py → spark-by-examples/pyspark-timestamp-date.py
diff --git a/pyspark-types.py → spark-by-examples/pyspark-types.py b/pyspark-types.py → spark-by-examples/pyspark-types.py
diff --git a/pyspark-udf.py → spark-by-examples/pyspark-udf.py b/pyspark-udf.py → spark-by-examples/pyspark-udf.py
diff --git a/pyspark-union.py → spark-by-examples/pyspark-union.py b/pyspark-union.py → spark-by-examples/pyspark-union.py
diff --git a/pyspark-unix-time.py → spark-by-examples/pyspark-unix-time.py b/pyspark-unix-time.py → spark-by-examples/pyspark-unix-time.py
diff --git a/pyspark-update-column.py → spark-by-examples/pyspark-update-column.py b/pyspark-update-column.py → spark-by-examples/pyspark-update-column.py
diff --git a/pyspark-when-otherwise.py → spark-by-examples/pyspark-when-otherwise.py b/pyspark-when-otherwise.py → spark-by-examples/pyspark-when-otherwise.py
diff --git a/pyspark-window-functions.py → ...k-by-examples/pyspark-window-functions.py b/pyspark-window-functions.py → ...k-by-examples/pyspark-window-functions.py
diff --git a/pyspark-withcolumn.py → spark-by-examples/pyspark-withcolumn.py b/pyspark-withcolumn.py → spark-by-examples/pyspark-withcolumn.py
diff --git a/python-pandas.py → spark-by-examples/python-pandas.py b/python-pandas.py → spark-by-examples/python-pandas.py
diff --git a/schema.py → spark-by-examples/schema.py b/schema.py → spark-by-examples/schema.py
diff --git a/spark-repartition-2.py → spark-by-examples/spark-repartition-2.py b/spark-repartition-2.py → spark-by-examples/spark-repartition-2.py
diff --git a/timediff.py → spark-by-examples/timediff.py b/timediff.py → spark-by-examples/timediff.py