Launcher, fixed "different versions" error

Author: wtsimple

spark_launcher.py

@@ -1,6 +1,13 @@
+import os
+import sys
+
 from pyspark.sql import SparkSession
 
 
 class SparkLauncher(object):
     def __init__(self):
-        self.session = SparkSession.builder.getOrCreate()
+        # Needed to prevent different worker and driver python versions error
+        for var in ['PYSPARK_PYTHON', 'PYSPARK_DRIVER_PYTHON']:
+            os.environ[var] = sys.executable
+
+        self.session = SparkSession.builder.getOrCreate()