draft memory/lag issues$

Author: percevalw

tests/conftest.py

@@ -67,7 +67,7 @@ def pytest_collection_modifyitems(session, config, items):
 
 
 @pytest.fixture(
-    scope="session",
+    scope="module",
     autouse=True,
 )
 def spark_session(pytestconfig, tmpdir_factory):
@@ -80,7 +80,6 @@ def spark_session(pytestconfig, tmpdir_factory):
         print("!! Creating spark session !!")
 
         from pyspark import SparkConf
-        from pyspark import __version__ as pyspark_version
 
         temp_warehouse_dir = tmpdir_factory.mktemp("spark")
         conf = (
@@ -103,14 +102,6 @@ def spark_session(pytestconfig, tmpdir_factory):
             .set("spark.driver.memory", "2g")
         )
 
-        if pyspark_version < "3":
-
-            # used to overwrite hive tables
-            conf = conf.set(
-                "spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation",
-                "true",
-            )
-
         session, _, _ = improve_performances(to_add_conf=list(conf.getAll()))
 
         # session is ready