feat: support pyspark 3 (via a databricks.koalas stub)

Author: percevalw

eds_scikit/biology/viz/aggregate.py

@@ -135,8 +135,6 @@ def aggregate_concepts_set(
 
         # Extract concept-set
         measurement_std_filtered = get_measurement_std(measurement_valid, src_to_std)
-        if is_koalas(measurement_std_filtered):
-            measurement_std_filtered.spark.cache()
         measurement_std_filtered = measurement_std_filtered.drop(
             columns="source_concept_id"
         )

tests/conftest.py

@@ -99,7 +99,6 @@ def spark_session(pytestconfig, tmpdir_factory):
                 f"jdbc:derby:;databaseName={temp_warehouse_dir}/metastore_db;create=true",
             )
             .set("spark.executor.cores", 1)
-            .set("spark.driver.memory", "2g")
         )
 
         session, _, _ = improve_performances(to_add_conf=list(conf.getAll()))

tests/test_biology.py

@@ -14,7 +14,10 @@ def tmp_biology_dir(tmp_path_factory):
 
 @pytest.fixture
 def data():
-    return load_biology_data(seed=42)
+    return load_biology_data(
+        seed=42,
+        mean_measurement=1000,
+    )
 
 
 @pytest.fixture
@@ -73,6 +76,7 @@ def test_biology_summary(data, concepts_sets, module, tmp_biology_dir):
         limit_count=("AnaBio", 500),
         stats_only=True,
         save_folder_path=tmp_biology_dir,
+        pd_limit_size=0,
     )