Add support for Spark 3.5.0 (#514)

* Add support for Spark 3.5.0

Signed-off-by: GBBBAS <[email protected]>

* Update for tests

Signed-off-by: GBBBAS <[email protected]>

* Run tests on 3.4.1

Signed-off-by: GBBBAS <[email protected]>

* Update mkdocs material package

Signed-off-by: GBBBAS <[email protected]>

---------

Signed-off-by: GBBBAS <[email protected]>

Author: GBBBAS

.github/workflows/test.yml

@@ -27,7 +27,7 @@ jobs:
         matrix:
           os: [ubuntu-latest]
           python-version: ["3.8", "3.9", "3.10", "3.11"]
-          pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0"]
+          pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1"]
           exclude:
             - pyspark: "3.4.0"
               python-version: "3.8"
@@ -49,7 +49,9 @@ jobs:
             - pyspark: "3.3.2"
               delta-spark: "2.3.0"
             - pyspark: "3.4.0"
-              delta-spark: "2.4.0"              
+              delta-spark: "2.4.0"
+            - pyspark: "3.4.1"
+              delta-spark: "2.4.0"                                
       runs-on: ${{ matrix.os }}
       steps:
         - uses: actions/checkout@v3

docs/blog/.authors.yml

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-GBBBAS:
-  name: Bryce Bartmann
-  description: Contributor
-  avatar: https://github.com/GBBBAS.png
+authors:
+  GBBBAS:
+    name: Bryce Bartmann
+    description: Contributor
+    avatar: https://github.com/GBBBAS.png

environment.yml

@@ -40,13 +40,14 @@ dependencies:
     - fastapi==0.100.1
     - httpx==0.24.1
     - trio==0.22.1
-    - pyspark>=3.3.0,<3.5.0
+    - pyspark>=3.3.0,<3.6.0
     - delta-spark>=2.2.0,<3.1.0
     - grpcio>=1.48.1
     - grpcio-status>=1.48.1
     - googleapis-common-protos>=1.56.4
     - openjdk==11.0.15
-    - openai==0.27.8   
+    - openai==0.27.8
+    - mkdocs-material==9.3.1
     - mkdocstrings==0.22.0
     - mkdocstrings-python==1.4.0
     - mkdocs-macros-plugin==1.0.1
@@ -75,5 +76,4 @@ dependencies:
         - langchain==0.0.291
         - build==0.10.0
         - deltalake==0.10.1
-        - mkdocs-material==9.2.0b3
 

setup.py

@@ -44,7 +44,7 @@
 ]
 
 PYSPARK_PACKAGES = [
-    "pyspark>=3.3.0,<3.5.0",
+    "pyspark>=3.3.0,<3.6.0",
     "delta-spark>=2.2.0,<2.5.0",
 ]
 

src/sdk/python/rtdip_sdk/pipelines/destinations/spark/pcdm_latest_to_delta.py

@@ -168,7 +168,6 @@ def _write_latest_to_delta(self, df: DataFrame, epoch_id=None):  # NOSONAR
         when_not_matched_insert_list = [
             DeltaMergeConditionValues(
                 values={
-                    "EventDate": "source.EventDate",
                     "TagName": "source.TagName",
                     "EventTime": "source.EventTime",
                     "Status": "source.Status",

tests/sdk/python/rtdip_sdk/pipelines/destinations/spark/test_pcdm_latest_to_delta.py

@@ -47,7 +47,6 @@ def create_delta_table(spark_session, name):
         spark=spark_session,
         table_name=name,
         columns=[
-            DeltaTableColumn(name="EventDate", type="date", nullable=True),
             DeltaTableColumn(name="TagName", type="string", nullable=True),
             DeltaTableColumn(name="EventTime", type="timestamp", nullable=True),
             DeltaTableColumn(name="Status", type="string", nullable=True),
@@ -119,7 +118,6 @@ def test_spark_pcdm_latest_to_delta_write_batch(spark_session: SparkSession):
     expected_df = spark_session.createDataFrame(
         [
             {
-                "EventDate": datetime(2023, 1, 20).date(),
                 "TagName": "Tag1",
                 "EventTime": datetime(2023, 1, 20, 1, 1),
                 "Status": "Good",
@@ -131,7 +129,6 @@ def test_spark_pcdm_latest_to_delta_write_batch(spark_session: SparkSession):
             }
         ]
     ).select(
-        "EventDate",
         "TagName",
         "EventTime",
         "Status",