Merge pull request #528 from rtdip/develop

v0.8.2

Author: GBBBAS

.github/release.yml

@@ -40,7 +40,10 @@ changelog:
         - documentation
     - title: Pipelines
       labels:
-        - pipelines      
+        - pipelines
+    - title: Queries
+      labels:
+        - queries        
     - title: Other Changes
       labels:
         - "*"

environment.yml

@@ -40,7 +40,7 @@ dependencies:
     - fastapi==0.100.1
     - httpx==0.24.1
     - trio==0.22.1
-    - pyspark>=3.3.0,<3.6.0
+    - pyspark>=3.3.0,<3.5.0
     - delta-spark>=2.2.0,<3.1.0
     - grpcio>=1.48.1
     - grpcio-status>=1.48.1

setup.py

@@ -44,7 +44,7 @@
 ]
 
 PYSPARK_PACKAGES = [
-    "pyspark>=3.3.0,<3.6.0",
+    "pyspark>=3.3.0,<3.5.0",
     "delta-spark>=2.2.0,<2.5.0",
 ]
 

src/sdk/python/rtdip_sdk/pipelines/deploy/databricks.py

@@ -125,7 +125,7 @@ def deploy(self) -> Union[bool, ValueError]:
         """
         # Add libraries to Databricks Job
         workspace_client = WorkspaceClient(
-            host=self.host, token=self.token, auth_type="token"
+            host=self.host, token=self.token, auth_type="pat"
         )
         for task in self.databricks_job.tasks:
             if task.notebook_task is None and task.spark_python_task is None:
@@ -263,7 +263,7 @@ def launch(self):
         Launches an RTDIP Pipeline Job in Databricks Workflows. This will perform the equivalent of a `Run Now` in Databricks Workflows
         """
         workspace_client = WorkspaceClient(
-            host=self.host, token=self.token, auth_type="token"
+            host=self.host, token=self.token, auth_type="pat"
         )
         job_found = False
         for existing_job in workspace_client.jobs.list(name=self.databricks_job.name):