Fix the local development environment and update documentation

.github/workflows/test.yml

@@ -135,6 +135,9 @@ jobs:
       - name: Run integration tests
         run: |
           hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration
+        env:
+          RAY_SPEC_FILENAME: "ray-gke.yaml"
+
       - name: Upload coverage to Github
         uses: actions/upload-artifact@v4
         with:

dev/dags/ray_taskflow_example.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime
 from pathlib import Path
 
@@ -6,7 +7,9 @@
 from ray_provider.decorators import ray
 
 CONN_ID = "ray_conn"
-RAY_SPEC = Path(__file__).parent / "scripts/ray.yaml"
+RAY_SPEC_FILENAME = os.getenv("RAY_SPEC_FILENAME", "ray.yaml")
+RAY_SPEC = Path(__file__).parent / "scripts" / RAY_SPEC_FILENAME
+
 FOLDER_PATH = Path(__file__).parent / "ray_scripts"
 RAY_TASK_CONFIG = {
     "conn_id": CONN_ID,

dev/dags/ray_taskflow_example_existing_cluster.py

@@ -5,7 +5,7 @@
 
 from ray_provider.decorators import ray
 
-CONN_ID = "ray_job"
+CONN_ID = "ray_conn"
 FOLDER_PATH = Path(__file__).parent / "ray_scripts"
 RAY_TASK_CONFIG = {
     "conn_id": CONN_ID,

dev/dags/scripts/ray-gke.yaml

@@ -0,0 +1,62 @@
+apiVersion: ray.io/v1
+kind: RayCluster
+metadata:
+  name: raycluster-complete
+spec:
+  rayVersion: "2.10.0"
+  enableInTreeAutoscaling: true
+  headGroupSpec:
+    serviceType: LoadBalancer
+    rayStartParams:
+      dashboard-host: "0.0.0.0"
+      block: "true"
+    template:
+      metadata:
+        labels:
+          ray-node-type: head
+      spec:
+        containers:
+        - name: ray-head
+          image: rayproject/ray-ml:latest
+          resources:
+            limits:
+              cpu: 1
+              memory: 3Gi
+            requests:
+              cpu: 1
+              memory: 3Gi
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh","-c","ray stop"]
+          ports:
+          - containerPort: 6379
+            name: gcs
+          - containerPort: 8265
+            name: dashboard
+          - containerPort: 10001
+            name: client
+          - containerPort: 8000
+            name: serve
+          - containerPort: 8080
+            name: metrics
+  workerGroupSpecs:
+  - groupName: small-group
+    replicas: 1
+    minReplicas: 1
+    maxReplicas: 2
+    rayStartParams:
+      block: "true"
+    template:
+      metadata:
+      spec:
+        containers:
+        - name: machine-learning
+          image: rayproject/ray-ml:latest
+          resources:
+            limits:
+              cpu: 1
+              memory: 1Gi
+            requests:
+              cpu: 1
+              memory: 1Gi

dev/dags/scripts/ray.yaml

@@ -1,7 +1,7 @@
 apiVersion: ray.io/v1
 kind: RayCluster
 metadata:
-  name: raycluster-complete
+  name: airflow-raycluster
 spec:
   rayVersion: "2.10.0"
   enableInTreeAutoscaling: true
@@ -15,9 +15,11 @@ spec:
         labels:
           ray-node-type: head
       spec:
+        imagePullSecrets:
+          - name: my-registry-secret
         containers:
         - name: ray-head
-          image: rayproject/ray-ml:latest
+          image: rayproject/ray:2.20.0-aarch64
           resources:
             limits:
               cpu: 1
@@ -50,9 +52,11 @@ spec:
     template:
       metadata:
       spec:
+        imagePullSecrets:
+          - name: my-registry-secret
         containers:
         - name: machine-learning
-          image: rayproject/ray-ml:latest
+          image: rayproject/ray:2.20.0-aarch64
           resources:
             limits:
               cpu: 1

dev/docker-compose.override.yml

@@ -0,0 +1,18 @@
+version: '3.8'
+
+services:
+  webserver:
+    networks:
+      - kind
+
+  scheduler:
+    networks:
+      - kind
+
+  triggerer:
+    networks:
+      - kind
+
+networks:
+  kind:
+    external: true

dev/kind-config.yaml

@@ -0,0 +1,16 @@
+kind: Cluster
+name: local
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerAddress: "0.0.0.0"
+  apiServerPort: 6443
+nodes:
+  - role: control-plane
+kubeadmConfigPatchesJSON6902:
+- group: kubeadm.k8s.io
+  version: v1beta3
+  kind: ClusterConfiguration
+  patch: |
+    - op: add
+      path: /apiServer/certSANs/-
+      value: host.docker.internal