ci: add more triggers

kwilteam · Mar 3, 2025 · 6ed2588 · 6ed2588
1 parent 73e7cc2
commit 6ed2588
Show file tree

Hide file tree

Showing 7 changed files with 205 additions and 67 deletions.
diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml
@@ -41,7 +41,7 @@ jobs:
     if: ${{ needs.changes.outputs.run == 'true' && !github.event.pull_request.draft}}
     runs-on: ubuntu-latest
 
-    services:
+    services: # for unit test
       postgres:
         image: kwildb/postgres:16.8-1
         env:
@@ -171,15 +171,7 @@ jobs:
           testGroupID=$(id -g)
           KACT_LOG_LEVEL=warn task test:act:nb -- -ugid "$testUserID:$testGroupID"
 
-      # - name: Run integration test
-      #   run: |
-      #     testUserID=$(id -u)
-      #     testGroupID=$(id -g)
-      #     cp test/integration/docker-compose.override.yml.example test/integration/docker-compose.override.yml
-      #     cp test/integration/docker-compose-migration.override.yml.example test/integration/docker-compose-migration.override.yml
-      #     sed -i "s/\${UID}:\${GID}/${testUserID}:${testGroupID}/g" test/integration/docker-compose.override.yml
-      #     sed -i "s/\${UID}:\${GID}/${testUserID}:${testGroupID}/g" test/integration/docker-compose-migration.override.yml
-      #     KIT_LOG_LEVEL=warn task test:it:nb
+      # integration tests take too long, we only manually trigger it
 
       - name: Move cache
         run: |
@@ -192,4 +184,4 @@ jobs:
 
       - name: Show error log
         if: ${{ failure() }}
-        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwilAct*/*.log  /tmp/TestKwilInt*/*.log /tmp/TestKwilInt*/*/*.log
+        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwil*/*.log /tmp/TestKwil*/*/*.log
diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml
@@ -35,17 +35,17 @@ jobs:
           filters: .github/filters-old.yaml
           #predicate-quantifier: 'every' # check dorny/paths-filter#225
 
+
   test:
     needs: changes
     #  only run only if there are changes and non-draft PRs
     if: ${{ needs.changes.outputs.run == 'true' && !github.event.pull_request.draft}}
     runs-on: ubuntu-latest
 
-    services:
+    services: # for unit test
       postgres:
         image: kwildb/postgres:16.8-1
         env:
-          POSTGRES_PORT: 5432
           POSTGRES_HOST_AUTH_METHOD: trust
         options: >-
           --health-cmd pg_isready
@@ -64,6 +64,9 @@ jobs:
 
       - name: Install Taskfile
         uses: arduino/setup-task@v2
+        with:
+          # higher Github API rate limiting
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       #ubuntu-latest has go 1.21 installed https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md#go
       #self-hosted also has go 1.21 installed
@@ -75,57 +78,35 @@ jobs:
           check-latest: true
           #cache: false
 
-      - name: Install dependencies
+      - name: Install tools
         run: |
           go version
-          task install:deps
+          task tools
 
       # checks
-      - name: Check tidiness of go.mod and go.sum
-        run: |
-          ./scripts/mods/check_tidy
-
-      - name: Ensure generate antlr Go code is up-to-date
-        run: |
-          ./scripts/kuneiform/check_tidy
+      #- name: Ensure generate antlr Go code is up-to-date
+      #  run: |
+      #    ./scripts/kuneiform/check_tidy
 
-      - name: Initialize Go workspace
-        run: |
-          task work
+      # NOTE: we don't want to use workspace when test release
 
       - name: Compile packages, apps, and specs
         run: |
-          go build -mod=readonly ./... ./core/... ./test/specifications/
+          go build -mod=readonly ./... ./core/...
 
       - name: Lint
         uses: golangci/[email protected]
         with:
           install-mode: "binary"
           version: "latest"
-          args: ./... ./core/... ./test/... --timeout=10m --config=.golangci.yml
+          args: ./... ./core/... --timeout=10m --config=.golangci.yml
 
       # unit test
       - name: Run unit test
         run: |
           task test:unit
 
       # integration test
-      - name: Generate go vendor
-        #for faster builds and private repos, need to run this after pb:compile:v1
-        run: |
-          task vendor
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Cache Docker layers for kwild # both restore and save
-        uses: actions/cache@v4
-        with:
-          path: /tmp/.buildx-cache-kwild
-          key: ${{ runner.os }}-buildx-kwild-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-kwild
-
       - name: manual git tag
         run: |
           version=`echo ${{ github.sha }} | cut -c 1-7`
@@ -140,19 +121,34 @@ jobs:
       - name: Build cli binaries
         run: |
           task build:cli
-          task build:admin
+          task build:kwild
 
       - name: compile the core/client/example app
         run: go build -o /dev/null
         working-directory: core/client/example
+      #
+      #      - name: compile the core/gatewayclient/example app
+      #        run: go build -o /dev/null
+      #        working-directory: core/gatewayclient/example
 
-      - name: compile the core/gatewayclient/example app
-        run: go build -o /dev/null
-        working-directory: core/gatewayclient/example
+      - name: Generate go vendor
+        run: |
+          task vendor
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers for kwild # both restore and save
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache-kwild
+          key: ${{ runner.os }}-buildx-kwild-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-kwild
 
       - name: Build kwild image
         id: docker_build_kwild
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           load: true
@@ -162,7 +158,7 @@ jobs:
             version=${{ env.GIT_TAG }}
             build_time=${{ env.BUILD_TIME }}
           #  go_race=-race
-          file: ./build/package/docker/kwild.dockerfile
+          file: ./contrib/docker/kwild.dockerfile
           push: false
           tags: kwild:latest
           cache-from: type=local,src=/tmp/.buildx-cache-kwild
@@ -172,19 +168,9 @@ jobs:
         run: |
           testUserID=$(id -u)
           testGroupID=$(id -g)
-          cp test/acceptance/docker-compose.override.yml.example test/acceptance/docker-compose.override.yml
-          sed -i "s/\${UID}:\${GID}/${testUserID}:${testGroupID}/g" test/acceptance/docker-compose.override.yml
-          KACT_LOG_LEVEL=warn task test:act:nb
+          KACT_LOG_LEVEL=warn task test:act:nb -- -ugid "$testUserID:$testGroupID"
 
-      - name: Run integration test
-        run: |
-          testUserID=$(id -u)
-          testGroupID=$(id -g)
-          cp test/integration/docker-compose.override.yml.example test/integration/docker-compose.override.yml
-          cp test/integration/docker-compose-migration.override.yml.example test/integration/docker-compose-migration.override.yml
-          sed -i "s/\${UID}:\${GID}/${testUserID}:${testGroupID}/g" test/integration/docker-compose.override.yml
-          sed -i "s/\${UID}:\${GID}/${testUserID}:${testGroupID}/g" test/integration/docker-compose-migration.override.yml
-          KIT_LOG_LEVEL=warn task test:it:nb
+      # integration tests take too long, we only manually trigger it
 
       - name: Move cache
         run: |
@@ -197,4 +183,4 @@ jobs:
 
       - name: Show error log
         if: ${{ failure() }}
-        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwilAct*/*.log  /tmp/TestKwilInt*/*.log /tmp/TestKwilInt*/*/*.log
+        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwil*/*.log /tmp/TestKwil*/*/*.log
diff --git a/.github/workflows/integration-test-reuse.yaml b/.github/workflows/integration-test-reuse.yaml
@@ -0,0 +1,141 @@
+name: integration-test-reuse
+
+on:
+  workflow_call:
+    inputs:
+      # for logging purpose, if kdb-ref is from PR.
+      kdb-repo:
+        type: string
+      # for workflow_run trigger, we need to check out to the SHA, bcz most of
+      # the pr come from forked repo, and we can't access the branch, but we can
+      # access the sha(i.e, pull/xxx/head)
+      # In that case, use SHA as the input.
+      kdb-ref: # can be branch/tag/SHA
+        required: true
+        type: string
+
+jobs:
+  # JOB to run change detection
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      run: ${{ steps.filter.outputs.code }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # This may save additional git fetch roundtrip if
+          # merge-base is found within latest 20 commits
+          fetch-depth: 20
+      - uses: dorny/[email protected]
+        id: filter
+        with:
+          filters: .github/filters.yaml
+          #predicate-quantifier: 'every'
+
+  integration-test:
+    needs: changes
+    #  only run only if there are changes, or manually triggered
+    if: ${{ needs.changes.outputs.run == 'true' || github.event_name == 'workflow_dispatch'}}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Show references
+        run: |
+          echo "====== integration test references ======"
+          echo "kdb repo: ${{ inputs.kdb-repo }}"
+          echo "kdb reference: ${{ inputs.kdb-ref }}"
+
+      - name: checkout kwil-db using ref from input
+        if: ${{ inputs.kdb-ref != '' }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.kdb-ref }}
+          submodules: true
+
+      - name: checkout kwil-db using the reference or SHA for this event
+        if: ${{ inputs.kdb-ref == '' }}
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23.x'
+          check-latest: true
+
+      - name: Install Taskfile
+        uses: arduino/setup-task@v2
+
+      - name: Init workspace
+        run: |
+          task work
+
+      - name: Generate go vendor
+        run: |
+          go version
+          task vendor
+
+      - name: Build cli binaries
+        run: |
+          task build:cli
+          task build:kwild
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers for kwild # both restore and save
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache-kwild
+          key: ${{ runner.os }}-buildx-kwild-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-kwild
+
+      - name: manual git tag
+        run: |
+          version=`echo ${{ github.sha }} | cut -c 1-7`
+          echo "GIT_TAG=$version" >> $GITHUB_ENV
+
+      - name: manual build time
+        run: |
+          build_time=`TZ=UTC date -u --date="@${SOURCE_DATE_EPOCH:-$(date +%s)}" +"%Y-%m-%dT%H:%M:%SZ"`
+          echo "BUILD_TIME=$build_time" >> $GITHUB_ENV
+
+      - name: Build kwild image
+        id: docker_build_kwild
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            git_commit=${{ github.sha }}
+            version=${{ env.GIT_TAG }}
+            build_time=${{ env.BUILD_TIME }}
+          file: ./contrib/docker/kwild.dockerfile
+          push: false
+          tags: kwild:latest
+          cache-from: type=local,src=/tmp/.buildx-cache-kwild
+          cache-to: type=local,dest=/tmp/.buildx-cache-kwild-new
+
+      - name: Run integration test
+        run: |
+          testUserID=$(id -u)
+          testGroupID=$(id -g)
+          KINT_LOG_LEVEL=warn task test:it:nb -- -ugid "$testUserID:$testGroupID"
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache-kwild
+          mv /tmp/.buildx-cache-kwild-new /tmp/.buildx-cache-kwild
+
+      - name: Prune Docker
+        if: ${{ always() }}
+        run: docker rm $(docker ps -a -q) -f ; docker network prune -f ; docker volume prune -f || true
+
+      - name: Show error log
+        if: ${{ failure() }}
+        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwil*/*.log /tmp/TestKwil*/*/*.log
diff --git a/.github/workflows/kgw-test-reuse.yaml b/.github/workflows/kgw-test-reuse.yaml
@@ -83,7 +83,7 @@ jobs:
 
       - name: Init workspace
         run: |
-          go work init . ./test ./core
+          task work
 
       - name: Generate go vendor
         run: |
@@ -227,6 +227,8 @@ jobs:
 
       - name: Move cache
         run: |
+          rm -rf /tmp/.buildx-cache-kwild
+          mv /tmp/.buildx-cache-kwild-new /tmp/.buildx-cache-kwild
           rm -rf /tmp/.buildx-cache-kgw
           mv /tmp/.buildx-cache-kgw-new /tmp/.buildx-cache-kgw
 
@@ -236,4 +238,4 @@ jobs:
 
       - name: Show error log
         if: ${{ failure() }}
-        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwilInt*/*.log /tmp/TestKwilInt*/*/*.log
+        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwil*/*.log /tmp/TestKwil*/*/*.log
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -7,7 +7,6 @@ on:
 
 jobs:
   kgw-test-main:
-    if: false
     name: Run KGW test on main
     uses: ./.github/workflows/kgw-test-reuse.yaml
     with:

diff --git a/.github/workflows/manual-test-integration.yaml b/.github/workflows/manual-test-integration.yaml
@@ -0,0 +1,18 @@
+# manual-test-integration.yaml run integration test with manual inputs: kdb-ref.
+name: manual-test-kgw
+
+on:
+  workflow_dispatch:
+    inputs:
+      kdb-ref:
+        description: 'kwil-db branch/tag/SHA to be tested'
+        required: true
+        type: string
+        default: 'main'
+
+jobs:
+  integration-test-manual:
+    name: Run manual integration test
+    uses: ./.github/workflows/manual-test-integration.yaml
+    with:
+      kdb-ref: ${{ github.event.inputs.kdb-ref }}