From bd7e43b494d4cf5e444e43e6468bef82c27c2c22 Mon Sep 17 00:00:00 2001 From: Yaiba <4yaiba@gmail.com> Date: Tue, 4 Mar 2025 14:11:01 -0600 Subject: [PATCH] cleanup --- .github/workflows/ci-release.yaml | 186 ------------------ .github/workflows/integration-test-reuse.yaml | 1 - .github/workflows/main.yaml | 11 +- .../workflows/manual-test-integration.yaml | 2 +- .github/workflows/{ci-main.yaml => pr.yaml} | 4 +- .github/workflows/release.yaml | 95 +-------- 6 files changed, 15 insertions(+), 284 deletions(-) delete mode 100644 .github/workflows/ci-release.yaml rename .github/workflows/{ci-main.yaml => pr.yaml} (99%) diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml deleted file mode 100644 index 9c1d9af34..000000000 --- a/.github/workflows/ci-release.yaml +++ /dev/null @@ -1,186 +0,0 @@ -name: CI - -on: - pull_request: - types: [ opened, synchronize, reopened, ready_for_review ] - branches: - - release* - # path filter won't work bcz PR status check will be 'pending' if workflow is - # skipped due to path filtering, i.e. blocked - #paths: - - workflow_dispatch: - -jobs: - # JOB to run change detection - changes: - runs-on: ubuntu-latest - permissions: - pull-requests: read - # Set job outputs to values from filter step - outputs: - run: ${{ steps.filter.outputs.code }} - steps: - # For pull requests it's not necessary to checkout the code - # but it's useful to see the effect of the path filter in current PR - - uses: actions/checkout@v4 - with: - # This may save additional git fetch roundtrip if - # merge-base is found within latest 20 commits - fetch-depth: 20 - - uses: dorny/paths-filter@v3.0.2 - id: filter - with: - # Path to file where filters are defined - filters: .github/filters-old.yaml - #predicate-quantifier: 'every' # check dorny/paths-filter#225 - - - test: - needs: changes - # only run only if there are changes and non-draft PRs - if: ${{ needs.changes.outputs.run == 'true' && !github.event.pull_request.draft}} - runs-on: ubuntu-latest - - services: # for unit test - postgres: - image: kwildb/postgres:16.8-1 - env: - POSTGRES_HOST_AUTH_METHOD: trust - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - steps: - # shared setup for all tests - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: true - - - name: Install Taskfile - uses: arduino/setup-task@v2 - with: - # higher Github API rate limiting - repo-token: ${{ secrets.GITHUB_TOKEN }} - - #ubuntu-latest has go 1.21 installed https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md#go - #self-hosted also has go 1.21 installed - #the default behavior here will load pre-installed go version - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.23.x' - check-latest: true - #cache: false - - - name: Install tools - run: | - go version - task tools - - # checks - #- name: Ensure generate antlr Go code is up-to-date - # run: | - # ./scripts/kuneiform/check_tidy - - # NOTE: we don't want to use workspace when test release - - - name: Compile packages, apps, and specs - run: | - go build -mod=readonly ./... ./core/... - - - name: Lint - uses: golangci/golangci-lint-action@v6.5.0 - with: - install-mode: "binary" - version: "latest" - args: ./... ./core/... --timeout=10m --config=.golangci.yml - - # unit test - - name: Run unit test - run: | - task test:unit - - # integration test - - name: manual git tag - run: | - version=`echo ${{ github.sha }} | cut -c 1-7` - echo "GIT_TAG=$version" >> $GITHUB_ENV - #run: echo "GIT_TAG=`git describe --match 'v[0-9]*' --dirty --always --tags | sed 's/^v//'`" >> $GITHUB_ENV - - - name: manual build time - run: | - build_time=`TZ=UTC date -u --date="@${SOURCE_DATE_EPOCH:-$(date +%s)}" +"%Y-%m-%dT%H:%M:%SZ"` - echo "BUILD_TIME=$build_time" >> $GITHUB_ENV - - - name: Build cli binaries - run: | - task build:cli - task build:kwild - - - name: compile the core/client/example app - run: go build -o /dev/null - working-directory: core/client/example - # - # - name: compile the core/gatewayclient/example app - # run: go build -o /dev/null - # working-directory: core/gatewayclient/example - - - name: Generate go vendor - run: | - task vendor - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Cache Docker layers for kwild # both restore and save - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache-kwild - key: ${{ runner.os }}-buildx-kwild-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-kwild - - - name: Build kwild image - id: docker_build_kwild - uses: docker/build-push-action@v6 - with: - context: . - load: true - builder: ${{ steps.buildx.outputs.name }} - build-args: | - git_commit=${{ github.sha }} - version=${{ env.GIT_TAG }} - build_time=${{ env.BUILD_TIME }} - # go_race=-race - file: ./contrib/docker/kwild.dockerfile - push: false - tags: kwild:latest - cache-from: type=local,src=/tmp/.buildx-cache-kwild - cache-to: type=local,dest=/tmp/.buildx-cache-kwild-new - - - name: Run acceptance test - run: | - testUserID=$(id -u) - testGroupID=$(id -g) - KACT_LOG_LEVEL=warn task test:act:nb -- -ugid "$testUserID:$testGroupID" - - # integration tests take too long, we only manually trigger it - - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache-kwild - mv /tmp/.buildx-cache-kwild-new /tmp/.buildx-cache-kwild - - - name: Prune Docker - if: ${{ always() }} - run: docker rm $(docker ps -a -q) -f ; docker network prune -f ; docker volume prune -f || true - - - name: Show error log - if: ${{ failure() }} - run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn' /tmp/TestKwil*/*.log /tmp/TestKwil*/*/*.log diff --git a/.github/workflows/integration-test-reuse.yaml b/.github/workflows/integration-test-reuse.yaml index 9e0e8eddf..795c84804 100644 --- a/.github/workflows/integration-test-reuse.yaml +++ b/.github/workflows/integration-test-reuse.yaml @@ -81,7 +81,6 @@ jobs: - name: Build cli binaries run: | task build:cli - task build:kwild - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f19954d54..9350839b9 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -6,8 +6,8 @@ on: - main jobs: - kgw-test-main: - name: Run KGW test on main + main-kgw-test: + name: Run acceptance test (with KGW) uses: ./.github/workflows/kgw-test-reuse.yaml with: kdb-ref: main @@ -15,3 +15,10 @@ jobs: kjs-ref: main secrets: kgw-access-token: ${{ secrets.KGW_MACH_SECRET_FOR_KWILDB }} + +#UNCOMMENT this to enable integration test on main branch update +# main-integration-test: +# name: Run integration test +# uses: ./.github/workflows/integration-test-reuse.yaml +# with: +# kdb-ref: main \ No newline at end of file diff --git a/.github/workflows/manual-test-integration.yaml b/.github/workflows/manual-test-integration.yaml index 4b05eb30a..3f2151656 100644 --- a/.github/workflows/manual-test-integration.yaml +++ b/.github/workflows/manual-test-integration.yaml @@ -1,5 +1,5 @@ # manual-test-integration.yaml run integration test with manual inputs: kdb-ref. -name: manual-test-kgw +name: manual-test-integration on: workflow_dispatch: diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/pr.yaml similarity index 99% rename from .github/workflows/ci-main.yaml rename to .github/workflows/pr.yaml index d861d46ea..7166a6054 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/pr.yaml @@ -1,4 +1,4 @@ -name: CI +name: PR on: pull_request: @@ -107,7 +107,7 @@ jobs: run: | task test:unit - # integration test + # acceptance test - name: manual git tag run: | version=`echo ${{ github.sha }} | cut -c 1-7` diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f734e420a..88ff9831f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -21,8 +21,8 @@ jobs: shell: bash run: | version=`echo ${{ github.event.release.tag_name }} | sed 's/^v//' | cut -d '.' -f 2` - kgw_version=$(v9="3"; v8="3"; v7="2"; v6="1"; eval echo \${v$version}) - kjs_version=$(v9="8"; v8="7"; v7="6"; v6="5"; eval echo \${v$version}) + kgw_version=$(v10="4"; v9="3"; v8="3"; v7="2"; v6="1"; eval echo \${v$version}) + kjs_version=$(v10="9"; v9="8"; v8="7"; v7="6"; v6="5"; eval echo \${v$version}) echo "kdb_version=v0.$version" >> $GITHUB_OUTPUT echo "kgw_version=v0.$kgw_version" >> $GITHUB_OUTPUT echo "kjs_version=v0.$kjs_version" >> $GITHUB_OUTPUT @@ -38,93 +38,4 @@ jobs: # kwil-js has no release branch, we'll use the latest tag of this minor version kjs-ref: ${{ needs.parse-release-version.outputs.vkjs }} secrets: - kgw-access-token: ${{ secrets.KGW_MACH_SECRET_FOR_KWILDB }} - - build-push-image: - name: Build & push image - if: false # temporary disable - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - token: ${{ secrets.KWIL_MACH_SECRET }} - - - name: Install Taskfile - uses: arduino/setup-task@v1 - with: - repo-token: ${{ secrets.KWIL_MACH_SECRET }} - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.23' - check-latest: true - - - name: Install dependencies - env: - GH_ACCESS_TOKEN: ${{ secrets.KWIL_MACH_SECRET }} - run: | - go version - git config --global url."https://${GH_ACCESS_TOKEN}:x-oauth-basic@github.com/kwilteam/".insteadOf "https://github.com/kwilteam/" - task tools - - - name: Generate go vendor - run: | - task vendor - - - name: configure variables - id: config - run: | - version=`echo ${{ github.event.release.tag_name }} | sed 's/^v//'` - echo "version=$version" >> $GITHUB_OUTPUT - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} - aws-region: us-east-1 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers for KD - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache-kwild - #key: ${{ runner.os }}-buildx-kwild-${{ github.sha }} - key: ${{ runner.os }}-buildx-kwild - restore-keys: | - ${{ runner.os }}-buildx-kwild - - - name: Build & push kwild image - id: docker_build_kwild - uses: docker/build-push-action@v4 - with: - context: . - builder: ${{ steps.buildx.outputs.name }} - build-args: | - git_commit=${{ github.sha }} - version=${{ steps.config.outputs.version }} - build_time=${{ github.event.release.created_at }} - file: ./build/package/docker/kwild.dockerfile - push: true - tags: | - ${{ steps.login-ecr.outputs.registry }}/${{ secrets.AWS_KWILD_ECR }}:${{ steps.config.outputs.version }} - ${{ steps.login-ecr.outputs.registry }}/${{ secrets.AWS_KWILD_ECR }}:latest - cache-from: type=local,src=/tmp/.buildx-cache-kwild - cache-to: type=local,dest=/tmp/.buildx-cache-kwild-new - - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache-kwild - mv /tmp/.buildx-cache-kwild-new /tmp/.buildx-cache-kwild + kgw-access-token: ${{ secrets.KGW_MACH_SECRET_FOR_KWILDB }} \ No newline at end of file