build_test_deploy.yml

name: CI Pipeline

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - '*'
  workflow_dispatch:
    inputs:
      debug_enabled:
        type: boolean
        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: false

env:
  USER_NAME: root
  USER_UID: 1000
  USER_GID: 1000
  DOCKER_USER: ${{ secrets.DOCKER_USER }}
  DOCKER_PASS: ${{ secrets.DOCKER_PASS }}

jobs:
  static_frontend_code_checks:
    runs-on: ubuntu-24.04
    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 5

    - uses: actions/setup-node@v4
      with:
        node-version: 18

    - name: Install frontend dependencies
      run: corepack enable && yarn install --immutable

    - name: Lint frontend code and check formatting
      run: yarn run check-frontend

    - name: Typecheck frontend code
      run: yarn typecheck

    - name: Check for cyclic dependencies in frontend
      run: yarn check-cyclic-dependencies


  build_test_deploy:
    runs-on: ubuntu-24.04
    env:
      # The hostname used to communicate with the PostgreSQL service container
      POSTGRES_HOST: localhost
      POSTGRES_PORT: 5432
    services:
        postgres:
          image: postgres:10-alpine
          env:
            POSTGRES_DB: webknossos
            POSTGRES_USER: webknossos_user
            POSTGRES_PASSWORD: secret_password
          # Set health checks to wait until postgres has started
          options: >-
            --health-cmd pg_isready
            --health-interval 2s
            --health-timeout 5s
            --health-retries 30
          ports:
            - 5432:5432
    steps:
      # Enable tmate debugging of manually-triggered workflows if the input option was provided
      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          fetch-depth: 5

      - name: "Custom environment variables"
        run: |
          if [[ ${{ github.ref_type }} == "branch" ]]; then
            NORMALIZED_BRANCH=$(echo ${{ github.ref_name }} | sed 's/[\/-]/_/g')
            echo "NORMALIZED_BRANCH=$NORMALIZED_BRANCH" >> $GITHUB_ENV
            DOCKER_TAG="${NORMALIZED_BRANCH}__${{ github.run_number }}"
            echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
          else
            echo "NORMALIZED_BRANCH=master" >> $GITHUB_ENV
            echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
          fi
      - uses: actions/setup-node@v4
        with:
          node-version: 18

      - name: Install frontend dependencies
        run: corepack enable && yarn install --immutable
      
      - name: Build webknossos (webpack)
        run: yarn build

      - name: Set up Java / SBT
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin' # See 'Supported distributions' for available options
          java-version: '21'
          cache: 'sbt'
      
      - name: Install OS dependencies / libs
        run: sudo apt-get update && sudo apt-get install -y \
          libdraco-dev \
          libblosc-dev

                    # build-essential \
          # cmake \

      # - name: Prepare dependency folders
      #   run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier

      # - name: Assert unique evolution numbers
      #   run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers

      # - name: Assert schema.sql and evolutions are equal
      #   run: |
      #     docker compose up -d postgres
      #     sleep 3
      #     docker compose run compile tools/postgres/dbtool.js check-evolutions-schema

      - name: Build webknossos (sbt)
        run: |
          if [ "${{ github.ref }}" == "refs/heads/master" ]; then
            sbt -no-colors clean compile stage
          else
            sbt -no-colors -DfailOnWarning compile stage
          fi

      - name: Build webknossos-datastore (sbt)
        run: sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage

      - name: Build webknossos-tracingstore (sbt)
        run: sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage

      - name: Checksum App Dirs
        run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt

      - name: Set up Docker
        uses: docker/setup-buildx-action@v2

      - name: Build webknossos docker image
        run: |
          docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
          DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos

      - name: Build webknossos-datastore docker image
        run: docker compose build --pull webknossos-datastore

      - name: Build webknossos-tracingstore docker image
        run: docker compose build --pull webknossos-tracingstore

      - name: Lint backend code and check formatting
        run: .github/not-on-master.sh docker compose run backend-lint-format

      - name: Run backend tests
        run: .github/not-on-master.sh docker compose run backend-tests

      - name: Run end-to-end tests
        run: |
          for i in {1..3}; do # retry
            .github/not-on-master.sh docker compose run e2e-tests && s=0 && break || s=$?
          done
          (exit $s)

      - name: Start webknossos, datastore, and tracingstore
        run: |
          docker compose up -d webknossos
          docker compose up -d webknossos-datastore
          docker compose up -d webknossos-tracingstore

      - name: Run webknossos smoke test
        uses: ./.github/actions/health_check_action
        with: 
          url: http://localhost:9000/api/health
          
      - name: Run webknossos-datastore smoke test
        uses: ./.github/actions/health_check_action
        with: 
          url: http://localhost:9090/data/health

      - name: Run webknossos-tracingstore smoke test
        uses: ./.github/actions/health_check_action
        with: 
          url: http://localhost:9050/tracings/health

      - name: Stop webknossos, datastore, and tracingstore
        run: docker compose down --volumes --remove-orphans

      - name: Push docker images
        run: |
          function retry() {
            for i in {1..5}; do
              "$@" && s=0 && break || s=$?
              sleep 10
            done
            return $s
          }
          retry docker login -u $DOCKER_USER -p $DOCKER_PASS
          retry docker compose push webknossos
          retry docker compose push webknossos-datastore
          retry docker compose push webknossos-tracingstore
          if [[ ${{ github.ref_type }} == "branch" ]]; then
            docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH}
            retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH}
            docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
            retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
            docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
            retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
            docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
            retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
          fi
          docker logout

      - name: Build frontend documentation
        run: |
          WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }}
          yarn run docs --project-version $WK_VERSION


      - name: Report coverage
        run: .github/not-on-master.sh docker compose run base yarn coverage || true

      # - name: Send Slack notification (master only)
      #   run: .circleci/slack-notification.sh