prestocpp-linux-build-and-unit-test.yml

name: prestocpp-linux-build-and-unit-test

on:
  workflow_dispatch:
  pull_request:
    paths:
      - 'presto-native-execution/**'
      - '.github/workflows/prestocpp-linux-build-and-unit-test.yml'
  push:
    branches:
      - master
    paths-ignore:
      - 'presto-docs/**'

jobs:
  prestocpp-linux-build-for-test:
    runs-on: ubuntu-22.04
    container:
      image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
    env:
      CCACHE_DIR: "${{ github.workspace }}/ccache"
    steps:
      - uses: actions/checkout@v4

      - name: Fix git permissions
        # Usually actions/checkout does this but as we run in a container
        # it doesn't work
        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

      - name: Update velox
        run: |
          cd presto-native-execution
          make velox-submodule

      - name: Install Github CLI for using apache/infrastructure-actions/stash
        run: |
          curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm
          rpm -iv gh_2.63.2_linux_amd64.rpm

      - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
        with:
          path: '${{ env.CCACHE_DIR }}'
          key: ccache-prestocpp-linux-build-for-test

      - name: Zero ccache statistics
        run: ccache -sz

      - name: Build engine
        run: |
          source /opt/rh/gcc-toolset-12/enable
          cd presto-native-execution
          cmake \
            -B _build/release \
            -GNinja \
            -DTREAT_WARNINGS_AS_ERRORS=1 \
            -DENABLE_ALL_WARNINGS=1 \
            -DCMAKE_BUILD_TYPE=Release \
            -DPRESTO_ENABLE_PARQUET=ON \
            -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
            -DPRESTO_ENABLE_JWT=ON \
            -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
            -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
            -DCMAKE_PREFIX_PATH=/usr/local \
            -DThrift_ROOT=/usr/local \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DMAX_LINK_JOBS=4
          ninja -C _build/release -j 4

      - name: Ccache after
        run: ccache -s

      - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
        with:
          path: '${{ env.CCACHE_DIR }}'
          key: ccache-prestocpp-linux-build-for-test

      - name: Run Unit Tests
        run: |
          # Ensure transitive dependency libboost-iostreams is found.
          ldconfig /usr/local/lib
          cd presto-native-execution/_build/release
          ctest -j 4 -VV --output-on-failure --exclude-regex velox.*

      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: presto-native-build
          path: |
            presto-native-execution/_build/release/presto_cpp/main/presto_server
            presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main

  prestocpp-linux-presto-e2e-tests:
    needs: prestocpp-linux-build-for-test
    runs-on: ubuntu-22.04
    container:
      image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
    env:
      MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
      MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
      MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
    steps:
      - uses: actions/checkout@v4

      - name: Fix git permissions
        # Usually actions/checkout does this but as we run in a container
        # it doesn't work
        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: presto-native-build
          path: presto-native-execution/_build/release

      # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
      - name: Restore execute permissions and library path
        run: |
          chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
          chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
          # Ensure transitive dependency libboost-iostreams is found.
          ldconfig /usr/local/lib

      - name: Install OpenJDK8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'

      - name: Cache local Maven repository
        id: cache-maven
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-2-

      - name: Populate maven cache
        if: steps.cache-maven.outputs.cache-hit != 'true'
        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies

      - name: Maven install
        env:
          # Use different Maven options to install.
          MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
        run: |
          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)

      - name: Run presto-native e2e tests
        run: |
          export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
          export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'`
          # Convert file paths to comma separated class names
          export TESTCLASSES=
          for test_file in $TESTFILES
          do
            tmp=${test_file##*/}
            test_class=${tmp%%\.*}
            export TESTCLASSES="${TESTCLASSES},$test_class"
          done
          export TESTCLASSES=${TESTCLASSES#,}
          echo "TESTCLASSES = $TESTCLASSES"
          # TODO: neeed to enable remote function tests with
          # "-Ppresto-native-execution-remote-functions" once
          # > https://github.com/facebookincubator/velox/discussions/6163
          # is fixed.

          mvn test \
            ${MAVEN_TEST} \
            -pl 'presto-native-execution' \
            -Dtest="${TESTCLASSES}" \
            -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
            -DDATA_DIR=${RUNNER_TEMP} \
            -Duser.timezone=America/Bahia_Banderas \
            -T1C

  prestocpp-linux-spark-e2e-tests:
    needs: prestocpp-linux-build-for-test
    runs-on: ubuntu-22.04
    container:
      image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
    env:
      MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
      MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
      MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
    steps:
      - uses: actions/checkout@v4
      - name: Fix git permissions
        # Usually actions/checkout does this but as we run in a container
        # it doesn't work
        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: presto-native-build
          path: presto-native-execution/_build/release

      # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
      - name: Restore execute permissions and library path
        run: |
          chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
          chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
          # Ensure transitive dependency libboost-iostreams is found.
          ldconfig /usr/local/lib

      - name: Check Java
        run: java --version

      - name: Cache local Maven repository
        id: cache-maven
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-2-

      - name: Populate maven cache
        if: steps.cache-maven.outputs.cache-hit != 'true'
        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies

      - name: Maven install
        env:
          # Use different Maven options to install.
          MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
        run: |
          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)

      - name: Run spark e2e tests
        run: |
          export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
          export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'`
          # Convert file paths to comma separated class names
          export TESTCLASSES=
          for test_file in $TESTFILES
          do
            tmp=${test_file##*/}
            test_class=${tmp%%\.*}
            export TESTCLASSES="${TESTCLASSES},$test_class"
          done
          export TESTCLASSES=${TESTCLASSES#,}
          # Override the possible test classes with test classes known to pass from the previous CI.
          # A number of newly added tests are flaky and are disabled.
          export TESTCLASSES=TestPrestoSparkExpressionCompiler,TestPrestoSparkNativeBitwiseFunctionQueries,TestPrestoSparkNativeTpchConnectorQueries,TestPrestoSparkNativeSimpleQueries,TestPrestoSparkSqlFunctions,TestPrestoSparkNativeTpchQueries
          echo "TESTCLASSES = $TESTCLASSES"
          mvn test \
            ${MAVEN_TEST} \
            -pl 'presto-native-execution' \
            -Dtest="${TESTCLASSES}" \
            -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
            -DDATA_DIR=${RUNNER_TEMP} \
            -Duser.timezone=America/Bahia_Banderas \
            -T1C

  prestocpp-linux-presto-sidecar-tests:
    needs: prestocpp-linux-build-for-test
    runs-on: ubuntu-22.04
    container:
      image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
    env:
      MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
      MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
      MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
    steps:
      - uses: actions/checkout@v4
      - name: Fix git permissions
        # Usually actions/checkout does this but as we run in a container
        # it doesn't work
        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: presto-native-build
          path: presto-native-execution/_build/release

      # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
      - name: Restore execute permissions and library path
        run: |
          chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
          chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
          # Ensure transitive dependency libboost-iostreams is found.
          ldconfig /usr/local/lib

      - name: Install OpenJDK8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'

      - name: Cache local Maven repository
        id: cache-maven
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-2-

      - name: Populate maven cache
        if: steps.cache-maven.outputs.cache-hit != 'true'
        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies

      - name: Maven install
        env:
          # Use different Maven options to install.
          MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
        run: |
          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)

      - name: Run presto-native sidecar tests
        run: |
          export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
          export TESTFILES=`find ./presto-native-sidecar-plugin/src/test -type f -name 'Test*.java'`
          # Convert file paths to comma separated class names
          export TESTCLASSES=
          for test_file in $TESTFILES
          do
            tmp=${test_file##*/}
            test_class=${tmp%%\.*}
            export TESTCLASSES="${TESTCLASSES},$test_class"
          done
          export TESTCLASSES=${TESTCLASSES#,}
          echo "TESTCLASSES = $TESTCLASSES"
          mvn test \
            ${MAVEN_TEST} \
            -pl 'presto-native-sidecar-plugin' \
            -Dtest="${TESTCLASSES}" \
            -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
            -DDATA_DIR=${RUNNER_TEMP} \
            -Duser.timezone=America/Bahia_Banderas \
            -T1C