cpp_integration.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Celeborn Cpp Integration Test

on:
  push:
    branches:
      - main
      - branch-*
  pull_request:
    branches:
      - main
      - branch-*

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  celeborn_cpp_check_lint:
    runs-on: ubuntu-22.04
    container: jraaaay/celeborn-cpp-dev:0.4
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: 'false'
          fetch-depth: 0
      - name: Check Cpp Code Lint
        working-directory: ./cpp
        run: |
          find ./ -iname '*.h' -o -iname '*.cpp' | \
            xargs clang-format-15 -style=file:./.clang-format -n --Werror
  celeborn_cpp_unit_test:
    runs-on: ubuntu-22.04
    container: jraaaay/celeborn-cpp-dev:0.4
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: 'false'
          fetch-depth: 0
      - name: Compile Celeborn Cpp
        working-directory: ./cpp
        run: |
          rm -rf build && mkdir -p build && cd build
          cmake .. && make -j
      - name: Run Unittests of Celeborn Cpp
        working-directory: ./cpp/build
        run: ctest
  celeborn_cpp_integration_test:
    runs-on: ubuntu-22.04
    container: jraaaay/celeborn-cpp-dev:0.4
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: 'false'
          fetch-depth: 0
      - name: Compile Celeborn Cpp
        working-directory: ./cpp
        run: |
          rm -rf build && mkdir -p build && cd build
          cmake .. && make -j
      # This step is necessary because within container the setup-java step would pollute the $PATH
      # and cause following steps fail. We manually back up the $PATH here to avoid the problem.
      - name: Backup Path
        run: |
          echo $PATH
          echo "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >>"$GITHUB_PATH"
      - name: Setup JDK 8
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: 8
          cache: maven
          check-latest: false
      - name: Compile & Install Celeborn Java
        run: build/mvn clean install -DskipTests
      - name: Run Java-Write Cpp-Read Hybrid Integration Test (NONE Decompression)
        run: |
          MAVEN_OPTS="-XX:MaxDirectMemorySize=2G" build/mvn -pl worker \
            test-compile exec:java \
            -Dexec.classpathScope="test" \
            -Dexec.mainClass="org.apache.celeborn.service.deploy.cluster.JavaWriteCppReadTestWithNONE"
      - name: Run Java-Write Cpp-Read Hybrid Integration Test (LZ4 Decompression)
        run: |
          MAVEN_OPTS="-XX:MaxDirectMemorySize=2G" build/mvn -pl worker \
            test-compile exec:java \
            -Dexec.classpathScope="test" \
            -Dexec.mainClass="org.apache.celeborn.service.deploy.cluster.JavaWriteCppReadTestWithLZ4"
      - name: Run Java-Write Cpp-Read Hybrid Integration Test (ZSTD Decompression)
        run: |
          MAVEN_OPTS="-XX:MaxDirectMemorySize=2G" build/mvn -pl worker \
            test-compile exec:java \
            -Dexec.classpathScope="test" \
            -Dexec.mainClass="org.apache.celeborn.service.deploy.cluster.JavaWriteCppReadTestWithZSTD"
      - name: Run Cpp-Write Java-Read Hybrid Integration Test (NONE Compression)
        run: |
          MAVEN_OPTS="-XX:MaxDirectMemorySize=2G" build/mvn -pl worker \
            test-compile exec:java \
            -Dexec.classpathScope="test" \
            -Dexec.mainClass="org.apache.celeborn.service.deploy.cluster.CppWriteJavaReadTestWithNONE"
      - name: Run Cpp-Write Java-Read Hybrid Integration Test (LZ4 Compression)
        run: |
          MAVEN_OPTS="-XX:MaxDirectMemorySize=2G" build/mvn -pl worker \
            test-compile exec:java \
            -Dexec.classpathScope="test" \
            -Dexec.mainClass="org.apache.celeborn.service.deploy.cluster.CppWriteJavaReadTestWithLZ4"
      - name: Run Cpp-Write Java-Read Hybrid Integration Test (ZSTD Compression)
        run: |
          MAVEN_OPTS="-XX:MaxDirectMemorySize=2G" build/mvn -pl worker \
            test-compile exec:java \
            -Dexec.classpathScope="test" \
            -Dexec.mainClass="org.apache.celeborn.service.deploy.cluster.CppWriteJavaReadTestWithZSTD"
      - name: Run Cpp-MergeWrite Java-Read Hybrid Integration Test (NONE Compression)
        run: |
          MAVEN_OPTS="-XX:MaxDirectMemorySize=2G" build/mvn -pl worker \
            test-compile exec:java \
            -Dexec.classpathScope="test" \
            -Dexec.mainClass="org.apache.celeborn.service.deploy.cluster.CppMergeWriteJavaReadTestWithNONE"
      - name: Run Cpp-MergeWrite Java-Read Hybrid Integration Test (LZ4 Compression)
        run: |
          MAVEN_OPTS="-XX:MaxDirectMemorySize=2G" build/mvn -pl worker \
            test-compile exec:java \
            -Dexec.classpathScope="test" \
            -Dexec.mainClass="org.apache.celeborn.service.deploy.cluster.CppMergeWriteJavaReadTestWithLZ4"
      - name: Run Cpp-MergeWrite Java-Read Hybrid Integration Test (ZSTD Compression)
        run: |
          MAVEN_OPTS="-XX:MaxDirectMemorySize=2G" build/mvn -pl worker \
            test-compile exec:java \
            -Dexec.classpathScope="test" \
            -Dexec.mainClass="org.apache.celeborn.service.deploy.cluster.CppMergeWriteJavaReadTestWithZSTD"