[FEATURE-3] Replace cppkafka with librdkafka, and removing everything using boost. (#6)

Signed-off-by: Tao He <[email protected]>

Author: sighingnow

.github/workflows/build-doc.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Install Dependencies
         run: |
           sudo apt update -y
-          sudo apt install -y libmpich-dev libgoogle-glog-dev libgflags-dev libboost-dev librdkafka-dev doxygen
+          sudo apt install -y libmpich-dev libgoogle-glog-dev libgflags-dev librdkafka-dev doxygen
       - name: Build Doc
         run: |
           mkdir build

.github/workflows/c-cpp.yml

@@ -23,13 +23,13 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           sudo apt update -y
-          sudo apt install -y libmpich-dev libgoogle-glog-dev libgflags-dev libboost-dev librdkafka-dev
+          sudo apt install -y libmpich-dev libgoogle-glog-dev libgflags-dev librdkafka-dev
       - name: Install Dependencies for macOS
         if: runner.os == 'macOS'
         run: |
           sudo scutil --set HostName my_computer.local
           echo '127.0.0.1       my_computer.local' | sudo tee -a /etc/hosts
-          brew install mpich glog gflags boost librdkafka
+          brew install mpich glog gflags librdkafka
       - name: Build
         run: |
           mkdir build

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "examples/gnn_sampler/thirdparty/cppkafka"]
-	path = examples/gnn_sampler/thirdparty/cppkafka
-	url = https://github.com/mfontanini/cppkafka.git

CMakeLists.txt

@@ -57,16 +57,13 @@ SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -g")
 # ------------------------------------------------------------------------------
 find_package(MPI REQUIRED)
 include_directories(SYSTEM ${MPI_CXX_INCLUDE_PATH})
-find_package(Boost REQUIRED)
-include_directories(${Boost_INCLUDE_DIRS})
 
 # find Threads------------------------------------------------------------------
 set(CMAKE_THREAD_PREFER_PTHREAD ON)
 find_package(Threads REQUIRED)
 
 # find glog---------------------------------------------------------------------
 include("cmake/FindGlog.cmake")
-
 if (NOT GLOG_FOUND)
     message(FATAL_ERROR "glog not found, please install the glog library")
 else ()
@@ -92,6 +89,12 @@ if (USE_JEMALLOC)
     endif ()
 endif ()
 
+# find rdkafka---------------------------------------------------------------------
+include("cmake/FindRdkafka.cmake")
+if (NOT RDKAFKA_FOUND)
+    message(STATUS "rdkafka not found, build without rdkafka")
+endif ()
+
 # ------------------------------------------------------------------------------
 # generete libgrape-lite
 # ------------------------------------------------------------------------------
@@ -107,7 +110,7 @@ if (JEMALLOC_FOUND)
 endif ()
 
 if (NOT GFLAGS_FOUND)
-    message(WARNING "Disable examples because gflags not found")
+    message(WARNING "Disable analytical_apps because gflags not found")
 else ()
     add_executable(analytical_apps examples/analytical_apps/flags.cc
             examples/analytical_apps/run_app.cc)
@@ -118,24 +121,19 @@ else ()
             ${GLOG_LIBRARIES} ${GFLAGS_LIBRARIES} ${CMAKE_DL_LIBS})
 endif ()
 
-if (NOT (EXISTS ${CMAKE_SOURCE_DIR}/examples/gnn_sampler/thirdparty/cppkafka/CMakeLists.txt))
-    message(WARNING "Disable gnn_sampler because submodule cppkafka isn't checkout not")
-elseif (NOT GFLAGS_FOUND)
-    message(WARNING "Disable examples because gflags not found")
+if (NOT GFLAGS_FOUND)
+    message(WARNING "Disable gnn_sampler because gflags not found")
+elseif (NOT RDKAFKA_FOUND)
+    message(WARNING "Disable gnn_sampler because rdkafka not found")
 else ()
-    set(CPPKAFKA_DISABLE_TESTS ON CACHE BOOL "Disable cppkafka tests")
-    set(CPPKAFKA_DISABLE_EXAMPLES ON CACHE BOOL "Disable cppkafka examples")
-    add_subdirectory(examples/gnn_sampler/thirdparty/cppkafka)
-
     add_executable(gnn_sampler examples/gnn_sampler/run_sampler.cc)
     set_target_properties(gnn_sampler PROPERTIES OUTPUT_NAME run_sampler)
     target_include_directories(gnn_sampler PRIVATE
             examples/gnn_sampler
             examples/gnn_sampler/thirdparty
-            exampler/gnn_sampler/thirdparty/cppkafka/include
             ${RDKAFKA_INCLUDE_DIR})
-    target_link_libraries(gnn_sampler grape-lite cppkafka ${MPI_CXX_LIBRARIES}
-            ${GLOG_LIBRARIES} ${GFLAGS_LIBRARIES} ${CMAKE_DL_LIBS} ${Boost_LIBRARIES})
+    target_link_libraries(gnn_sampler grape-lite  ${MPI_CXX_LIBRARIES}
+            ${GLOG_LIBRARIES} ${GFLAGS_LIBRARIES} ${CMAKE_DL_LIBS} ${RDKAFKA_LIBRARIES})
 endif ()
 
 # ------------------------------------------------------------------------------

LICENSE

@@ -248,7 +248,7 @@ DEALINGS IN THE SOFTWARE.
 
 -------------------------------------------------------------------------------
 
-cmake/{FindGFlags.cmake,FindGlog.cmake,FindJemalloc.cmake} is based on code
+cmake/{FindGFlags.cmake,FindGlog.cmake,FindJemalloc.cmake,FindRdkafka.cmake} is based on code
 from BVLC/Caffe, has the following license:
 
 COPYRIGHT
@@ -665,36 +665,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 -------------------------------------------------------------------------------
 
-examples/gnn_sampler/thirdparty/cppkafka: BSD 2-Clause License
-
-Copyright (c) 2016-2017, Matias Fontanini
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-* Redistributions of source code must retain the above copyright
-  notice, this list of conditions and the following disclaimer.
-* Redistributions in binary form must reproduce the above
-  copyright notice, this list of conditions and the following disclaimer
-  in the documentation and/or other materials provided with the
-  distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
--------------------------------------------------------------------------------
-
 examples/gnn_sampler/thirdparty/xoroshiro: The MIT License (MIT)
 
 The MIT License (MIT)

NOTICE.txt

@@ -19,10 +19,6 @@ This product includes software from the Caffe project (BSD, 2-clause)
 This product includes software from the libcxx project (Apache License v2.0 with LLVM Exceptions)
  * https://github.com/llvm-mirror/libcxx
 
-This product includes software from the cppkafka project (BSD, 2-Clause)
- * Copyright (c) 2016-2017, Matias Fontanini
- * https://github.com/mfontanini/cppkafka
-
 This product includes software from the xoroshiro project (MIT)
  * Copyright (c) 2017-19 Melissa E. O'Neill
  * http://prng.di.unimi.it/

README.md

@@ -30,9 +30,8 @@ Here are the dependencies for optional features:
 
 Extra dependencies are required by examples:
 - [gflags](https://github.com/gflags/gflags) (>= 2.2.0);
-- [boost](https://www.boost.org/) (>= 1.58);
 - [Apache Kafka](https://github.com/apache/kafka) (>= 2.3.0);
-- [librdkafka](https://github.com/edenhill/librdkafka)(>1.4.0) as the dependency for [cppkafka](https://github.com/mfontanini/cppkafka) (included as a submodule).
+- [librdkafka](https://github.com/edenhill/librdkafka)(>= 0.11.3);
 
 
 ### Building libgrape-lite and examples
@@ -107,7 +106,6 @@ The latest version of online documentation can be found at [https://alibaba.gith
 
 - [flat_hash_map](https://github.com/skarupke/flat_hash_map), an efficient hashmap implementation;
 - [granula](https://github.com/atlarge-research/granula), a tool for gathering performance information for LDBC Benchmark;
-- [cppkafka](https://github.com/mfontanini/cppkafka), a C++ wrapper of librdkafka;
 - [xoroshiro](http://prng.di.unimi.it/), a pseudo-random number generator.
 
 

cmake/FindRdkafka.cmake

@@ -0,0 +1,39 @@
+# This file is used to find librdkafka library in CMake script, modifeid from the
+# code from
+#
+#   https://github.com/BVLC/caffe/blob/master/cmake/Modules/FindGlog.cmake
+#
+# which is licensed under the 2-Clause BSD License.
+#
+# - Try to find librdkafka
+#
+# The following variables are optionally searched for defaults
+#  RDKAFKA_ROOT_DIR:            Base directory where all RDKAFKA components are found
+#
+# The following are set after configuration is done:
+#  RDKAFKA_FOUND
+#  RDKAFKA_INCLUDE_DIRS
+#  RDKAFKA_LIBRARIES
+#  RDKAFKA_LIBRARY_DIRS
+
+include(FindPackageHandleStandardArgs)
+
+set(RDKAFKA_ROOT_DIR "" CACHE PATH "Folder contains librdkafka")
+
+# We are testing only a couple of files in the include directories
+find_path(RDKAFKA_INCLUDE_DIR librdkafka PATHS ${RDKAFKA_ROOT_DIR}/include)
+
+find_library(RDKAFKA_LIBRARY rdkafka PATHS  ${RDKAFKA_ROOT_DIR}/lib)
+find_library(RDKAFKA++_LIBRARY rdkafka++ PATHS  ${RDKAFKA_ROOT_DIR}/lib)
+
+find_package_handle_standard_args(RDKAFKA DEFAULT_MSG RDKAFKA_INCLUDE_DIR RDKAFKA_LIBRARY)
+
+
+if(RDKAFKA_FOUND)
+    set(RDKAFKA_INCLUDE_DIRS ${RDKAFKA_INCLUDE_DIR})
+    # The RDKAFKA_LIBRARY comes later, since it is depended by the former.
+    set(RDKAFKA_LIBRARIES ${RDKAFKA++_LIBRARY} ${RDKAFKA_LIBRARY})
+    message(STATUS "Found rdkafka (include: ${RDKAFKA_INCLUDE_DIRS}, library: ${RDKAFKA_LIBRARIES})")
+    mark_as_advanced(RDKAFKA_LIBRARY_DEBUG RDKAFKA_LIBRARY_RELEASE
+                     RDKAFKA_LIBRARY RDKAFKA_INCLUDE_DIR RDKAFKA_ROOT_DIR)
+endif()