Skip to content

TsFile C++ batch read/write optimization #823

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ColinLeeo wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

TsFile C++ batch read/write optimization #823

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 34 additions & 94 deletions cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#[[
#[[
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
Expand Down Expand Up @@ -26,38 +26,16 @@ else()
message(STATUS "Not using ToolChain")
endif ()

if (POLICY CMP0079)
cmake_policy(SET CMP0079 NEW)
endif ()
cmake_policy(SET CMP0079 NEW)
set(TsFile_CPP_VERSION 2.2.1.dev)

if (MSVC)
# MSVC does not provide a /std:c++11 flag; C++11 is its implicit baseline.
# The lowest explicitly settable standard is /std:c++14. Without this flag,
# the default varies by VS version (VS2017+ defaults to C++14 mode with some
# C++17 extensions), so we pin it explicitly for reproducibility.
set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} /W3 /utf-8 /EHsc /bigobj /Zc:__cplusplus /std:c++14")
add_definitions(-DNOMINMAX -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_WARNINGS
-D_SCL_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS)
# Export all symbols of the tsfile shared library automatically so that
# consumers do not need __declspec(dllexport) annotations.
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
else ()
set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall")
endif ()
set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall")

if (CMAKE_CXX_COMPILER_ID MATCHES "GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wunused -Wuninitialized -D__STDC_FORMAT_MACROS")
endif ()

message("cmake using: USE_CPP11=${USE_CPP11}")
# MSVC has no /std:c++11; CMake maps this to the closest supported standard
# (C++14 default on MSVC), which compiles the C++11 codebase fine.
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED OFF)
if (NOT MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
endif ()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")

if (DEFINED ENV{CXX})
set(CMAKE_CXX_COMPILER $ENV{CXX})
Expand All @@ -80,13 +58,6 @@ if (${COV_ENABLED})
message("add_definitions -DCOV_ENABLED=1")
endif ()

option(ENABLE_MEM_STAT "Enable memory status" ON)

if (ENABLE_MEM_STAT)
add_definitions(-DENABLE_MEM_STAT)
message("add_definitions -DENABLE_MEM_STAT")
endif ()


if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE)
Expand All @@ -105,52 +76,25 @@ else ()
endif ()

message("CMAKE BUILD TYPE " ${CMAKE_BUILD_TYPE})
# MSVC provides sensible per-configuration optimization flags by default; the
# GCC-style flags below would be rejected by cl.exe, so skip them on MSVC.
if (NOT MSVC)
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g")
elseif (CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O2")
elseif (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O2 -g")
elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")
set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} -ffunction-sections -fdata-sections -Os")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
endif ()
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g")
elseif (CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -march=native -flto")
elseif (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O2 -g")
elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")
set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} -ffunction-sections -fdata-sections -Os")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
endif ()
message("CMAKE DEBUG: CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")

# disable asan by default.
option(ENABLE_ASAN "Enable Address Sanitizer" OFF)

if (ENABLE_ASAN)
message("Address Sanitizer is enabled.")
if (MSVC)
# MSVC ships AddressSanitizer; it requires Visual Studio 2019 16.9 or
# newer (MSVC_VERSION >= 1928). Only the address sanitizer is available
# (there is no UndefinedBehaviorSanitizer for MSVC).
if (MSVC_VERSION LESS 1928)
message(FATAL_ERROR
"ENABLE_ASAN requires MSVC 19.28+ (Visual Studio 2019 16.9); "
"detected MSVC_VERSION=${MSVC_VERSION}.")
endif ()
# /fsanitize=address is incompatible with the /RTC* runtime checks that
# CMake injects into Debug builds, and with incremental linking. Strip
# /RTC* from the per-config flags and force non-incremental linking.
#
# ASan also needs debug info: /Zi (compile) + /DEBUG (link). Without it
# MSVC emits warning C5072 ("ASAN enabled without debug information
# emission"), which the bundled googletest build promotes to an error
# via /WX in Release builds, and ASan reports lose symbol/line info.
add_compile_options(/fsanitize=address /Zi)
foreach (flagsVar
CMAKE_C_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG
CMAKE_C_FLAGS_RELWITHDEBINFO CMAKE_CXX_FLAGS_RELWITHDEBINFO)
string(REGEX REPLACE "/RTC[1csu]+" "" ${flagsVar} "${${flagsVar}}")
endforeach ()
add_link_options(/INCREMENTAL:NO /DEBUG)
elseif (NOT WIN32)
if (NOT WIN32)
if (ENABLE_ASAN)
message("Address Sanitizer is enabled.")

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address,undefined -fno-omit-frame-pointer")

if (NOT APPLE)
Expand All @@ -160,12 +104,8 @@ if (ENABLE_ASAN)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address,undefined")
endif ()
else ()
message(WARNING
"ENABLE_ASAN on Windows is only supported with the MSVC toolchain; "
"ignoring it for the current generator.")
message("Address Sanitizer is disabled.")
endif ()
else ()
message("Address Sanitizer is disabled.")
endif ()

option(BUILD_TEST "Build tests" ON)
Expand Down Expand Up @@ -203,20 +143,21 @@ if (ENABLE_ZLIB)
add_definitions(-DENABLE_GZIP)
endif()

option(ENABLE_SIMD "Enable SIMD acceleration via SIMDe" ON)
message("cmake using: ENABLE_SIMD=${ENABLE_SIMD}")

option(ENABLE_THREADS "Enable multi-threaded read/write (requires pthreads)" ON)
message("cmake using: ENABLE_THREADS=${ENABLE_THREADS}")

if (ENABLE_THREADS)
add_definitions(-DENABLE_THREADS)
find_package(Threads REQUIRED)
link_libraries(Threads::Threads)
endif()

option(ENABLE_SIMDE "Enable SIMDe (SIMD Everywhere)" OFF)
message("cmake using: ENABLE_SIMDE=${ENABLE_SIMDE}")
option(ENABLE_MEM_STAT "Enable per-module memory allocation statistics" ON)
message("cmake using: ENABLE_MEM_STAT=${ENABLE_MEM_STAT}")

if (ENABLE_SIMDE)
add_definitions(-DENABLE_SIMDE)
if (ENABLE_MEM_STAT)
add_definitions(-DENABLE_MEM_STAT)
endif()

# All libs will be stored here, including libtsfile, compress-encoding lib.
Expand All @@ -231,15 +172,7 @@ set(LIBRARY_INCLUDE_DIR ${PROJECT_BINARY_DIR}/include CACHE STRING "TsFile inclu
set(THIRD_PARTY_INCLUDE ${PROJECT_BINARY_DIR}/third_party)

set(SAVED_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
if (MSVC)
# MSVC does not provide a /std:c++11 flag; C++11 is its implicit baseline.
# The lowest explicitly settable standard is /std:c++14. Without this flag,
# the default varies by VS version (VS2017+ defaults to C++14 mode with some
# C++17 extensions), so we pin it explicitly for reproducibility.
set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} /W3 /utf-8 /EHsc /bigobj /Zc:__cplusplus /std:c++14")
else ()
set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -std=c++11")
endif ()
set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -std=c++11")
add_subdirectory(third_party)
set(CMAKE_CXX_FLAGS "${SAVED_CXX_FLAGS}")

Expand All @@ -253,5 +186,12 @@ else()
message("BUILD_TEST is OFF, skipping test directory")
endif ()

add_subdirectory(examples)
option(BUILD_EXAMPLES "Build examples (requires Arrow/Parquet)" OFF)
if (BUILD_EXAMPLES)
add_subdirectory(examples)
endif()

if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/experiment/CMakeLists.txt")
add_subdirectory(experiment)
endif()

36 changes: 16 additions & 20 deletions cpp/examples/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,42 +10,38 @@ with the License. You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
software distributed under the LICENSE is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
]]
cmake_minimum_required(VERSION 3.10)
project(examples)
message("Running in exampes directory")
message("Running in examples directory")

if (NOT MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
endif ()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")

# TsFile include dir
# TsFile include dirs
set(SDK_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/../src/)
message("SDK_INCLUDE_DIR: ${SDK_INCLUDE_DIR}")

# TsFile shared object dir
set(SDK_LIB_DIR_RELEASE ${PROJECT_SOURCE_DIR}/../build/Release/lib)
message("SDK_LIB_DIR_RELEASE: ${SDK_LIB_DIR_RELEASE}")

set(SDK_LIB_DIR_DEBUG ${PROJECT_SOURCE_DIR}/../build/Debug/lib)
message("SDK_LIB_DIR_DEBUG: ${SDK_LIB_DIR_DEBUG}")
include_directories(${SDK_INCLUDE_DIR})
include_directories(${PROJECT_SOURCE_DIR}/../third_party/antlr4-cpp-runtime-4/runtime/src)

set(BUILD_TYPE "Release")
include_directories(${SDK_INCLUDE_DIR})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -DNDEBUG")

if (NOT MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g")
endif ()
# Arrow + Parquet are required (for bench_read)
if(APPLE)
list(APPEND CMAKE_PREFIX_PATH
"/opt/homebrew/opt/apache-arrow/lib/cmake"
"/usr/local/opt/apache-arrow/lib/cmake")
endif()
find_package(Arrow CONFIG REQUIRED)
find_package(Parquet CONFIG REQUIRED)

add_subdirectory(cpp_examples)
add_subdirectory(c_examples)

add_executable(examples examples.cc)
target_link_libraries(examples cpp_examples_obj c_examples_obj)
target_link_libraries(examples tsfile)
find_package(Threads REQUIRED)
target_link_libraries(examples tsfile Arrow::arrow_shared Parquet::parquet_shared Threads::Threads)
16 changes: 14 additions & 2 deletions cpp/examples/cpp_examples/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,17 @@ under the License.
]]

message("Running in examples/cpp_examples directory")
aux_source_directory(. cpp_SRC_LIST)
add_library(cpp_examples_obj OBJECT ${cpp_SRC_LIST})

add_library(cpp_examples_obj OBJECT
demo_read.cpp
demo_write.cpp
bench_read.cpp)

# bench_read.cpp requires C++17 (TsFile headers use [[maybe_unused]])
# and Arrow/Parquet headers. Both are provided by the parent scope.
set_target_properties(cpp_examples_obj PROPERTIES
CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON)
target_compile_options(cpp_examples_obj PRIVATE -std=c++17)
target_link_libraries(cpp_examples_obj PRIVATE
Arrow::arrow_shared
Parquet::parquet_shared)
Loading
Loading