Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(dipu): partially clean cmake #883

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

refactor(dipu): partially clean cmake #883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b3f6536
refactor: clean cmake and move torch version from cmake to file
wiryls Jul 8, 2024
b282aa6
fix: manually set WORKING_DIRECTORY
wiryls Jul 9, 2024
adc409f
debug: CMakeLists.txt
wiryls Jul 9, 2024
d122629
fix: rollback version-detection
wiryls Jul 16, 2024
56597fc
Merge remote-tracking branch 'dev/main' into refactor/w/better_cmake
wiryls Jul 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 28 additions & 54 deletions dipu/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,6 @@
cmake_minimum_required(VERSION 3.20)
project(TorchDIPU LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)

set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

# the default CMAKE_BUILD_TYPE is Release
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release")
endif()

option(ENABLE_LTO "Enable Link Time Optimization in Release build" ON)
if(ENABLE_LTO AND CMAKE_BUILD_TYPE STREQUAL "Release")
Expand All @@ -29,13 +19,17 @@ else()
message(STATUS "IPO / LTO disabled")
endif()

option(TESTS "Whether to build unit tests" OFF)
option(LIBS "Whether to build dipu lib, default on" ON)

# use gcover
option(DPIU_ENABLE_TEST "Whether to build unit tests" OFF)
option(DPIU_ENABLE_LIB "Whether to build dipu lib, default on" ON)
option(ENABLE_COVERAGE "Use gcov" OFF)

message(STATUS ENABLE_COVERAGE=${ENABLE_COVERAGE})

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)


# device related
set(DEVICE
"camb"
Expand Down Expand Up @@ -102,7 +96,7 @@ add_compile_definitions(${DIPU_VENDOR_NAME_FLAG_DEF}=1)
message(STATUS "Vendor name flag definition: " ${DIPU_VENDOR_NAME_FLAG_DEF})

string(REGEX MATCH "[a-z0-9_]+" DIPU_GIT_HASH ${DIPU_GIT_HASH})
execute_process(COMMAND sh -c "git status --porcelain | egrep '^(M| M)' | wc -l"
execute_process(COMMAND "git status --porcelain | egrep '^(M| M)' | wc -l"
OUTPUT_VARIABLE DIPU_MODIFY_LEN)
if(DIPU_MODIFY_LEN GREATER 0)
set(DIPU_GIT_HASH ${DIPU_GIT_HASH}-dirty)
Expand All @@ -112,16 +106,13 @@ add_compile_definitions(DIPU_GIT_HASH="${DIPU_GIT_HASH}")

# Automatically generate a list of supported diopi functions
execute_process(
COMMAND
sh -c
"grep -Po 'diopi[a-zA-Z0-9]+(?=\\()' ${CMAKE_CURRENT_SOURCE_DIR}/scripts/autogen_diopi_wrapper/diopi_functions.yaml | sort -uf > ${CMAKE_CURRENT_SOURCE_DIR}/SupportedDiopiFunctions.txt"
)
COMMAND "grep -Po 'diopi[a-zA-Z0-9]+(?=\\()' ./scripts/autogen_diopi_wrapper/diopi_functions.yaml | sort -uf > ./SupportedDiopiFunctions.txt"
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")

# abi_v, torch dir, abi flag, cmake path
execute_process(
COMMAND
sh -c
"python ${CMAKE_CURRENT_SOURCE_DIR}/detect_env.py"
COMMAND python detect_env.py
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
OUTPUT_VARIABLE CHECK_TORCH_OUT
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REGEX REPLACE "\n" ";" CHECK_TORCH_OUT_LIST "${CHECK_TORCH_OUT}")
Expand Down Expand Up @@ -157,16 +148,10 @@ endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=${DIPU_COMPILED_WITH_CXX11_ABI}")
message(STATUS "DIPU_COMPILED_WITH_CXX11_ABI:" ${DIPU_COMPILED_WITH_CXX11_ABI})

execute_process(COMMAND sh -c "dirname $(find $(dirname $(which python))/../ -name Python.h)"
OUTPUT_VARIABLE PYTHON_INCLUDE_DIR)
message(STATUS "PYTHON_INCLUDE_DIR: " ${PYTHON_INCLUDE_DIR})

# config
# Import BaseFuncions
include(cmake/BaseFuncions.cmake)
_set_cpp_flags()

# if add new version, please also update dipu/__init__.py torch_ver_XX
list(APPEND DIPU_SUPPORT_TORCHS "2.0.0" "2.1.0" "2.1.1")
# TODO: fix it.
# _set_cpp_flags()

# use muxi torch. if device is muxi and torch has 'cuda'(muxi).
if (${DEVICE} IN_LIST DEVICE_MUXI AND DIPU_TORCH_HAS_CUDA)
Expand All @@ -175,31 +160,20 @@ if (${DEVICE} IN_LIST DEVICE_MUXI AND DIPU_TORCH_HAS_CUDA)
add_compile_definitions(USE_MACA=1)
endif()

# if add new version, please also update dipu/__init__.py torch_ver_XX
set(DIPU_SUPPORT_TORCHS "2.0.0;2.1.0;2.1.1")

find_package(Torch REQUIRED)

message(STATUS "Found Torch Version: ${Torch_VERSION}")
if(NOT ${Torch_VERSION} IN_LIST DIPU_SUPPORT_TORCHS)
message(FATAL_ERROR "this torch version is not supported by DIPU ${DIPU_SUPPORT_TORCHS}")
if(NOT Torch_VERSION IN_LIST DIPU_SUPPORT_TORCHS)
message(FATAL_ERROR "Torch ${Torch_VERSION} is not in DIPU supported version ${DIPU_SUPPORT_TORCHS}")
else()
string(REGEX REPLACE "^ERR[0-9]*([0-9][0-9])\$" "\\1" Torch_VERSION_MINOR "ERR00${Torch_VERSION_MINOR}")
string(REGEX REPLACE "^ERR[0-9]*([0-9][0-9])\$" "\\1" Torch_VERSION_PATCH "ERR00${Torch_VERSION_PATCH}")
set(DIPU_TORCH_VERSION "${Torch_VERSION_MAJOR}${Torch_VERSION_MINOR}${Torch_VERSION_PATCH}")
message(STATUS "DIPU_TORCH_VERSION: ${DIPU_TORCH_VERSION}")
endif()

# message(STATUS "Torch TORCH_LIBRARY: ${TORCH_LIBRARY}")
include_directories(SYSTEM ${TORCH_INCLUDE_DIRS})
message(STATUS "Torch TORCH_INCLUDE_DIRS: ${TORCH_INCLUDE_DIRS}")
message(STATUS "Torch TORCH_CXX_FLAGS: ${TORCH_CXX_FLAGS}")

string(REGEX REPLACE "^.*(..)\$" "\\1" Torch_VERSION_MINOR "0000${Torch_VERSION_MINOR}")
string(REGEX REPLACE "^.*(..)\$" "\\1" Torch_VERSION_PATCH "0000${Torch_VERSION_PATCH}")

string(CONCAT Torch_VERSION ${Torch_VERSION_MAJOR}${Torch_VERSION_MINOR}${Torch_VERSION_PATCH})
# expand to pattern XYYZZ by leading zero on minor and patch ver
message(STATUS "change to dipu Torch Version: ${Torch_VERSION}")

# add_compile_definitions(DIPU_TORCH${Torch_VERSION})
add_compile_definitions(DIPU_TORCH_VERSION=${Torch_VERSION})

link_directories(${PYTORCH_DIR}/torch/lib)
set(DIPU_TORCH_VERSION ${Torch_VERSION})

# end torch find

if(ENABLE_COVERAGE)
Expand All @@ -209,10 +183,10 @@ endif()

add_subdirectory(third_party)

if(LIBS)
if(DPIU_ENABLE_LIB)
add_subdirectory(torch_dipu/csrc_dipu)
endif()

if(TESTS)
if(DPIU_ENABLE_TEST)
add_subdirectory(tests/cpp)
endif()
13 changes: 3 additions & 10 deletions dipu/detect_env.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,10 @@

sys.path.append(os.getenv("PYTORCH_DIR", default=""))
import torch
import builtins
from pathlib import Path

print(
next(
item[-4:-2]
for item in dir(builtins)
if "__pybind11_internals_v4_gcc_libstdcpp_cxxabi10" in item
)
)
print(torch._C._PYBIND11_BUILD_ABI[-2:])
print(Path(torch.__path__[0]).parent.absolute())
print(1 if torch.compiled_with_cxx11_abi() else 0)
print(int(torch.compiled_with_cxx11_abi()))
print(torch.utils.cmake_prefix_path)
print(1 if torch.cuda.is_available() else 0)
print(int(torch.cuda.is_available()))
3 changes: 2 additions & 1 deletion dipu/third_party/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ message(STATUS "Using DIOPI_LIBRARY_PATH='${DIOPI_LIBRARY_PATH}', DIOPI_INCLUDE_
set(KINETO_BUILD_TESTS OFF CACHE INTERNAL "turn off tests")
set(KINETO_USE_DEVICE_ACTIVITY ON CACHE INTERNAL "enable device activity")
set(KINETO_COMPILED_WITH_CXX11_ABI "${DIPU_COMPILED_WITH_CXX11_ABI}" CACHE INTERNAL "pass through ABI settings")

# KINETO_COMPILED_WITH_CXX11_ABI might be removed from libkineto as we are
# using add_subdirectory instead of ExternalProject.
add_subdirectory(kineto/libkineto SYSTEM)
target_compile_definitions(kineto PRIVATE DIPU_TORCH_VERSION=${DIPU_TORCH_VERSION})
4 changes: 3 additions & 1 deletion dipu/torch_dipu/csrc_dipu/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ set(TORCH_DIPU_SOURCE

add_library(torch_dipu SHARED ${TORCH_DIPU_SOURCE} ${GENERATED_KERNELS} ${VENDOR_FILES} ${VENDOR_OUTPUT_HEADER})
target_include_directories(torch_dipu PUBLIC ..)
target_compile_definitions(torch_dipu PUBLIC DIPU_TORCH_VERSION=${DIPU_TORCH_VERSION})

# TODO(vendor) - replace those code with target_link_libraries.
target_include_directories(torch_dipu SYSTEM PUBLIC ${VENDOR_INCLUDE_DIRS} ${VENDOR_DIST_DIR})
Expand Down Expand Up @@ -166,12 +167,13 @@ Python3_add_library(torch_dipu_python SHARED ${TORCH_DIPU_PYBIND_SOURCE})
# cause diopirt hidden, so temporarily use this target level setting. enhance in
# future.
set_target_properties(torch_dipu_python PROPERTIES CXX_VISIBILITY_PRESET hidden)
target_link_libraries(torch_dipu_python PUBLIC torch_dipu PRIVATE kineto)
target_link_libraries(torch_dipu_python PUBLIC torch_dipu torch kineto)
target_include_directories(torch_dipu_python SYSTEM PRIVATE ..)

#[[ Target: torch_dipu_cpython_extension ]]
Python3_add_library(torch_dipu_cpython_extension SHARED stub.cpp)
target_include_directories(torch_dipu_cpython_extension SYSTEM PRIVATE ..)
target_link_directories(torch_dipu_cpython_extension PRIVATE "${PYTORCH_DIR}/torch/lib")
target_link_libraries(torch_dipu_cpython_extension PUBLIC torch_dipu_python PRIVATE torch_python)
target_compile_options(torch_dipu_cpython_extension PRIVATE -fstack-protector-all)
set_target_properties(torch_dipu_cpython_extension PROPERTIES
Expand Down
Loading