From d68503fa8c4e68b6c5155eb77ad487cf598818f0 Mon Sep 17 00:00:00 2001 From: Awawa <69086569+awawa-dev@users.noreply.github.com> Date: Wed, 22 Jan 2025 20:01:36 +0100 Subject: [PATCH] Replace XZ with ZSTD --- .gitmodules | 6 +- CMakeLists.txt | 48 +++----- HyperhdrConfig.h.in | 4 +- cmake/cmake_modules/Findzstd.cmake | 29 +++++ cmake/installers.cmake | 20 ++-- external/CMakeLists.txt | 23 ++-- external/xz | 1 - external/zstd | 1 + include/utils-xz/utils-xz.h | 10 -- include/utils-zstd/utils-zstd.h | 10 ++ sources/CMakeLists.txt | 4 +- sources/api/BaseAPI.cpp | 10 +- sources/api/HyperAPI.cpp | 2 +- sources/hyperhdr/CMakeLists.txt | 8 +- sources/utils-xz/CMakeLists.txt | 34 ------ sources/utils-xz/utils-xz.cpp | 182 ----------------------------- sources/utils-zstd/CMakeLists.txt | 33 ++++++ sources/utils-zstd/utils-zstd.cpp | 103 ++++++++++++++++ sources/utils/LutLoader.cpp | 14 +-- 19 files changed, 236 insertions(+), 306 deletions(-) create mode 100644 cmake/cmake_modules/Findzstd.cmake delete mode 160000 external/xz create mode 160000 external/zstd delete mode 100644 include/utils-xz/utils-xz.h create mode 100644 include/utils-zstd/utils-zstd.h delete mode 100644 sources/utils-xz/CMakeLists.txt delete mode 100644 sources/utils-xz/utils-xz.cpp create mode 100644 sources/utils-zstd/CMakeLists.txt create mode 100644 sources/utils-zstd/utils-zstd.cpp diff --git a/.gitmodules b/.gitmodules index 712f09c06..d30b0c5d6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,9 +13,6 @@ path = external/qmqtt url = https://github.com/emqx/qmqtt.git ignore = dirty -[submodule "external/xz"] - path = external/xz - url = https://github.com/tukaani-project/xz.git [submodule "external/mdns"] path = external/mdns url = https://github.com/mjansson/mdns.git @@ -34,3 +31,6 @@ [submodule "external/linalg"] path = external/linalg url = https://github.com/sgorsten/linalg.git +[submodule "external/zstd"] + path = external/zstd + url = https://github.com/facebook/zstd.git diff --git a/CMakeLists.txt b/CMakeLists.txt index c4304e058..7990093a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,7 @@ SET ( DEFAULT_BONJOUR ON ) SET ( DEFAULT_MQTT ON ) SET ( DEFAULT_STATIC_QT_PLUGINS OFF ) SET ( DEFAULT_PRECOMPILED_HEADERS ON ) -SET ( DEFAULT_XZ ON ) +SET ( DEFAULT_ZSTD ON ) SET ( DEFAULT_POWER_MANAGEMENT ON ) SET ( DEFAULT_SYSTRAY ON ) SET ( DEFAULT_SHARED_LIBS ON ) @@ -382,26 +382,6 @@ if(DEFAULT_CEC) endif() endif() -# 7-zip as support -if (WIN32) - # 7zip - cmake_policy(SET CMP0053 NEW) - set(MYENV "PROGRAMFILES(X86)") - - find_program(SEVENZIP_BIN - NAMES 7z 7za - HINTS "${MINGWDIR}" "${MINGWLIBS}/bin" "$MYENV/7-zip" "$ENV{ProgramFiles}/7-zip" "$ENV{ProgramW6432}/7-zip" - PATH_SUFFIXES bin - DOC "7zip executable" - ) - - if (SEVENZIP_BIN) - message( STATUS "7-zip found: ${SEVENZIP_BIN}") - else() - message( FATAL_ERROR "Please install 7-zip") - endif (SEVENZIP_BIN) -endif() - if (CMAKE_CROSSCOMPILING) set ( DEFAULT_USE_SYSTEM_FLATBUFFERS_LIBS OFF ) SET ( DEFAULT_SHARED_LIBS OFF ) @@ -504,8 +484,8 @@ colorMe("ENABLE_PROTOBUF = " ${ENABLE_PROTOBUF}) option(ENABLE_SYSTRAY "Enable SYSTRAY" ${DEFAULT_SYSTRAY}) colorMe("ENABLE_SYSTRAY = " ${ENABLE_SYSTRAY}) -option(ENABLE_XZ "Enable XZ support" ${DEFAULT_XZ}) -colorMe("ENABLE_XZ = " ${ENABLE_XZ}) +option(ENABLE_ZSTD "Enable ZSTD support" ${DEFAULT_ZSTD}) +colorMe("ENABLE_ZSTD = " ${ENABLE_ZSTD}) message( STATUS "\n${CyanColor}BUILD FEATURES${ColorReset}") @@ -547,8 +527,8 @@ SET ( FLATBUFFERS_INSTALL_LIB_DIR ${CMAKE_BINARY_DIR}/flatbuf ) find_package(GitVersion) -if(ENABLE_XZ) - find_package(LibLZMA) +if(ENABLE_ZSTD) + find_package(zstd) endif() # configure a header file to pass some of the CMake settings @@ -694,13 +674,6 @@ if (WIN32) endif() endif() -# Add the source/lib directories -add_subdirectory(external) -add_subdirectory(sources) - -# Add resources directory -add_subdirectory(resources) - # remove generated files on make cleaan too LIST( APPEND GENERATED_QRC ${CMAKE_BINARY_DIR}/WebConfig.qrc @@ -712,12 +685,19 @@ set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${GENERATED_QRC configure_file( "${OUR_CMAKE_MODULES}/cmake_uninstall.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY) add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) +# Add the source/lib directories +add_subdirectory(external) +add_subdirectory(sources) + +# Add resources directory +add_subdirectory(resources) + # enable make package - no code after this line ! include (${CMAKE_CURRENT_SOURCE_DIR}/cmake/packages.cmake) # external targets -if (WIN32 AND TARGET stb AND TARGET flatbuffers AND TARGET protobuf-nanopb AND TARGET lunasvg AND TARGET flatc AND TARGET qmqtt AND TARGET liblzma AND TARGET sqlite3 AND TARGET precompiled_hyperhdr_headers) - set_target_properties(stb qmqtt flatbuffers protobuf-nanopb lunasvg flatc resources uninstall liblzma sqlite3 precompiled_hyperhdr_headers PROPERTIES FOLDER ExternalLibsTargets) +if (WIN32 AND TARGET stb AND TARGET flatbuffers AND TARGET protobuf-nanopb AND TARGET lunasvg AND TARGET flatc AND TARGET qmqtt AND TARGET libzstd_static AND TARGET libzstd_shared AND TARGET zstd AND TARGET clean-all AND TARGET sqlite3 AND TARGET precompiled_hyperhdr_headers) + set_target_properties(stb qmqtt flatbuffers protobuf-nanopb lunasvg flatc resources uninstall libzstd_static libzstd_shared zstd clean-all sqlite3 precompiled_hyperhdr_headers PROPERTIES FOLDER ExternalLibsTargets) else() set_target_properties(resources uninstall PROPERTIES FOLDER ExternalLibsTargets) endif() diff --git a/HyperhdrConfig.h.in b/HyperhdrConfig.h.in index fcc8b2565..1cfa4f2e5 100644 --- a/HyperhdrConfig.h.in +++ b/HyperhdrConfig.h.in @@ -66,8 +66,8 @@ // Define to enable system qmqtt #cmakedefine USE_SYSTEM_MQTT_LIBS -// Define to enable protobuf -#cmakedefine ENABLE_XZ +// Define to enable zstd +#cmakedefine ENABLE_ZSTD // Define to enable protobuf #cmakedefine ENABLE_PROTOBUF diff --git a/cmake/cmake_modules/Findzstd.cmake b/cmake/cmake_modules/Findzstd.cmake new file mode 100644 index 000000000..44547365d --- /dev/null +++ b/cmake/cmake_modules/Findzstd.cmake @@ -0,0 +1,29 @@ +# - Find zstd +# Find the zstd compression library and includes +# +# ZSTD_INCLUDE_DIRS - where to find zstd.h, etc. +# ZSTD_LIBRARIES - List of libraries when using zstd. +# ZSTD_FOUND - True if zstd found. + +find_path(ZSTD_INCLUDE_DIRS + NAMES zstd.h + HINTS ${zstd_ROOT_DIR}/include) + +find_library(ZSTD_LIBRARIES + NAMES zstd + HINTS ${zstd_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIRS) + +mark_as_advanced( + ZSTD_LIBRARIES + ZSTD_INCLUDE_DIRS) + +if(ZSTD_FOUND AND NOT (TARGET zstd::zstd)) + add_library (zstd::zstd UNKNOWN IMPORTED) + set_target_properties(zstd::zstd + PROPERTIES + IMPORTED_LOCATION ${ZSTD_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${ZSTD_INCLUDE_DIRS}) +endif() \ No newline at end of file diff --git a/cmake/installers.cmake b/cmake/installers.cmake index 90b0b2dae..378fda4dc 100644 --- a/cmake/installers.cmake +++ b/cmake/installers.cmake @@ -20,9 +20,9 @@ macro(DeployApple TARGET) install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/hyperhdr.app/Contents/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() - # Copy utils-xz - if (USE_SHARED_LIBS AND TARGET utils-xz) - install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/hyperhdr.app/Contents/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") + # Copy utils-zstd + if (USE_SHARED_LIBS AND TARGET utils-zstd) + install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/hyperhdr.app/Contents/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() # Copy utils-image @@ -243,9 +243,9 @@ macro(DeployUnix TARGET) install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/share/hyperhdr/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() - # Copy UTILS-XZ lib - if (TARGET utils-image) - install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/share/hyperhdr/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") + # Copy UTILS-ZSTD lib + if (TARGET utils-zstd) + install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/share/hyperhdr/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() @@ -354,7 +354,7 @@ macro(DeployUnix TARGET) "librt" "libstdc++" "libudev" - "libz" + "libz.so" "libxrender1" "libxi6" "libxext6" @@ -593,9 +593,9 @@ macro(DeployWindows TARGET) install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() - # Copy utils-xz - if (USE_SHARED_LIBS AND TARGET utils-xz) - install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") + # Copy utils-zstd + if (USE_SHARED_LIBS AND TARGET utils-zstd) + install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() # Copy utils-image diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 23fce071d..b4d23ad06 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -237,17 +237,22 @@ if ( ENABLE_MQTT ) endif() #============================================================================= -# XZ +# ZSTD #============================================================================= -if ( ENABLE_XZ ) - if (NOT LIBLZMA_FOUND) - message( STATUS "System library xz could not be found. Using embedded xz library.") - option(BUILD_TESTING "" OFF) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/xz) - set_target_properties(liblzma PROPERTIES INTERFACE_INCLUDE_DIRECTORIES $) - set_property(TARGET liblzma PROPERTY POSITION_INDEPENDENT_CODE ON) - add_library(LibLZMA::LibLZMA ALIAS liblzma) +if ( ENABLE_ZSTD ) + if (NOT zstd_FOUND) + message( STATUS "System library zstd could not be found. Using embedded zstd library.") + if(WIN32) + set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -I${CMAKE_SOURCE_DIR}/external/zstd/lib") + endif() + set(ZSTD_BUILD_TESTS OFF) + set(ZSTD_LEGACY_SUPPORT OFF) + set(ZSTD_BUILD_PROGRAMS OFF) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/zstd/build/cmake) + set_target_properties(libzstd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES $) + set_property(TARGET libzstd PROPERTY POSITION_INDEPENDENT_CODE ON) + add_library(zstd::zstd ALIAS libzstd) endif() ENDIF() diff --git a/external/xz b/external/xz deleted file mode 160000 index 238b4e545..000000000 --- a/external/xz +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 238b4e5458b4bd2cadefb768b8ea7c6b70a191ac diff --git a/external/zstd b/external/zstd new file mode 160000 index 000000000..794ea1b0a --- /dev/null +++ b/external/zstd @@ -0,0 +1 @@ +Subproject commit 794ea1b0afca0f020f4e57b6732332231fb23c70 diff --git a/include/utils-xz/utils-xz.h b/include/utils-xz/utils-xz.h deleted file mode 100644 index f8a6f65af..000000000 --- a/include/utils-xz/utils-xz.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -#ifndef _XZ_SHARED_API - #define _XZ_SHARED_API -#endif - -#include - -_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName); -_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize); diff --git a/include/utils-zstd/utils-zstd.h b/include/utils-zstd/utils-zstd.h new file mode 100644 index 000000000..c23c31a9b --- /dev/null +++ b/include/utils-zstd/utils-zstd.h @@ -0,0 +1,10 @@ +#pragma once + +#ifndef _ZSTD_SHARED_API + #define _ZSTD_SHARED_API +#endif + +#include + +_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName); +_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize); diff --git a/sources/CMakeLists.txt b/sources/CMakeLists.txt index 3ca0cf597..157722a67 100644 --- a/sources/CMakeLists.txt +++ b/sources/CMakeLists.txt @@ -33,8 +33,8 @@ add_subdirectory(utils) add_subdirectory(utils-image) add_subdirectory(webserver) -if(ENABLE_XZ) - add_subdirectory(utils-xz) +if(ENABLE_ZSTD) + add_subdirectory(utils-zstd) endif() diff --git a/sources/api/BaseAPI.cpp b/sources/api/BaseAPI.cpp index 11fd1e5d0..62d582c34 100644 --- a/sources/api/BaseAPI.cpp +++ b/sources/api/BaseAPI.cpp @@ -31,8 +31,8 @@ #include #include -#ifdef ENABLE_XZ - #include +#ifdef ENABLE_ZSTD + #include #endif #ifdef _WIN32 @@ -410,21 +410,21 @@ bool BaseAPI::saveSettings(const QJsonObject& data) QString BaseAPI::installLut(QNetworkReply* reply, QString fileName, int hardware_brightness, int hardware_contrast, int hardware_saturation, qint64 time) { -#ifdef ENABLE_XZ +#ifdef ENABLE_ZSTD QString error = nullptr; if (reply->error() == QNetworkReply::NetworkError::NoError) { QByteArray downloadedData = reply->readAll(); - error = DecompressXZ(downloadedData.size(), reinterpret_cast(downloadedData.data()), QSTRING_CSTR(fileName)); + error = DecompressZSTD(downloadedData.size(), reinterpret_cast(downloadedData.data()), QSTRING_CSTR(fileName)); } else error = "Could not download LUT file"; return error; #else - return "XZ support was disabled in the build configuration"; + return "ZSTD support was disabled in the build configuration"; #endif } diff --git a/sources/api/HyperAPI.cpp b/sources/api/HyperAPI.cpp index b27dd3a69..222a0272f 100644 --- a/sources/api/HyperAPI.cpp +++ b/sources/api/HyperAPI.cpp @@ -568,7 +568,7 @@ void HyperAPI::lutDownloaded(QNetworkReply* reply, int hardware_brightness, int void HyperAPI::handleLutInstallCommand(const QJsonObject& message, const QString& command, int tan) { - const QString& address = QString("%1/lut_lin_tables.3d.xz").arg(message["subcommand"].toString().trimmed()); + const QString& address = QString("%1/lut_lin_tables.3d.zst").arg(message["subcommand"].toString().trimmed()); int hardware_brightness = message["hardware_brightness"].toInt(0); int hardware_contrast = message["hardware_contrast"].toInt(0); int hardware_saturation = message["hardware_saturation"].toInt(0); diff --git a/sources/hyperhdr/CMakeLists.txt b/sources/hyperhdr/CMakeLists.txt index 500c562a5..2e6c2f2eb 100644 --- a/sources/hyperhdr/CMakeLists.txt +++ b/sources/hyperhdr/CMakeLists.txt @@ -54,12 +54,8 @@ else() PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) endif() -if(ENABLE_XZ) - if (LIBLZMA_FOUND) - target_link_libraries(hyperhdr LibLZMA::LibLZMA utils-xz) - else() - target_link_libraries(hyperhdr liblzma utils-xz) - endif() +if(ENABLE_ZSTD) + target_link_libraries(hyperhdr zstd::zstd utils-zstd) endif() if (USE_STATIC_QT_PLUGINS) diff --git a/sources/utils-xz/CMakeLists.txt b/sources/utils-xz/CMakeLists.txt deleted file mode 100644 index 6c2ac048b..000000000 --- a/sources/utils-xz/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -# Define the current source locations - -SET(CURRENT_HEADER_DIR ${CMAKE_SOURCE_DIR}/include/utils-xz) -SET(CURRENT_SOURCE_DIR ${CMAKE_SOURCE_DIR}/sources/utils-xz) - -FILE ( GLOB_RECURSE utils_xz_SOURCES "${CURRENT_HEADER_DIR}/*.h" "${CURRENT_SOURCE_DIR}/*.cpp" ) - -if (USE_SHARED_LIBS) - add_library(utils-xz SHARED ${utils_xz_SOURCES}) - if(WIN32) - target_compile_definitions(utils-xz - INTERFACE - "_XZ_SHARED_API=__declspec(dllimport)" - PRIVATE - "_XZ_SHARED_API=__declspec(dllexport)" - ) - else() - target_compile_definitions(utils-xz - INTERFACE - "_XZ_SHARED_API=__attribute__((visibility(\"default\")))" - ) - endif() - install( - TARGETS utils-xz - LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" - ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" - ) -else() - add_library(utils-xz STATIC ${utils_xz_SOURCES}) -endif() - -target_link_libraries(utils-xz PRIVATE LibLZMA::LibLZMA) -target_include_directories(utils-xz PRIVATE LibLZMA::LibLZMA) - diff --git a/sources/utils-xz/utils-xz.cpp b/sources/utils-xz/utils-xz.cpp deleted file mode 100644 index 508bfca02..000000000 --- a/sources/utils-xz/utils-xz.cpp +++ /dev/null @@ -1,182 +0,0 @@ -#include -#include -#include -#include -#include -#include - -_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName) -{ - size_t outSize = 67174456; - std::vector outBuffer; - const char* error = nullptr; - - std::ofstream file; - file.open(fileName, std::ios::out | std::ios::trunc | std::ios::binary); - - if (!file.is_open()) - { - return "Could not open file for writing"; - } - - try - { - outBuffer.resize(outSize); - } - catch(...) - { - error = "Could not allocate buffer"; - } - - if (error == nullptr) - { - const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED; - lzma_stream strm = LZMA_STREAM_INIT; - strm.next_in = downloadedData; - strm.avail_in = downloadedDataSize; - lzma_ret lzmaRet = lzma_stream_decoder(&strm, outSize, flags); - if (lzmaRet == LZMA_OK) - { - do { - strm.next_out = outBuffer.data(); - strm.avail_out = outSize; - lzmaRet = lzma_code(&strm, LZMA_FINISH); - if (lzmaRet == LZMA_MEMLIMIT_ERROR) - { - outSize = lzma_memusage(&strm); - try - { - outBuffer.resize(outSize); - } - catch (...) - { - error = "Could not increase buffer size"; - break; - } - lzma_memlimit_set(&strm, outSize); - strm.avail_out = 0; - } - else if (lzmaRet != LZMA_OK && lzmaRet != LZMA_STREAM_END) - { - error = "LZMA decoder returned error"; - break; - } - else - { - std::streamsize toWrite = static_cast(outSize - strm.avail_out); - file.write(reinterpret_cast(outBuffer.data()), toWrite); - } - } while (strm.avail_out == 0 && lzmaRet != LZMA_STREAM_END); - file.flush(); - } - else - { - error = "Could not initialize LZMA decoder"; - } - - lzma_end(&strm); - } - - file.close(); - - if (error != nullptr) - std::remove(fileName); - - return error; -} - -_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize) -{ - size_t outSize = 16842808/2; - std::vector outBuffer; - const char* error = nullptr; - int currentPos = 0; - int writePos = 0; - - try - { - outBuffer.resize(outSize); - } - catch (...) - { - error = "Could not allocate buffer"; - } - - if (error == nullptr) - { - const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED; - lzma_stream strm = LZMA_STREAM_INIT; - strm.next_in = downloadedData; - strm.avail_in = downloadedDataSize; - lzma_ret lzmaRet = lzma_stream_decoder(&strm, outSize, flags); - if (lzmaRet == LZMA_OK) - { - do { - strm.next_out = outBuffer.data(); - strm.avail_out = outSize; - lzmaRet = lzma_code(&strm, LZMA_FINISH); - if (lzmaRet == LZMA_MEMLIMIT_ERROR) - { - outSize = lzma_memusage(&strm); - try - { - outBuffer.resize(outSize); - } - catch (...) - { - error = "Could not increase buffer size"; - break; - } - lzma_memlimit_set(&strm, outSize); - strm.avail_out = 0; - } - else if (lzmaRet != LZMA_OK && lzmaRet != LZMA_STREAM_END) - { - error = "LZMA decoder returned error"; - break; - } - else - { - int toWrite = outSize - strm.avail_out; - int endPos = currentPos + toWrite; - int chunkSize = -1, chunkPos = 0; - - if (currentPos <= destSeek && destSeek + destSize >= endPos) - { - chunkPos = destSeek - currentPos; - chunkSize = toWrite - chunkPos; - } - else if (currentPos <= destSeek + destSize && destSeek + destSize < endPos) - { - chunkSize = (destSeek + destSize) - currentPos; - } - else if (currentPos >= destSeek && destSeek + destSize >= endPos) - { - chunkSize = toWrite; - } - - if (chunkSize > 0) - { - memcpy(dest + writePos, outBuffer.data() + chunkPos, chunkSize); - writePos += chunkSize; - } - - currentPos = endPos; - } - } while (strm.avail_out == 0 && lzmaRet != LZMA_STREAM_END && (writePos < destSize)); - } - else - { - error = "Could not initialize LZMA decoder"; - } - - lzma_end(&strm); - } - - if (writePos != destSize) - { - error = "Incorrect final LUT size"; - } - - return error; -} diff --git a/sources/utils-zstd/CMakeLists.txt b/sources/utils-zstd/CMakeLists.txt new file mode 100644 index 000000000..c7fa8b096 --- /dev/null +++ b/sources/utils-zstd/CMakeLists.txt @@ -0,0 +1,33 @@ +# Define the current source locations + +SET(CURRENT_HEADER_DIR ${CMAKE_SOURCE_DIR}/include/utils-zstd) +SET(CURRENT_SOURCE_DIR ${CMAKE_SOURCE_DIR}/sources/utils-zstd) + +FILE ( GLOB_RECURSE utils_zstd_SOURCES "${CURRENT_HEADER_DIR}/*.h" "${CURRENT_SOURCE_DIR}/*.cpp" ) + +if (USE_SHARED_LIBS) + add_library(utils-zstd SHARED ${utils_zstd_SOURCES}) + if(WIN32) + target_compile_definitions(utils-zstd + INTERFACE + "_ZSTD_SHARED_API=__declspec(dllimport)" + PRIVATE + "_ZSTD_SHARED_API=__declspec(dllexport)" + ) + else() + target_compile_definitions(utils-zstd + INTERFACE + "_ZSTD_SHARED_API=__attribute__((visibility(\"default\")))" + ) + endif() + install( + TARGETS utils-zstd + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" + ) +else() + add_library(utils-zstd STATIC ${utils_zstd_SOURCES}) +endif() + +target_link_libraries(utils-zstd PRIVATE zstd::zstd) +target_include_directories(utils-zstd PRIVATE zstd::zstd) \ No newline at end of file diff --git a/sources/utils-zstd/utils-zstd.cpp b/sources/utils-zstd/utils-zstd.cpp new file mode 100644 index 000000000..b5c0068d8 --- /dev/null +++ b/sources/utils-zstd/utils-zstd.cpp @@ -0,0 +1,103 @@ +#include +#include +#include +#include +#include +#include +#include + +_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName) +{ + const size_t SINGLE_LUT_SIZE = (static_cast(256 * 256 * 256) * 3); + + size_t outSize = SINGLE_LUT_SIZE / 2; + std::vector outBuffer; + const char* error = nullptr; + + std::ofstream file; + file.open(fileName, std::ios::out | std::ios::trunc | std::ios::binary); + + if (!file.is_open()) + { + return "Could not open file for writing"; + } + + try + { + outBuffer.resize(outSize); + } + catch(...) + { + error = "Could not allocate buffer"; + } + + + if (error == nullptr) + { + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + + if (dctx == NULL) + { + error = "ZSTD_createDCtx() failed!"; + } + else + { + int total = SINGLE_LUT_SIZE * 3; + ZSTD_inBuffer input = { downloadedData, downloadedDataSize, 0 }; + while (input.pos < input.size) + { + ZSTD_outBuffer outputSeek = { outBuffer.data(), outBuffer.size(), 0 }; + size_t const ret = ZSTD_decompressStream(dctx, &outputSeek, &input); + if (ZSTD_isError(ret) || outputSeek.pos != outBuffer.size() || total < 0) + { + error = "Error during decompression"; + break; + } + else + { + file.write(reinterpret_cast(outBuffer.data()), outBuffer.size()); + } + total -= outBuffer.size(); + } + ZSTD_freeDCtx(dctx); + } + } + + file.close(); + + if (error != nullptr) + std::remove(fileName); + + return error; +} + +_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize) +{ + const char* error = nullptr; + + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + + if (dctx == NULL) + { + error = "ZSTD_createDCtx() failed!"; + } + else + { + int totalOutput = 0; + ZSTD_inBuffer input = { downloadedData, downloadedDataSize, 0 }; + while (totalOutput <= destSeek && input.pos < input.size) + { + ZSTD_outBuffer outputSeek = { dest, static_cast(destSize), static_cast(0) }; + size_t const ret = ZSTD_decompressStream(dctx, &outputSeek, &input); + if (ZSTD_isError(ret) || outputSeek.pos != destSize) + { + error = "Error during decompression"; + break; + } + totalOutput += outputSeek.pos; + } + ZSTD_freeDCtx(dctx); + } + + return error; +} diff --git a/sources/utils/LutLoader.cpp b/sources/utils/LutLoader.cpp index 356cac7c1..55037f2b2 100644 --- a/sources/utils/LutLoader.cpp +++ b/sources/utils/LutLoader.cpp @@ -31,8 +31,8 @@ #include #include -#ifdef ENABLE_XZ - #include +#ifdef ENABLE_ZSTD + #include #endif @@ -61,8 +61,8 @@ void LutLoader::loadLutFile(Logger* _log, PixelFormat color, const QList(compressedFile.data()), _lut.data(), index, LUT_FILE_SIZE); + #ifdef ENABLE_ZSTD + retVal = DecompressZSTD(compressedFile.size(), reinterpret_cast(compressedFile.data()), _lut.data(), index, LUT_FILE_SIZE); #endif if (retVal != nullptr && _log)