Conventional commit (#15)

* ffi: Add support for serializing/deserializing auto-generated and user-generated schema tree node IDs. (y-scope#557)

Co-authored-by: kirkrodrigues <[email protected]>

* clp: Add missing C++ standard library includes in IR parsing files. (y-scope#561)

Co-authored-by: kirkrodrigues <[email protected]>

* log-viewer-webui: Update `yscope-log-viewer` to the latest version (which uses `clp-ffi-js`). (y-scope#562)

* package: Upgrade dependencies to resolve security issues. (y-scope#536)

* clp-s: Implement table packing (y-scope#466)

Co-authored-by: wraymo <[email protected]>
Co-authored-by: Kirk Rodrigues <[email protected]>
Co-authored-by: wraymo <[email protected]>

* log-viewer-webui: Update `yscope-log-viewer` to the latest version. (y-scope#565)

* ci: Switch GitHub macOS build workflow to use macos-13 (x86) and macos-14 (ARM) runners. (y-scope#566)

* core: Add support for user-defined HTTP headers in `NetworkReader`. (y-scope#568)

Co-authored-by: Lin Zhihao <[email protected]>
Co-authored-by: Xiaochong Wei <[email protected]>

* chore: Update to the latest version of yscope-dev-utils. (y-scope#574)

* build(core): Upgrade msgpack to v7.0.0. (y-scope#575)

* feat(ffi): Update IR stream protocol version handling in preparation for releasing the kv-pair IR stream format: (y-scope#573)

- Bump the IR stream protocol version to 0.1.0 for the kv-pair IR stream format.
- Treat the previous IR stream format's versions as backwards compatible.
- Differentiate between backwards-compatible and supported versions during validation.

Co-authored-by: kirkrodrigues <[email protected]>

* fix(taskfiles): Trim trailing slash from URL prefix in `download-and-extract-tar` (fixes y-scope#577). (y-scope#578)

* fix(ffi): Correct `clp::ffi::ir_stream::Deserializer::deserialize_next_ir_unit`'s return value when failing to read the next IR unit's type tag. (y-scope#579)

* fix(taskfiles): Update `yscope-log-viewer` sources in `log-viewer-webui-clients` sources list (fixes y-scope#576). (y-scope#580)

* fix(cmake): Add Homebrew path detection for `mariadb-connector-c` to fix macOS build failure. (y-scope#582)

Co-authored-by: kirkrodrigues <[email protected]>

* refactor(ffi): Make `get_schema_subtree_bitmap` a public method of `KeyValuePairLogEvent`. (y-scope#581)

* ci: Schedule GitHub workflows to daily run to detect failures due to upgraded dependencies or environments. (y-scope#583)

* docs: Update the required version of task. (y-scope#567)

* Add pr check workflow

---------

Co-authored-by: kirkrodrigues <[email protected]>
Co-authored-by: Junhao Liao <[email protected]>
Co-authored-by: Henry8192 <[email protected]>
Co-authored-by: Devin Gibson <[email protected]>
Co-authored-by: wraymo <[email protected]>
Co-authored-by: wraymo <[email protected]>
Co-authored-by: Xiaochong(Eddy) Wei <[email protected]>
Co-authored-by: Xiaochong Wei <[email protected]>
Co-authored-by: haiqi96 <[email protected]>

Author: 9 people

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,9 @@
 <!--
-Set the PR title to a meaningful commit message in imperative form. E.g.:
-
-clp-s: Don't add implicit wildcards ('*') at the beginning and the end of a query (fixes #390).
+Set the PR title to a meaningful commit message that:
+- follows the Conventional Commits specification (https://www.conventionalcommits.org).
+- is in imperative form.
+Example:
+fix: Don't add implicit wildcards ('*') at the beginning and the end of a query (fixes #390).
 -->
 
 # Description

.github/workflows/clp-core-build-macos.yaml

@@ -8,7 +8,7 @@ on:
       - "components/core/CMakeLists.txt"
       - "components/core/src/**"
       - "components/core/tests/**"
-      - "components/core/tools/scripts/lib_install/macos-12/**"
+      - "components/core/tools/scripts/lib_install/macos/**"
       - "components/core/tools/scripts/deps-download/**"
       - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
       - "deps-tasks.yml"
@@ -21,12 +21,15 @@ on:
       - "components/core/CMakeLists.txt"
       - "components/core/src/**"
       - "components/core/tests/**"
-      - "components/core/tools/scripts/lib_install/macos-12/**"
+      - "components/core/tools/scripts/lib_install/macos/**"
       - "components/core/tools/scripts/deps-download/**"
       - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
       - "deps-tasks.yml"
       - "Taskfile.yml"
       - "tools/scripts/deps-download/**"
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
   workflow_dispatch:
 
 concurrency:
@@ -38,8 +41,9 @@ jobs:
   build-macos:
     strategy:
       matrix:
+        runner: ["macos-13", "macos-14"]
         use_shared_libs: [true, false]
-    runs-on: "macos-12"
+    runs-on: "${{matrix.runner}}"
     steps:
       - uses: "actions/checkout@v4"
         with:
@@ -55,7 +59,7 @@ jobs:
           rm -f /usr/local/bin/python3*
 
       - name: "Install dependencies"
-        run: "./components/core/tools/scripts/lib_install/macos-12/install-all.sh"
+        run: "./components/core/tools/scripts/lib_install/macos/install-all.sh"
 
       - run: "./tools/scripts/deps-download/init.sh"
         shell: "bash"

.github/workflows/clp-core-build.yaml

@@ -11,7 +11,7 @@ on:
       - "deps-tasks.yml"
       - "Taskfile.yml"
       - "tools/scripts/deps-download/**"
-      - "!components/core/tools/scripts/lib_install/macos-12/**"
+      - "!components/core/tools/scripts/lib_install/macos/**"
   push:
     paths:
       - ".github/actions/clp-core-build/action.yaml"
@@ -22,7 +22,10 @@ on:
       - "deps-tasks.yml"
       - "Taskfile.yml"
       - "tools/scripts/deps-download/**"
-      - "!components/core/tools/scripts/lib_install/macos-12/**"
+      - "!components/core/tools/scripts/lib_install/macos/**"
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
   workflow_dispatch:
 
 env:

.github/workflows/clp-docs.yaml

@@ -3,6 +3,9 @@ name: "clp-docs"
 on:
   pull_request:
   push:
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
   workflow_dispatch:
 
 concurrency:

.github/workflows/clp-execution-image-build.yaml

@@ -11,6 +11,9 @@ on:
       - ".github/actions/clp-execution-image-build/action.yaml"
       - ".github/workflows/clp-execution-image-build.yaml"
       - "tools/docker-images/**/*"
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
   workflow_dispatch:
 
 concurrency:

.github/workflows/clp-lint.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
   push:
   schedule:
-    # Run at midnight UTC every day with 15 minutes delay added to avoid high load periods
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
     - cron: "15 0 * * *"
   workflow_dispatch:
 

.github/workflows/clp-pr-title-checks.yaml

@@ -0,0 +1,23 @@
+name: "clp-pr-title-checks"
+
+on:
+  pull_request_target:
+    types: ["edited", "opened", "reopened"]
+    branches: ["main"]
+
+permissions:
+  pull-requests: "read"
+
+concurrency:
+  group: "${{github.workflow}}-${{github.ref}}"
+
+  # Cancel in-progress jobs for efficiency
+  cancel-in-progress: true
+
+jobs:
+  conventional-commits:
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: "amannn/action-semantic-pull-request@v5"
+        env:
+          GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}"

Taskfile.yml

@@ -228,10 +228,10 @@ tasks:
       - "client/src/**/*.css"
       - "client/src/**/*.jsx"
       - "client/src/webpack.config.js"
-      - "yscope-log-viewer/.babelrc"
-      - "yscope-log-viewer/customized-packages/**/*"
       - "yscope-log-viewer/package.json"
+      - "yscope-log-viewer/public/**/*"
       - "yscope-log-viewer/src/**/*"
+      - "yscope-log-viewer/tsconfig.json"
       - "yscope-log-viewer/webpack.common.js"
       - "yscope-log-viewer/webpack.prod.js"
     dir: "components/log-viewer-webui"
@@ -348,7 +348,7 @@ tasks:
       - "mkdir -p '{{.OUTPUT_TMP_DIR}}'"
       - >-
         curl --fail --location --show-error
-        "{{.URL_PREFIX}}/{{.TAR_NAME}}"
+        "{{trimSuffix "/" .URL_PREFIX}}/{{.TAR_NAME}}"
         --output "{{.TAR_PATH}}"
       - "tar xf '{{.TAR_PATH}}' --directory '{{.OUTPUT_TMP_DIR}}'"
       - "mv '{{.EXTRACTED_DIR}}' '{{.OUTPUT_DIR}}'"

components/core/CMakeLists.txt

@@ -194,7 +194,7 @@ else()
 endif()
 
 # Find and setup msgpack
-find_package(msgpack-cxx 6.0.0 REQUIRED)
+find_package(msgpack-cxx 7.0.0 REQUIRED)
 if(msgpack-cxx_FOUND)
     message(STATUS "Found msgpack-cxx ${msgpack-cxx_VERSION}")
 else()

components/core/cmake/Modules/FindMariaDBClient.cmake

@@ -20,6 +20,28 @@ include(cmake/Modules/FindLibraryDependencies.cmake)
 find_package(PkgConfig)
 pkg_check_modules(mariadbclient_PKGCONF QUIET "lib${mariadbclient_LIBNAME}")
 
+if(NOT mariadbclient_PKGCONF_FOUND AND APPLE)
+    execute_process(
+        COMMAND brew --prefix mariadb-connector-c
+        RESULT_VARIABLE mariadbclient_BREW_RESULT
+        OUTPUT_VARIABLE mariadbclient_MACOS_PREFIX
+    )
+    if(NOT mariadbclient_BREW_RESULT EQUAL 0)
+        message(
+            FATAL_ERROR
+            "pkg-config cannot find ${mariadbclient_LIBNAME} and mariadb-connector-c isn't"
+            " installed via Homebrew"
+        )
+    endif()
+    string(STRIP "${mariadbclient_MACOS_PREFIX}" mariadbclient_MACOS_PREFIX)
+    list(PREPEND CMAKE_PREFIX_PATH ${mariadbclient_MACOS_PREFIX})
+    pkg_check_modules(mariadbclient_PKGCONF QUIET "lib${mariadbclient_LIBNAME}")
+endif()
+
+if(NOT mariadbclient_PKGCONF_FOUND)
+    message(FATAL_ERROR "pkg-config cannot find ${mariadbclient_LIBNAME}")
+endif()
+
 # Set include directory
 find_path(MariaDBClient_INCLUDE_DIR mysql.h
         HINTS ${mariadbclient_PKGCONF_INCLUDEDIR}

components/core/src/clp/CurlDownloadHandler.cpp

@@ -1,13 +1,22 @@
 #include "CurlDownloadHandler.hpp"
 
+#include <algorithm>
+#include <array>
+#include <cctype>
 #include <chrono>
 #include <cstddef>
 #include <memory>
+#include <optional>
 #include <string>
 #include <string_view>
+#include <unordered_map>
+#include <unordered_set>
 #include <utility>
 
 #include <curl/curl.h>
+#include <fmt/core.h>
+
+#include "ErrorCode.hpp"
 
 namespace clp {
 CurlDownloadHandler::CurlDownloadHandler(
@@ -19,7 +28,8 @@ CurlDownloadHandler::CurlDownloadHandler(
         size_t offset,
         bool disable_caching,
         std::chrono::seconds connection_timeout,
-        std::chrono::seconds overall_timeout
+        std::chrono::seconds overall_timeout,
+        std::optional<std::unordered_map<std::string, std::string>> const& http_header_kv_pairs
 )
         : m_error_msg_buf{std::move(error_msg_buf)} {
     if (nullptr != m_error_msg_buf) {
@@ -48,13 +58,55 @@ CurlDownloadHandler::CurlDownloadHandler(
     m_easy_handle.set_option(CURLOPT_TIMEOUT, static_cast<long>(overall_timeout.count()));
 
     // Set up http headers
+    constexpr std::string_view cRangeHeaderName{"range"};
+    constexpr std::string_view cCacheControlHeaderName{"cache-control"};
+    constexpr std::string_view cPragmaHeaderName{"pragma"};
+    std::unordered_set<std::string_view> const reserved_headers{
+            cRangeHeaderName,
+            cCacheControlHeaderName,
+            cPragmaHeaderName
+    };
     if (0 != offset) {
-        std::string const range{"Range: bytes=" + std::to_string(offset) + "-"};
-        m_http_headers.append(range);
+        m_http_headers.append(fmt::format("{}: bytes={}-", cRangeHeaderName, offset));
     }
     if (disable_caching) {
-        m_http_headers.append("Cache-Control: no-cache");
-        m_http_headers.append("Pragma: no-cache");
+        m_http_headers.append(fmt::format("{}: no-cache", cCacheControlHeaderName));
+        m_http_headers.append(fmt::format("{}: no-cache", cPragmaHeaderName));
+    }
+    if (http_header_kv_pairs.has_value()) {
+        for (auto const& [key, value] : http_header_kv_pairs.value()) {
+            // HTTP header field-name (key) is case-insensitive:
+            // https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
+            // Therefore, we convert keys to lowercase for comparison with the reserved keys.
+            // NOTE: We do not check for duplicate keys due to case insensitivity, leaving duplicate
+            // handling to the server.
+            auto lower_key{key};
+            std::transform(
+                    lower_key.begin(),
+                    lower_key.end(),
+                    lower_key.begin(),
+                    [](unsigned char c) -> char {
+                        // Implicitly cast the input character into `unsigned char` to avoid UB:
+                        // https://en.cppreference.com/w/cpp/string/byte/tolower
+                        return static_cast<char>(std::tolower(c));
+                    }
+            );
+            if (reserved_headers.contains(lower_key) || value.ends_with("\r\n")) {
+                throw CurlOperationFailed(
+                        ErrorCode_Failure,
+                        __FILE__,
+                        __LINE__,
+                        CURLE_BAD_FUNCTION_ARGUMENT,
+                        fmt::format(
+                                "`CurlDownloadHandler` failed to construct with the following "
+                                "invalid header: {}:{}",
+                                key,
+                                value
+                        )
+                );
+            }
+            m_http_headers.append(fmt::format("{}: {}", key, value));
+        }
     }
     if (false == m_http_headers.is_empty()) {
         m_easy_handle.set_option(CURLOPT_HTTPHEADER, m_http_headers.get_raw_list());

components/core/src/clp/CurlDownloadHandler.hpp

@@ -5,7 +5,10 @@
 #include <chrono>
 #include <cstddef>
 #include <memory>
+#include <optional>
+#include <string>
 #include <string_view>
+#include <unordered_map>
 
 #include <curl/curl.h>
 
@@ -53,6 +56,9 @@ class CurlDownloadHandler {
      * Doc: https://curl.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html
      * @param overall_timeout Maximum time that the transfer may take. Note that this includes
      * `connection_timeout`. Doc: https://curl.se/libcurl/c/CURLOPT_TIMEOUT.html
+     * @param http_header_kv_pairs Key-value pairs representing HTTP headers to pass to the server
+     * in the download request. Doc: https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html
+     * @throw CurlOperationFailed if an error occurs.
      */
     explicit CurlDownloadHandler(
             std::shared_ptr<ErrorMsgBuf> error_msg_buf,
@@ -63,7 +69,9 @@ class CurlDownloadHandler {
             size_t offset = 0,
             bool disable_caching = false,
             std::chrono::seconds connection_timeout = cDefaultConnectionTimeout,
-            std::chrono::seconds overall_timeout = cDefaultOverallTimeout
+            std::chrono::seconds overall_timeout = cDefaultOverallTimeout,
+            std::optional<std::unordered_map<std::string, std::string>> const& http_header_kv_pairs
+            = std::nullopt
     );
 
     // Disable copy/move constructors/assignment operators

components/core/src/clp/NetworkReader.cpp

@@ -6,7 +6,10 @@
 #include <memory>
 #include <mutex>
 #include <optional>
+#include <string>
 #include <string_view>
+#include <unordered_map>
+#include <utility>
 
 #include <curl/curl.h>
 
@@ -118,7 +121,8 @@ NetworkReader::NetworkReader(
         std::chrono::seconds overall_timeout,
         std::chrono::seconds connection_timeout,
         size_t buffer_pool_size,
-        size_t buffer_size
+        size_t buffer_size,
+        std::optional<std::unordered_map<std::string, std::string>> http_header_kv_pairs
 )
         : m_src_url{src_url},
           m_offset{offset},
@@ -130,7 +134,12 @@ NetworkReader::NetworkReader(
     for (size_t i = 0; i < m_buffer_pool_size; ++i) {
         m_buffer_pool.emplace_back(m_buffer_size);
     }
-    m_downloader_thread = std::make_unique<DownloaderThread>(*this, offset, disable_caching);
+    m_downloader_thread = std::make_unique<DownloaderThread>(
+            *this,
+            offset,
+            disable_caching,
+            std::move(http_header_kv_pairs)
+    );
     m_downloader_thread->start();
 }
 
@@ -215,7 +224,8 @@ auto NetworkReader::DownloaderThread::thread_method() -> void {
                 m_offset,
                 m_disable_caching,
                 m_reader.m_connection_timeout,
-                m_reader.m_overall_timeout
+                m_reader.m_overall_timeout,
+                m_http_header_kv_pairs
         };
         auto const ret_code{curl_handler.perform()};
         // Enqueue the last filled buffer, if any