Add tsan for vortex-ffi, fix asan instrumentation

.github/workflows/ci.yml

@@ -379,51 +379,132 @@ jobs:
           flags: ${{ matrix.suite }}
           use_oidc: true
 
-  rust-test:
-    name: "Rust tests (sanitizer)"
-    timeout-minutes: 40
+  rust-test-sanitizer:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sanitizer: asan
+            sanitizer_flags: "-Zsanitizer=address -Zsanitize=leak"
+          - sanitizer: msan
+            sanitizer_flags: "-Zsanitizer=memory"
+          - sanitizer: tsan
+            sanitizer_flags: "-Zsanitizer=thread"
+    name: "Rust tests (${{ matrix.sanitizer }})"
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
           && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-test-sanitizer', github.run_id)
           || 'ubuntu-latest' }}
+    timeout-minutes: 40
     env:
-      # Add debug symbols and enable ASAN/LSAN with better output
-      ASAN_OPTIONS: "symbolize=1:print_stats=1:check_initialization_order=1:detect_leaks=1:halt_on_error=0:verbosity=1:leak_check_at_exit=1"
-      LSAN_OPTIONS: "verbosity=1:report_objects=1"
+      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
+      LSAN_OPTIONS: "report_objects=1"
       ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
-      # Link against DuckDB debug build
-      VX_DUCKDB_DEBUG: "1"
-      # Keep frame pointers for better stack traces
-      CARGO_PROFILE_DEV_DEBUG: "true"
-      CARGO_PROFILE_TEST_DEBUG: "true"
-      # Skip slow tests that are too expensive under sanitizer
+      MSAN_OPTIONS: "symbolize=1"
+      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      TSAN_OPTIONS: "symbolize=1"
+      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
       VORTEX_SKIP_SLOW_TESTS: "1"
+      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
+      # unset -Zsanitizer flag and we should allow that.
+      RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none"
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'
         with:
           sccache: s3
       - uses: actions/checkout@v6
       - uses: ./.github/actions/setup-prebuild
-      - name: Install nightly for sanitizer
+      - name: Install Rust nightly toolchain
         run: |
           rustup toolchain install $NIGHTLY_TOOLCHAIN
           rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
-      - name: Rust Tests
-        env:
-          RUSTFLAGS: "-A warnings -Zsanitizer=address -Zsanitizer=leak --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none"
+          export RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}"
+      - name: Build tests with sanitizer
         run: |
-          # Build with full debug info first (helps with caching)
-          cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features  \
-            --target x86_64-unknown-linux-gnu \
-            -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
-          # Run tests with sanitizers and debug output
-          cargo +$NIGHTLY_TOOLCHAIN nextest run \
-            --locked \
-            --all-features \
-            --no-fail-fast \
-            --target x86_64-unknown-linux-gnu \
-            -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
+          cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \
+            --target x86_64-unknown-linux-gnu -Zbuild-std \
+            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
+
+      - name: Run tests with sanitizer
+        run: |
+          cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features \
+            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
+            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
+
+        # vortex-ffi requires --no-default-features as otherwise we pull in
+        # Mimalloc which interferes with sanitizers
+        # cargo nextest reports less sanitizer issues than cargo test
+        # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in
+      - name: Run vortex-ffi tests with sanitizer
+        run: |
+          cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \
+            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
+            -p vortex-ffi -- --no-capture
+
+  rust-ffi-test-sanitizer:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # We don't run memory sanitizer as it's clang-only and provides many
+          # false positives for Catch2
+          - sanitizer: asan
+            sanitizer_flags: "-Zsanitizer=address -Zsanitize=leak"
+          - sanitizer: tsan
+            sanitizer_flags: "-Zsanitizer=thread"
+    name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})"
+    timeout-minutes: 40
+    env:
+      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
+      LSAN_OPTIONS: "report_objects=1"
+      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      MSAN_OPTIONS: "symbolize=1"
+      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      TSAN_OPTIONS: "symbolize=1"
+      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      VORTEX_SKIP_SLOW_TESTS: "1"
+      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
+      # unset -Zsanitizer flag and we should allow that.
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id)
+          || 'ubuntu-latest' }}
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup-prebuild
+      - name: Install rustfilt
+        run: |
+          cargo install rustfilt
+      - name: Install Rust nightly toolchain
+        run: |
+          rustup toolchain install $NIGHTLY_TOOLCHAIN
+          rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
+
+          # Export flags here so that rustfilt won't be built with sanitizers
+          export RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \
+                --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 \
+                -C opt-level=0 -C strip=none -Zexternal-clangrt \
+                ${{ matrix.sanitizer_flags }}"
+      - name: Build FFI library
+        run: |
+          # TODO(myrrc): remove --no-default-features
+          cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \
+            --target x86_64-unknown-linux-gnu -Zbuild-std \
+            -p vortex-ffi
+      - name: Build FFI library tests
+        run: |
+          cd vortex-ffi
+          cmake -Bbuild -DBUILD_TESTS=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
+          cmake --build build -j
+      - name: Run tests
+        run: |
+          set -o pipefail
+          ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt -i-
 
   cuda-build-lint:
     if: github.repository == 'vortex-data/vortex'

vortex-ffi/CMakeLists.txt

@@ -5,19 +5,40 @@ cmake_minimum_required(VERSION 3.10)
 project(VortexFFI
     VERSION 0.0.1
     LANGUAGES C)
-set(CMAKE_C_STANDARD 17)
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_C_STANDARD 17)
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -Wextra -Wpedantic")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -Wextra -Wpedantic")
 
-option(BUILD_TESTING "Build tests" ON)
+option(BUILD_TESTS "Build tests" OFF)
+
+set(SANITIZER "" CACHE STRING "Build with sanitizers")
+set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library")
 
 if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Debug)
 endif()
 
+if (NOT SANITIZER STREQUAL "")
+    message(NOTICE "Sanitizer: ${SANITIZER}")
+    if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+        message(FATAL_ERROR "Only debug build is supported for sanitizer builds")
+    endif()
+
+    if (SANITIZER STREQUAL "asan")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address,undefined,leak")
+        if (BUILD_TESTS)
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address,undefined,leak")
+        endif()
+    elseif (SANITIZER STREQUAL "tsan")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread")
+        if (BUILD_TESTS)
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
+        endif()
+    else()
+        message(FATAL_ERROR "Unknown sanitizer ${SANITIZER}")
+    endif()
+endif()
+
 message(NOTICE "Build type: ${CMAKE_BUILD_TYPE}")
 if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND
     NOT CMAKE_BUILD_TYPE STREQUAL "Release" AND
@@ -26,11 +47,11 @@ if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND
 endif()
 
 if (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
-    set(LIBRARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../target/release_debug")
+    set(CMAKE_BUILD_TYPE_LOWER "release_debug")
 else()
     string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)
-    set(LIBRARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../target/${CMAKE_BUILD_TYPE_LOWER}")
 endif()
+set(LIBRARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../target/${TARGET_TRIPLE}/${CMAKE_BUILD_TYPE_LOWER}")
 
 if(WIN32)
     set(LIBRARY_PATH "${LIBRARY_DIR}/libvortex_ffi.lib")
@@ -71,8 +92,12 @@ set_target_properties(vortex_ffi_shared PROPERTIES
     INTERFACE_LINK_OPTIONS "LINKER:-rpath,${LIBRARY_DIR}"
 )
 
-if (BUILD_TESTING)
+if (BUILD_TESTS)
     enable_language(CXX)
+    set(CMAKE_CXX_STANDARD 20)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -Wextra -Wpedantic")
+
     enable_testing()
     add_subdirectory(test)
 endif()

vortex-ffi/README.md

@@ -26,32 +26,90 @@ item cannot be referenced in the documentation if it does not have a documentati
 
 ## Updating Headers
 
-To rebuild the header file (requires nightly toolchain):
+To rebuild the header file:
 
-```shell
+```sh
 cargo +nightly build -p vortex-ffi
 ```
 
 The header generation uses cbindgen's macro expansion feature which requires nightly.
 Stable builds use the checked-in header file at `cinclude/vortex.h`.
 
-### Development Workflow
-
-- **For header changes**: Use nightly toolchain to regenerate headers after modifying FFI code
-- **For regular development**: Stable toolchain builds work with existing checked-in headers
-- **CI validation**: Automated checks verify header freshness using nightly toolchain
-
-### Testing
+### Testing C part
 
 Build the test library
 
-```
+```sh
 cmake -Bbuild
 cmake --build build -j $(nproc)
 ```
 
 Run the tests
 
-```
+```sh
 ctest --test-dir build -j $(nproc)
 ```
+
+You would need C++ compiler toolchain to run the tests since they use Catch2.
+
+### Testing Rust part with sanitizers
+
+AddressSanitizer:
+
+```sh
+# inside vortex-ffi
+RUSTFLAGS="-Z sanitizer=address" \
+cargo +nightly test -Zbuild-std \
+    --no-default-features --target <target triple> \
+    -- --no-capture
+```
+
+MemorySanitizer:
+
+```sh
+RUSTFLAGS="-Z sanitizer=memory -Cunsafe-allow-abi-mismatch=sanitizer" \
+cargo +nightly test -Zbuild-std \
+    --no-default-features --target <target triple> \
+    -- --no-capture
+```
+
+ThreadSanitizer:
+
+```sh
+RUSTFLAGS="-Z sanitizer=thread -Cunsafe-allow-abi-mismatch=sanitizer" \
+cargo +nightly test -Zbuild-std \
+    --no-default-features --target <target triple> \
+    -- --no-capture
+```
+
+- `-Zbuild-std` is needed as memory and thread sanitizers report std errors
+  otherwise.
+- `--no-default-features` is needed as we use Mimalloc otherwise which interferes
+with sanitizers.
+- `allow-abi-mismatch` is safe because in our dependency graph only crates like
+  `compiler_builtins` unset sanitization, and they do it on purpose.
+- Make sure to use `cargo test` and not `cargo nextest` as nextest reports less
+leaks.
+- If you want stack trace symbolization, install `llvm-symbolizer`.
+
+### Testing Rust and C with sanitizers
+
+1. Build FFI library with external sanitizer runtime:
+
+```sh
+RUSTFLAGS="-Zsanitizer=address -Zexternal-clangrt" \
+cargo +nightly build -Zbuild-std --target=<target triple> \
+--no-default-features -p vortex-ffi
+```
+
+2. Build tests with target triple
+
+```sh
+cmake -Bbuild -DWITH_ASAN=1 -DTARGET_TRIPLE=<target triple>
+```
+
+3. Run the tests (ctest doesn't output failures in detail):
+
+```
+./build/test/vortex_ffi_test 2>& 1 | rustfilt -i-
+```