MCOL-5464: Fixes of bugs from ASAN warnings, part one (mariadb-corporation#2792)

* Fixes of bugs from ASAN warnings, part one

* MQC as static library, with nifty counter for global map and mutex

* Switch clang to 16

* link messageqcpp to execplan

Author: mariadb-LeonidFedorov

.drone.jsonnet

@@ -31,7 +31,7 @@ local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_CONFIG=mysql_relea
                    '-DPLUGIN_GSSAPI=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' +
                    '-DWITH_EMBEDDED_SERVER=NO -DWITH_WSREP=NO -DWITH_COREDUMPS=ON';
 
-local clang_version = '14';
+local clang_version = '16';
 local gcc_version = '11';
 
 local clang_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-' + clang_version + ' 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-' + clang_version + ' && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 ';

CMakeLists.txt

@@ -205,7 +205,7 @@ IF (NOT INSTALL_LAYOUT)
     ENDIF()
     OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protector, relro, etc)" ${security_default})
     OPTION(SECURITY_HARDENED_NEW "Use new security-enhancing compilier features" OFF)
-    IF(SECURITY_HARDENED)
+    IF(SECURITY_HARDENED AND NOT WITH_ASAN AND NOT WITH_UBSAN AND NOT WITH_TSAN AND NOT WITH_GPROF)
       # security-enhancing flags
       MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC")
       MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now")
@@ -237,10 +237,15 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-register")
 endif()
 
+IF (WITH_COLUMNSTORE_ASAN)
+    MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO)
+    MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC")
+ENDIF()
+
 MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-copy" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL)
 MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-declarations" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL)
 
-MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall")
+MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall -Wextra")
 SET (ENGINE_LDFLAGS    "-Wl,--no-as-needed -Wl,--add-needed")
 SET (ENGINE_DT_LIB datatypes)
 SET (ENGINE_COMMON_LIBS     messageqcpp loggingcpp configcpp idbboot boost_thread xml2 pthread rt ${ENGINE_DT_LIB})

build/bootstrap_mcs.sh

@@ -18,17 +18,25 @@ message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal
 
 BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
 DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "Rocky")
+
+cd $SCRIPT_LOCATION
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo))
+cd -
+
 
 optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE
 optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS
-optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true
+optparse.define short=D long=install-deps desc="Install dependences" variable=INSTALL_DEPS default=false value=true
 optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
 optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true
 optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
 optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
-optparse.define short=b long=branch desc="Choouse git branch ('none' for menu)" variable=BRANCH
+optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH
 optparse.define short=D long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
+optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
+optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
+optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core"
 
 source $( optparse.build )
 
@@ -45,27 +53,34 @@ INSTALL_PREFIX="/usr/"
 DATA_DIR="/var/lib/mysql/data"
 CMAKE_BIN_NAME=cmake
 CTEST_BIN_NAME=ctest
+CONFIG_DIR="/etc/my.cnf.d"
+
+if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then
+    CONFIG_DIR="/etc/mysql/mariadb.conf.d"
+fi
+
 
 select_branch()
 {
+    cd $SCRIPT_LOCATION
+    CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
     if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then
-        if [[ $BRANCH = 'none' ]]; then
+        if [[ $BRANCH = "" ]]; then
             getChoice -q "Select your branch" -o BRANCHES
             BRANCH=$selectedChoice
         fi
-        cd $SCRIPT_LOCATION
-        message "Selecting $BRANCH branch for Columnstore"
-        git checkout $BRANCH
-        cd -
+        if [[ $BRANCH != $CURRENT_BRANCH ]]; then
+            message "Selecting $BRANCH branch for Columnstore"
+            git checkout $BRANCH
+        fi
 
         message "Turning off Columnstore submodule auto update via gitconfig"
         cd $MDB_SOURCE_PATH
         git config submodule.storage/columnstore/columnstore.update none
         cd -
     fi
 
-    cd $SCRIPT_LOCATION
-    CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
     cd -
     message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch"
 }
@@ -136,7 +151,9 @@ clean_old_installation()
     rm -rf /var/lib/columnstore/local/
     rm -f /var/lib/columnstore/storagemanager/storagemanager-lock
     rm -f /var/lib/columnstore/storagemanager/cs-initialized
+    rm -rf /var/log/mariadb/columnstore/*
     rm -rf /tmp/*
+    rm -rf $REPORT_PATH
     rm -rf /var/lib/mysql
     rm -rf /var/run/mysqld
     rm -rf $DATA_DIR
@@ -161,7 +178,8 @@ build()
                      -DWITH_WSREP=OFF
                      -DWITH_SSL=system
                      -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX
-                     -DCMAKE_EXPORT_COMPILE_COMMANDS=1"
+                     -DCMAKE_EXPORT_COMPILE_COMMANDS=1
+                     "
 
 
     if [[ $SKIP_UNIT_TESTS = true ]] ; then
@@ -172,11 +190,21 @@ build()
         message "Buiding with unittests"
     fi
 
+    if [[ $ASAN = true ]] ; then
+        warn "Building with ASAN"
+        MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
+    fi
+
     if [[ $WITHOUT_COREDUMPS = true ]] ; then
         warn "Cores are not dumped"
-
     else
         MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON"
+        echo "${REPORT_PATH}/core_%e.%p" | sudo tee /proc/sys/kernel/core_pattern
+    fi
+
+    if [[ $MAKEFILE_VERBOSE = true ]] ; then
+        warn "Verbosing Makefile Commands"
+        MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
     fi
 
     if [[ $RUN_BENCHMARKS = true ]] ; then
@@ -197,7 +225,6 @@ build()
         message "Buiding without microbenchmarks"
     fi
 
-
     cd $MDB_SOURCE_PATH
 
     if [[ $SKIP_SUBMODULES = true ]] ; then
@@ -228,11 +255,13 @@ build()
     message "building with flags $MDB_CMAKE_FLAGS"
 
     local CPUS=$(getconf _NPROCESSORS_ONLN)
-    ${CMAKE_BIN_NAME} . -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \
-    make -j $CPUS install
+    ${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ |
+    make -j $CPUS
+    message "Installing silently"
+    make -j $CPUS install > /dev/null
 
     if [ $? -ne 0 ]; then
-        error "!!!! BUILD FAILED"
+        error "!!!! BUILD FAILED !!!!"
         exit 1
     fi
     cd -
@@ -259,7 +288,7 @@ run_unit_tests()
     else
         message "Running unittests"
         cd $MDB_SOURCE_PATH
-        ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc)
+        ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress
         cd -
     fi
 }
@@ -271,14 +300,56 @@ run_microbenchmarks_tests()
     else
         message "Runnning microbenchmarks"
         cd $MDB_SOURCE_PATH
-        ${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc)
+        ${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
         cd -
     fi
 }
 
+disable_plugins_for_bootstrap()
+{
+    find /etc -type f -exec sed -i 's/plugin-load-add=auth_gssapi.so//g' {} +
+    find /etc -type f -exec sed -i 's/plugin-load-add=ha_columnstore.so//g' {} +
+}
+
+enable_columnstore_back()
+{
+    echo plugin-load-add=ha_columnstore.so >> $CONFIG_DIR/columnstore.cnf
+}
+
+fix_config_files()
+{
+    message Fixing config files
+
+    THREAD_STACK_SIZE="20M"
+
+    SYSTEMD_SERVICE_DIR="/usr/lib/systemd/system"
+    if [[ $ASAN = true ]] ; then
+        COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf
+        if grep -q thread_stack $COLUMNSTORE_CONFIG; then
+            warn "MDB Server has thread_stack settings on $COLUMNSTORE_CONFIG check it's compatibility with ASAN"
+        else
+            echo "thread_stack = ${THREAD_STACK_SIZE}" >> $COLUMNSTORE_CONFIG
+            message "thread_stack was set to ${THREAD_STACK_SIZE} in $COLUMNSTORE_CONFIG"
+        fi
+
+        MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service
+        if grep -q ASAN $MDB_SERVICE_FILE; then
+            warn "MDB Server has ASAN options in $MDB_SERVICE_FILE, check it's compatibility"
+        else
+            echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}'" >> $MDB_SERVICE_FILE
+            message "ASAN options were added to $MDB_SERVICE_FILE"
+        fi
+    fi
+    systemctl daemon-reload
+}
+
 install()
 {
     message "Installing MariaDB"
+    disable_plugins_for_bootstrap
+
+    mkdir -p $REPORT_PATH
+    chmod 777 $REPORT_PATH
 
     check_user_and_group
 
@@ -288,10 +359,15 @@ install()
 socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
 
     mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so
+    mkdir -p /var/lib/mysql
+    chown mysql:mysql /var/lib/mysql
+
     message "Running mysql_install_db"
-    mysql_install_db --rpm --user=mysql
+    sudo -u mysql mysql_install_db --rpm --user=mysql > /dev/null
     mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so
 
+    enable_columnstore_back
+
     mkdir -p /etc/columnstore
 
     cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
@@ -308,7 +384,7 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
         > /etc/mysql/debian.cnf
     fi
 
-    systemctl daemon-reload
+    fix_config_files
 
     if [ -d "/etc/mysql/mariadb.conf.d/" ]; then
         message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d"
@@ -345,9 +421,23 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
     chmod 777 /var/log/mariadb/columnstore
 }
 
+
+smoke()
+{
+    message "Creating test database"
+    mariadb -e "create database if not exists test;"
+    message "Selecting magic numbers"
+    MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql`
+    if [[ $MAGIC == '42' ]] ; then
+        message "Great answer correct"
+    else
+        warn "Smoke failed, answer is '$MAGIC'"
+    fi
+}
+
 select_branch
 
-if [[ $SKIP_DEPS = false ]] ; then
+if [[ $INSTALL_DEPS = true ]] ; then
     install_deps
 fi
 
@@ -358,4 +448,6 @@ run_unit_tests
 run_microbenchmarks_tests
 install
 start_service
+smoke
+
 message "$color_green FINISHED $color_normal"

datatypes/mcs_decimal.h

@@ -165,6 +165,27 @@ const int128_t mcs_pow_10_128[20] = {
     100000000000000000000000000000000000000_xxl,
 };
 
+const long long columnstore_precision[19] = {0,
+                                             9,
+                                             99,
+                                             999,
+                                             9999,
+                                             99999,
+                                             999999,
+                                             9999999,
+                                             99999999,
+                                             999999999,
+                                             9999999999LL,
+                                             99999999999LL,
+                                             999999999999LL,
+                                             9999999999999LL,
+                                             99999999999999LL,
+                                             999999999999999LL,
+                                             9999999999999999LL,
+                                             99999999999999999LL,
+                                             999999999999999999LL};
+
+
 const int128_t ConversionRangeMaxValue[20] = {9999999999999999999_xxl,
                                               99999999999999999999_xxl,
                                               999999999999999999999_xxl,