Run tests in parallel via ctest

Author: mvandeberg

Dockerfile

@@ -6,6 +6,8 @@ ARG STEEM_STATIC_BUILD=ON
 ENV STEEM_STATIC_BUILD ${STEEM_STATIC_BUILD}
 ARG BUILD_STEP
 ENV BUILD_STEP ${BUILD_STEP}
+ARG CI_BUILD
+ENV CI_BUILD ${CI_BUILD}
 
 ENV LANG=en_US.UTF-8
 
@@ -53,6 +55,12 @@ RUN \
 
 ADD . /usr/local/src/steem
 
+RUN \
+    if [ "$CI_BUILD" ] ; then \
+        pip3 install awscli --user && \
+        aws s3 cp s3://steemit-dev-ci/steemd-CTestCostData.txt /usr/local/src/steem/CTestCostData.txt ; \
+    fi
+
 RUN \
     if [ "$BUILD_STEP" = "1" ] || [ ! "$BUILD_STEP" ] ; then \
     cd /usr/local/src/steem && \
@@ -68,8 +76,13 @@ RUN \
         -DENABLE_MIRA=ON \
         .. && \
     make -j$(nproc) chain_test mira_test test_fixed_string plugin_test && \
-    ./tests/chain_test && \
-    ./tests/plugin_test && \
+    if [ "$CI_BUILD" ] ; then \
+        mkdir -p build/tests/Testing/Temporary && \
+        cp /usr/local/src/steem/CTestCostData.txt build/tests/Testing/Temporary ; \
+    fi && \
+    cd tests && \
+    ctest -j8 --output-on-failure && \
+    cd .. && \
     ./libraries/mira/test/mira_test && \
     ./programs/util/test_fixed_string && \
     cd /usr/local/src/steem && \
@@ -99,14 +112,23 @@ RUN \
         .. && \
     make -j$(nproc) chain_test test_fixed_string plugin_test && \
     make install && \
-    ./tests/chain_test && \
-    ./tests/plugin_test && \
+    if [ "$CI_BUILD" ] ; then \
+        mkdir -p build/tests/Testing/Temporary && \
+        cp /usr/local/src/steem/CTestCostData.txt build/tests/Testing/Temporary ; \
+    fi && \
+    cd tests && \
+    ctest -j8 --output-on-failure && \
+    cd .. && \
     ./programs/util/test_fixed_string && \
     cd /usr/local/src/steem && \
     doxygen && \
     PYTHONPATH=programs/build_helpers \
     python3 -m steem_build_helpers.check_reflect && \
     programs/build_helpers/get_config_check.sh && \
+    if [ "$CI_BUILD" ] ; then \
+        aws s3 cp s3://steemit-dev-ci/steemd-CTestCostData.txt s3://steemit-dev-ci/steemd-CTestCostData.txt.bk && \
+        aws s3 cp build/tests/Testing/Temporary/CTestCostData.txt s3://steemit-dev-ci/steemd-CTestCostData.txt; \
+    fi && \
     rm -rf /usr/local/src/steem/build ; \
     fi
 
@@ -127,8 +149,13 @@ RUN \
         -DCHAINBASE_CHECK_LOCKING=OFF \
         .. && \
     make -j$(nproc) chain_test plugin_test && \
-    ./tests/chain_test && \
-    ./tests/plugin_test && \
+    if [ "$CI_BUILD" ] ; then \
+        mkdir -p build/tests/Testing/Temporary && \
+        cp /usr/local/src/steem/CTestCostData.txt build/tests/Testing/Temporary ; \
+    fi && \
+    cd tests && \
+    ctest -j8 --output-on-failure && \
+    cd .. && \
     mkdir -p /var/cobertura && \
     gcovr --object-directory="../" --root=../ --xml-pretty --gcov-exclude=".*tests.*" --gcov-exclude=".*fc.*" --gcov-exclude=".*app*" --gcov-exclude=".*net*" --gcov-exclude=".*plugins*" --gcov-exclude=".*schema*" --gcov-exclude=".*time*" --gcov-exclude=".*utilities*" --gcov-exclude=".*wallet*" --gcov-exclude=".*programs*" --gcov-exclude=".*vendor*" --output="/var/cobertura/coverage.xml" && \
     cd /usr/local/src/steem && \

ciscripts/buildscript.sh

@@ -4,7 +4,7 @@ export IMAGE_NAME="steemit/steem:$BRANCH_NAME"
 if [[ $IMAGE_NAME == "steemit/steem:stable" ]] ; then
   IMAGE_NAME="steemit/steem:latest"
 fi
-sudo docker build --build-arg BUILD_STEP=2 -t=$IMAGE_NAME .
+sudo docker build --build-arg CI_BUILD=1 --build-arg BUILD_STEP=2 -t=$IMAGE_NAME .
 sudo docker login --username=$DOCKER_USER --password=$DOCKER_PASS
 sudo docker push $IMAGE_NAME
 # make docker cleanup after itself and delete all exited containers

ciscripts/buildtests.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-sudo docker build --build-arg BUILD_STEP=1 -t=steemit/steem:tests .
+sudo docker build --build-arg CI_BUILD=1 --build-arg BUILD_STEP=1 -t=steemit/steem:tests .
 sudo docker run -v $WORKSPACE:/var/jenkins steemit/steem:tests cp -r /var/cobertura /var/jenkins
 # make docker cleanup after itself and delete all exited containers
 sudo docker rm -v $(docker ps -a -q -f status=exited) || true

libraries/chain/include/steem/chain/database.hpp

@@ -35,10 +35,15 @@ namespace steem { namespace chain {
    using abstract_plugin = appbase::abstract_plugin;
 
    class database;
+
+#ifdef ENABLE_MIRA
    using set_index_type_func = std::function< void(database&, mira::index_type, const boost::filesystem::path&, const boost::any&) >;
+#endif
 
    struct index_delegate {
+#ifdef ENABLE_MIRA
       set_index_type_func set_index_type;
+#endif
    };
 
    using index_delegate_map = std::map< std::string, index_delegate >;

libraries/chain/include/steem/chain/index.hpp

@@ -59,6 +59,8 @@ void add_plugin_index( database& db )
 
 } }
 
+#ifdef ENABLE_MIRA
+
 #define STEEM_ADD_CORE_INDEX(db, index_name)                                                                 \
    do {                                                                                                      \
       steem::chain::add_core_index< index_name >( db );                                                      \
@@ -78,3 +80,21 @@ void add_plugin_index( database& db )
             { _db.get_mutable_index< index_name >().mutable_indices().set_index_type( type, p, cfg ); };     \
       db.set_index_delegate( #index_name, std::move( delegate ) );                                           \
    } while( false )
+
+#else
+
+#define STEEM_ADD_CORE_INDEX(db, index_name)                                                                 \
+   do {                                                                                                      \
+      steem::chain::add_core_index< index_name >( db );                                                      \
+      steem::chain::index_delegate delegate;                                                                 \
+      db.set_index_delegate( #index_name, std::move( delegate ) );                                           \
+   } while( false )
+
+#define STEEM_ADD_PLUGIN_INDEX(db, index_name)                                                               \
+   do {                                                                                                      \
+      steem::chain::add_plugin_index< index_name >( db );                                                    \
+      steem::chain::index_delegate delegate;                                                                 \
+      db.set_index_delegate( #index_name, std::move( delegate ) );                                           \
+   } while( false )
+
+#endif

libraries/chain/index.cpp

@@ -56,8 +56,8 @@ void initialize_core_indexes( database& db )
    STEEM_ADD_CORE_INDEX(db, nai_pool_index);
    STEEM_ADD_CORE_INDEX(db, smt_token_emissions_index);
 #endif
-   add_core_index< proposal_index                          >( db );
-   add_core_index< proposal_vote_index                     >( db );
+   STEEM_ADD_CORE_INDEX(db, proposal_index);
+   STEEM_ADD_CORE_INDEX(db, proposal_vote_index);
 }
 
 index_info::index_info() {}

libraries/protocol/include/steem/protocol/version.hpp

@@ -2,7 +2,7 @@
 
 #include <fc/string.hpp>
 #include <fc/time.hpp>
-#include <sys/sysmacros.h>
+//#include <sys/sysmacros.h>
 
 namespace steem { namespace protocol {