Modify ci for parallel build examples

jason-mao · jason-mao · commit 943c93847623 · 2020-11-16T17:41:38.000+08:00
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -94,6 +94,63 @@ before_script:
     - mkdir -p ${LOG_PATH}
     - ${ADF_PATH}/tools/ci/build_examples.sh "${CI_JOB_NAME}"
 
+.build_examples_make: &build_examples_make
+  <<: *build_template
+  parallel: 4
+  # This is a workaround for a rarely encountered issue with building examples in CI.
+  # Probably related to building of Kconfig in 'make clean' stage
+  retry: 1
+  artifacts:
+    when: always
+    paths:
+      - build_examples/*/*/*/build/*.bin
+      - build_examples/*/*/*/sdkconfig
+      - build_examples/*/*/*/build/*.elf
+      - build_examples/*/*/*/build/*.map
+      - build_examples/*/*/*/build/download.config
+      - build_examples/*/*/*/build/bootloader/*.bin
+      - build_examples/*/*/*/*/build/partition_table/*.bin
+      - $LOG_PATH
+    expire_in: 4 days
+  variables:
+    IDF_CI_BUILD: "1"
+    LOG_PATH: "$CI_PROJECT_DIR/log_examples_make"
+  # This is IDF CI limits
+  # only:
+    # # Here both 'variables' and 'refs' conditions are given. They are combined with "AND" logic.
+    # variables:
+    #   - $BOT_TRIGGER_WITH_LABEL == null
+    #   - $BOT_LABEL_BUILD
+    #   - $BOT_LABEL_EXAMPLE_TEST
+    #   - $BOT_LABEL_REGULAR_TEST
+    #   - $BOT_LABEL_WEEKEND_TEST
+    # refs:
+    #   - master
+    #   - /^release\/v/
+    #   - /^v\d+\.\d+(\.\d+)?($|-)/
+    #   - triggers
+    #   - schedules
+    #   - pipelines
+    #   - web
+  script:
+    # it's not possible to build 100% out-of-tree and have the "artifacts"
+    # mechanism work, but this is the next best thing
+    - rm -rf build_examples
+    - mkdir build_examples
+    - cd build_examples
+    # build some of examples
+    - mkdir -p ${LOG_PATH}
+
+    # enable pedantic flags for compilation
+    - export PEDANTIC_CFLAGS="-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function"
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} && export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
+
+    - ${ADF_PATH}/tools/ci/build_examples.sh
+    # Check if the tests demand Make built binaries. If not, delete them
+    - if [ "$EXAMPLE_TEST_BUILD_SYSTEM" == "make" ]; then exit 0; fi
+    - cd ..
+    - rm -rf build_examples
+
 # same as above, but for CMake
 .build_examples_cmake: &build_examples_cmake
   extends: .build_template
@@ -156,17 +213,8 @@ build_examples_v40_cmake:
     IDF_TARGET: esp32s2beta
     IDF_VERSION: release/v4.1
 
-build_examples_00:
-  <<: *build_examples_template
-
-build_examples_01:
-  <<: *build_examples_template
-
-.build_examples_02:
-  <<: *build_examples_template
-
-.build_examples_03:
-  <<: *build_examples_template
+build_examples_v33_make:
+  <<: *build_examples_make
 
 build_adf_test:
   <<: *build_template
@@ -231,7 +279,8 @@ assign_test:
   # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
   dependencies:
     - build_adf_test
-    - build_examples_00
+    - build_examples_v33_make
+    - build_examples_v40_cmake
   variables:
     TEST_FW_PATH: "$CI_PROJECT_DIR/general_test_framework"
     EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
@@ -254,7 +303,8 @@ assign_test:
   when: on_success
   dependencies:
     - assign_test
-    - build_examples_00
+    - build_examples_v33_make
+    - build_examples_v40_cmake
     - build_adf_test
   artifacts:
     when: always
diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Build all examples from the examples directory, out of tree to
 # ensure they can run when copied to a new directory.
@@ -23,7 +23,7 @@
 # -----------------------------------------------------------------------------
 # Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
 
-if [[ ! -z ${DEBUG} ]]
+if [[ ! -z ${DEBUG_SHELL} ]]
 then
   set -x # Activate the expand mode if DEBUG is anything but empty.
 fi
@@ -49,9 +49,6 @@ die() {
 
 echo "build_examples running in ${PWD}"
 
-# only 0 or 1 arguments
-[ $# -le 1 ] || die "Have to run as $(basename $0) [<JOB_NAME>]"
-
 export BATCH_BUILD=1
 export V=0 # only build verbose if there's an error
 
@@ -62,41 +59,37 @@ FAILED_EXAMPLES=""
 RESULT_ISSUES=22  # magic number result code for issues found
 LOG_SUSPECTED=${LOG_PATH}/common_log.txt
 touch ${LOG_SUSPECTED}
+SDKCONFIG_DEFAULTS_CI=sdkconfig.ci
+
+EXAMPLE_PATHS=$( find ${ADF_PATH}/examples/ -type f -name Makefile | grep -v "/build_system/cmake/" | sort )
 
-if [ $# -eq 0 ]
+if [ -z "${CI_NODE_TOTAL:-}" ]
 then
     START_NUM=0
+    if [ "${1:-}" ]; then
+        START_NUM=$1
+    fi
     END_NUM=999
 else
-    JOB_NAME=$1
-
-    # parse text prefix at the beginning of string 'some_your_text_NUM'
-    # (will be 'some_your_text' without last '_')
-    JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' )
-    [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad"
-
-    # parse number 'NUM' at the end of string 'some_your_text_NUM'
-    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_([0-9]+)$/\1/p' )
-    [ -z ${JOB_NUM} ] && die "JOB_NUM is bad"
-
+    JOB_NUM=${CI_NODE_INDEX}
     # count number of the jobs
-    NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${ADF_PATH}/.gitlab-ci.yml" )
-    [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad"
+    NUM_OF_JOBS=${CI_NODE_TOTAL}
 
     # count number of examples
-    NUM_OF_EXAMPLES=$( find ${ADF_PATH}/examples/ -type f -name Makefile | wc -l )
+    NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l )
     [ -z ${NUM_OF_EXAMPLES} ] && die "NUM_OF_EXAMPLES is bad"
 
+
     # separate intervals
     #57 / 5 == 12
     NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} ))
     [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
 
     # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
-    START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
+    START_NUM=$(( (${JOB_NUM} - 1) * ${NUM_OF_EX_PER_JOB} ))
     [ -z ${START_NUM} ] && die "START_NUM is bad"
 
-    END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} ))
+    END_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
     [ -z ${END_NUM} ] && die "END_NUM is bad"
 fi
 
@@ -109,45 +102,75 @@ build_example () {
     local EXAMPLE_DIR=$(dirname "${MAKE_FILE}")
     local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}")
 
+    # Check if the example needs a different base directory.
+    # Path of the Makefile relative to $IDF_PATH
+    local MAKE_FILE_REL=${MAKE_FILE#"${IDF_PATH}/"}
+    # Look for it in build_example_dirs.txt:
+    local COPY_ROOT_REL=$(sed -n -E "s|${MAKE_FILE_REL}[[:space:]]+(.*)|\1|p" < ${IDF_PATH}/tools/ci/build_example_dirs.txt)
+    if [[ -n "${COPY_ROOT_REL}" && -d "${IDF_PATH}/${COPY_ROOT_REL}/" ]]; then
+        local COPY_ROOT=${IDF_PATH}/${COPY_ROOT_REL}
+    else
+        local COPY_ROOT=${EXAMPLE_DIR}
+    fi
+
     echo "Building ${EXAMPLE_NAME} as ${ID}..."
     mkdir -p "example_builds/${ID}"
-    cp -r "${EXAMPLE_DIR}" "example_builds/${ID}"
-    pushd "example_builds/${ID}/${EXAMPLE_NAME}"
+    cp -r "${COPY_ROOT}" "example_builds/${ID}"
+    local COPY_ROOT_PARENT=$(dirname ${COPY_ROOT})
+    local EXAMPLE_DIR_REL=${EXAMPLE_DIR#"${COPY_ROOT_PARENT}"}
+    pushd "example_builds/${ID}/${EXAMPLE_DIR_REL}"
         # be stricter in the CI build than the default IDF settings
-        export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations"
-        export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
+        # export in CI script
+        # export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+        # export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+
+        # sdkconfig files are normally not checked into git, but may be present when
+        # a developer runs this script locally
+        rm -f sdkconfig
+
+        # If sdkconfig.ci file is present, append it to sdkconfig.defaults,
+        # replacing environment variables
+        if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then
+            cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults
+        fi
 
         # build non-verbose first
         local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt
         touch ${BUILDLOG}
 
+        local FLASH_ARGS=build/download.config
+
         make clean >>${BUILDLOG} 2>&1 &&
         make defconfig >>${BUILDLOG} 2>&1 &&
         make all >>${BUILDLOG} 2>&1 &&
-        ( make print_flash_cmd | tail -n 1 >build/download.config ) >>${BUILDLOG} 2>&1 ||
+        make print_flash_cmd >${FLASH_ARGS}.full 2>>${BUILDLOG} ||
         {
             RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ;
         }
 
+        tail -n 1 ${FLASH_ARGS}.full > ${FLASH_ARGS} || :
+        test -s ${FLASH_ARGS} || die "Error: ${FLASH_ARGS} file is empty"
+
         cat ${BUILDLOG}
     popd
 
-    grep -i "error\|warning" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || :
+    grep -i "error\|warning\|command not found" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || :
 }
 
 EXAMPLE_NUM=0
 
-find ${ADF_PATH}/examples/ -type f -name Makefile | sort | \
-while read FN
+echo "Current job will build example ${START_NUM} - ${END_NUM}"
+echo "Current job will build example ${EXAMPLE_PATHS}"
+
+for EXAMPLE_PATH in ${EXAMPLE_PATHS}
 do
     if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]
     then
         EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
         continue
     fi
-    echo ">>> example [ ${EXAMPLE_NUM} ] - $FN"
 
-    build_example "${EXAMPLE_NUM}" "${FN}"
+    build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}"
 
     EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
 done
@@ -156,11 +179,21 @@ done
 echo -e "\nFound issues:"
 
 #       Ignore the next messages:
-# files end with error: "error.o" or "error.c" or "error.h" or "error.d"
-# "-Werror" in compiler's command line
+# "error.o" or "-Werror" in compiler's command line
 # "reassigning to symbol" or "changes choice state" in sdkconfig
-sort -u "${LOG_SUSPECTED}" | \
-grep -v "error.[ochd]\|\ -Werror\|reassigning to symbol\|changes choice state" \
+# 'Compiler and toochain versions is not supported' from make/project.mk
+IGNORE_WARNS="\
+library/error\.o\
+\|\ -Werror\
+\|.*error.*\.o\
+\|.*error.*\.d\
+\|reassigning to symbol\
+\|changes choice state\
+\|Compiler version is not supported\
+\|Toolchain version is not supported\
+"
+
+sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \
     && RESULT=$RESULT_ISSUES \
     || echo -e "\tNone"
 
@@ -169,4 +202,4 @@ grep -v "error.[ochd]\|\ -Werror\|reassigning to symbol\|changes choice state" \
 
 echo -e "\nReturn code = $RESULT"
 
-exit $RESULT
+exit $RESULT
