Update CI to use colcon cache (ros-navigation#2343)

* Disable dockerhub workflow for testing

* Disable debug jobs for testing

* Disable extra rmw jobs for testing

* Rename docker images for testing

* Default to cache from main branch

* Use colcon cache

* Install colcon cache

* Fix xargs typo

* List names only

* Checksum overlay ws src after restore and store
to restore ws cache for incremental builds

* Disable test jobs

* Revert checksum update but use underlay src

* Only cache ws build and install folder
to avoide overwritting ws src folder with restore

* Don't remove git files from underlay
so vcs tools can export underlay to checksum

* Save build cache regardless of failure

* cache tests

* Update colcon cache

* No need to remove log folder
as its no longer cached

* Bypass .dockerignore for dockerhub build hooks
to include .git folder in overlay for colcon cache

* Simplify checkout
now that underlay and overload have lockfiles

* Remove overlay src folder for checkout

* Echo env for debug

* Use if checks for empty package sets
as --packages-above with empty args will not skip any package

* Save workspace log to cache

* Set TEST_PACKAGES to empty instead of skipping

* Update colcon cache

* Update colcon cache

* Explicitltly list packages to build
using an environment variable

* Enable debug and nightly jobs

* Test all packages if cache_test is false

* Revert testing fork changes

* Install pip in rolling image

* Rename checksum file to lockfile.txt

* Config formatting

* Simplify testing

Author: ruffsl

.circleci/config.yml

@@ -12,11 +12,17 @@ _commands:
       steps:
         - restore_cache:
             name: Restore Cache << parameters.key >>
-            key: "<< parameters.key >>-v1\
-              -{{ arch }}\
-              -{{ .Branch }}\
-              -{{ .Environment.CIRCLE_PR_NUMBER }}\
-              -{{ checksum  \"<< parameters.workspace >>/checksum.txt\" }}"
+            keys:
+              - "<< parameters.key >>-v1\
+                -{{ arch }}\
+                -{{ .Branch }}\
+                -{{ .Environment.CIRCLE_PR_NUMBER }}\
+                -{{ checksum  \"<< parameters.workspace >>/lockfile.txt\" }}"
+              - "<< parameters.key >>-v1\
+                -{{ arch }}\
+                -main\
+                -<no value>\
+                -{{ checksum  \"<< parameters.workspace >>/lockfile.txt\" }}"
     save_to_cache:
       description: "Save To Cache"
       parameters:
@@ -36,10 +42,12 @@ _commands:
               -{{ arch }}\
               -{{ .Branch }}\
               -{{ .Environment.CIRCLE_PR_NUMBER }}\
-              -{{ checksum  \"<< parameters.workspace >>/checksum.txt\" }}\
+              -{{ checksum  \"<< parameters.workspace >>/lockfile.txt\" }}\
               -{{ epoch }}"
             paths:
-              - << parameters.path >>
+              - << parameters.path >>/build
+              - << parameters.path >>/install
+              - << parameters.path >>/log
             when: << parameters.when >>
     install_dependencies:
       description: "Install Dependencies"
@@ -54,13 +62,13 @@ _commands:
             working_directory: << parameters.workspace >>
             command: |
               . << parameters.underlay >>/install/setup.sh
-              cat << parameters.underlay >>/checksum.txt > checksum.txt
-              vcs export --exact src | \
-                (echo vcs_export && cat) >> checksum.txt
-              sha256sum $PWD/checksum.txt >> checksum.txt
+              cat << parameters.underlay >>/lockfile.txt > lockfile.txt
+              vcs export --exact << parameters.underlay >>/src | \
+                (echo vcs_export && cat) >> lockfile.txt
+              sha256sum $PWD/lockfile.txt >> lockfile.txt
               apt-get update
               rosdep update
-              
+
               # workarround for OMPL and rosdep
               # https://github.com/ompl/ompl/issues/753
               # Prevent searching $ROS_WS/install given it's too big for rosdep
@@ -85,8 +93,8 @@ _commands:
                 awk -F'[][]' '{print $2}' | \
                 tr -d \, | xargs -n1 | sort -u | xargs)
               dpkg -s $dependencies | \
-                (echo workspace_dependencies && cat) >> checksum.txt
-              sha256sum $PWD/checksum.txt >> checksum.txt
+                (echo workspace_dependencies && cat) >> lockfile.txt
+              sha256sum $PWD/lockfile.txt >> lockfile.txt
     setup_workspace:
       description: "Setup Workspace"
       parameters:
@@ -103,7 +111,7 @@ _commands:
           type: boolean
       steps:
         - store_artifacts:
-            path: << parameters.workspace >>/checksum.txt
+            path: << parameters.workspace >>/lockfile.txt
         - restore_from_cache:
             key: << parameters.key >>
             workspace: << parameters.workspace >>
@@ -114,21 +122,55 @@ _commands:
                   name: Build Workspace | << parameters.workspace >>
                   working_directory: << parameters.workspace >>
                   command: |
-                    BUILD_UNFINISHED=$(colcon list --packages-skip-build-finished)
-                    BUILD_FAILED=$(colcon list --packages-select-build-failed)
-                    if [ -n "$BUILD_UNFINISHED" ] || [ -n "$BUILD_FAILED" ]
+                    colcon cache lock
+
+                    BUILD_UNFINISHED=$(
+                      colcon list \
+                        --names-only \
+                        --packages-skip-build-finished \
+                      | xargs)
+                    echo BUILD_UNFINISHED: $BUILD_UNFINISHED
+
+                    BUILD_FAILED=$(
+                      colcon list \
+                        --names-only \
+                        --packages-select-build-failed \
+                      | xargs)
+                    echo BUILD_FAILED: $BUILD_FAILED
+
+                    BUILD_INVALID=$(
+                      colcon list \
+                        --names-only \
+                        --packages-select-cache-invalid \
+                        --packages-select-cache-key build \
+                      | xargs)
+                    echo BUILD_INVALID: $BUILD_INVALID
+
+                    BUILD_PACKAGES=""
+                    if [ -n "$BUILD_UNFINISHED" ] || \
+                       [ -n "$BUILD_FAILED" ] || \
+                       [ -n "$BUILD_INVALID" ]
                     then
-                      . << parameters.underlay >>/install/setup.sh
-                      rm -rf build install log
-                      colcon build \
-                        --mixin << parameters.mixins >>
-                    else
-                      echo "Skipping Build"
+                      BUILD_PACKAGES=$(
+                        colcon list \
+                          --names-only \
+                          --packages-above \
+                            $BUILD_UNFINISHED \
+                            $BUILD_FAILED \
+                            $BUILD_INVALID \
+                        | xargs)
                     fi
+                    echo BUILD_PACKAGES: $BUILD_PACKAGES
+
+                    . << parameters.underlay >>/install/setup.sh
+                    colcon build \
+                      --packages-select ${BUILD_PACKAGES} \
+                      --mixin << parameters.mixins >>
               - save_to_cache:
                   key: << parameters.key >>
                   path: << parameters.workspace >>
                   workspace: << parameters.workspace >>
+                  when: always
               - run:
                   name: Copy Build Logs
                   working_directory: << parameters.workspace >>
@@ -139,28 +181,72 @@ _commands:
     test_workspace:
       description: "Test Workspace"
       parameters:
+        key:
+          type: string
         workspace:
           type: string
+        cache_test:
+          type: boolean
       steps:
         - run:
             name: Test Workspace | << parameters.workspace >>
             working_directory: << parameters.workspace >>
             command: |
-              . install/setup.sh
+              TEST_FAILURES=$(
+                colcon list \
+                  --names-only \
+                  --packages-select-test-failures \
+                | xargs)
+              echo TEST_FAILURES: $TEST_FAILURES
+
+              TEST_INVALID=$(
+                colcon list \
+                  --names-only \
+                  --packages-select-cache-invalid \
+                  --packages-select-cache-key test \
+                | xargs)
+              echo TEST_INVALID: $TEST_INVALID
+
+              TEST_PACKAGES=""
+              if [ -n "$TEST_FAILURES" ] || \
+                 [ -n "$TEST_INVALID" ]
+              then
+                TEST_PACKAGES=$(
+                  colcon list \
+                    --names-only \
+                    --packages-above \
+                      $TEST_FAILURES \
+                      $TEST_INVALID)
+              fi
+              if ( ! << parameters.cache_test >> )
+              then
+                TEST_PACKAGES=$(
+                  colcon list \
+                    --names-only)
+              fi
               TEST_PACKAGES=$(
-                colcon list --names-only | \
-                  circleci tests split \
-                    --split-by=timings \
-                    --timings-type=classname \
-                    --show-counts | \
-                  xargs)
+                echo $TEST_PACKAGES \
+                | circleci tests split \
+                  --split-by=timings \
+                  --timings-type=classname \
+                  --show-counts \
+                | xargs)
+              echo TEST_PACKAGES: $TEST_PACKAGES
+
+              . install/setup.sh
               set -o xtrace
               colcon test \
-                --packages-select ${TEST_PACKAGES} \
-                --retest-until-pass ${RETEST_UNTIL_PASS} \
-                --ctest-args --test-regex "test_.*"
+                --packages-select ${TEST_PACKAGES}
               colcon test-result \
                 --verbose
+        - when:
+            condition: << parameters.cache_test >>
+            steps:
+              - save_to_cache:
+                  key: << parameters.key >>
+                  path: << parameters.workspace >>
+                  workspace: << parameters.workspace >>
+                  when: always
         - run:
             name: Copy Test Logs
             working_directory: << parameters.workspace >>
@@ -211,16 +297,16 @@ _steps:
     run:
       name: Pre Checkout
       command: |
-        mkdir -p $ROS_WS && cd $ROS_WS
+        mkdir -p $ROS_WS/src && cd $ROS_WS
         ln -s /opt/ros/$ROS_DISTRO install
         echo $CACHE_NONCE | \
-          (echo cache_nonce && cat) >> checksum.txt
-        sha256sum $PWD/checksum.txt >> checksum.txt
+          (echo cache_nonce && cat) >> lockfile.txt
+        sha256sum $PWD/lockfile.txt >> lockfile.txt
         TZ=utc stat -c '%y' /ros_entrypoint.sh | \
-          (echo ros_entrypoint && cat) >> checksum.txt
-        sha256sum $PWD/checksum.txt >> checksum.txt
-        rm -rf $OVERLAY_WS/*
+          (echo ros_entrypoint && cat) >> lockfile.txt
+        sha256sum $PWD/lockfile.txt >> lockfile.txt
         mv ~/.ccache $CCACHE_DIR
+        rm -rf $OVERLAY_WS/src
   on_checkout: &on_checkout
     checkout:
       path: src/navigation2
@@ -233,11 +319,9 @@ _steps:
           $OVERLAY_WS/src/navigation2/tools/underlay.repos \
           $UNDERLAY_WS/underlay.repos >/dev/null 2>&1
         then
-          echo "Cleaning Underlay"
-          rm -rf $UNDERLAY_WS/*
+          echo "Importing Underlay"
           cp $OVERLAY_WS/src/navigation2/tools/underlay.repos \
             $UNDERLAY_WS/underlay.repos
-          mkdir -p $UNDERLAY_WS/src
           vcs import $UNDERLAY_WS/src \
             < $UNDERLAY_WS/underlay.repos
         fi
@@ -292,7 +376,9 @@ _steps:
       when: always
   test_overlay_workspace: &test_overlay_workspace
     test_workspace:
+      key: overlay_ws
       workspace: /opt/overlay_ws
+      cache_test: << parameters.cache_test >>
   collect_overlay_coverage: &collect_overlay_coverage
     run:
       name: Collect Code Coverage
@@ -360,6 +446,9 @@ commands:
       - *restore_overlay_workspace
   test_build:
     description: "Test Build"
+    parameters:
+      cache_test:
+        type: boolean
     steps:
       - *test_overlay_workspace
   report_coverage:
@@ -378,14 +467,14 @@ _environments:
     ROS_WS: "/opt/ros_ws"
     UNDERLAY_WS: "/opt/underlay_ws"
     OVERLAY_WS: "/opt/overlay_ws"
+    UNDERLAY_MIXINS: "release ccache"
     CCACHE_DIR: "/tmp/.ccache"
     CCACHE_LOGFILE: "/tmp/ccache.log"
     CCACHE_MAXSIZE: "200M"
     MAKEFLAGS: "-j 2 -l 2 "
     COLCON_DEFAULTS_FILE: "/tmp/defaults.yaml"
     RCUTILS_LOGGING_BUFFERED_STREAM: "0"
     RCUTILS_LOGGING_USE_STDOUT: "0"
-    RETEST_UNTIL_PASS: "2"
     DEBIAN_FRONTEND: "noninteractive"
 
 executors:
@@ -397,7 +486,6 @@ executors:
       <<: *common_environment
       CACHE_NONCE: "Debug"
       OVERLAY_MIXINS: "debug ccache coverage-gcc"
-      UNDERLAY_MIXINS: "release ccache"
   release_exec:
     docker:
       - image: rosplanning/navigation2:main.release
@@ -406,11 +494,13 @@ executors:
       <<: *common_environment
       CACHE_NONCE: "Release"
       OVERLAY_MIXINS: "release ccache"
-      UNDERLAY_MIXINS: "release ccache"
 
 _jobs:
   job_test: &job_test
     parameters:
+      cache_test:
+        type: boolean
+        default: false
       rmw:
         default: "rmw_cyclonedds_cpp"
         type: string
@@ -433,14 +523,16 @@ jobs:
     executor: debug_exec
     steps:
       - restore_build
-      - test_build
+      - test_build:
+          cache_test: << parameters.cache_test >>
       - report_coverage
   release_test:
     <<: *job_test
     executor: release_exec
     steps:
       - restore_build
-      - test_build
+      - test_build:
+          cache_test: << parameters.cache_test >>
   rebuild_dockerhub:
     executor: release_exec
     steps:
@@ -454,10 +546,12 @@ workflows:
       - debug_test:
           requires:
             - debug_build
+          cache_test: true
       - release_build
       - release_test:
           requires:
             - release_build
+          cache_test: true
   nightly:
     jobs:
       - debug_build

.dockerhub/debug/hooks/build

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -ex
+rm -rf ../../.dockerignore
 
 export FROM_IMAGE=ros:rolling
 export FAIL_ON_BUILD_FAILURE=""

.dockerhub/release/hooks/build

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -ex
+rm -rf ../../.dockerignore
 
 export FROM_IMAGE=ros:rolling
 export FAIL_ON_BUILD_FAILURE=""