Topic/christian/release 6.0 ci updates

.cirrus.yml

@@ -11,11 +11,11 @@ btest_retries: &BTEST_RETRIES 2
 memory: &MEMORY 16GB
 
 config: &CONFIG --build-type=release --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install --ccache
+no_spicy_config: &NO_SPICY_CONFIG --build-type=release --disable-broker-tests --disable-spicy --prefix=$CIRRUS_WORKING_DIR/install --ccache
 static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install --ccache
 asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage --disable-spicy --ccache
 ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers --disable-spicy --ccache
 tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers --disable-spicy --ccache
-openssl30_config: &OPENSSL30_CONFIG --build-type=release --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install --ccache
 
 resources_template: &RESOURCES_TEMPLATE
   cpu: *CPUS
@@ -44,13 +44,6 @@ freebsd_environment: &FREEBSD_ENVIRONMENT
     ZEEK_CI_CPUS: 8
     ZEEK_CI_BTEST_JOBS: 8
 
-sanitizers_resource_template: &SANITIZERS_RESOURCE_TEMPLATE
-  cpu: 4
-  # Sanitizers uses a lot more memory than a typical config.
-  memory: 12GB
-  # For greediness, see https://medium.com/cirruslabs/introducing-greedy-container-instances-29aad06dc2b4
-  greedy: true
-
 builds_only_if_template: &BUILDS_ONLY_IF_TEMPLATE
   # Rules for skipping builds:
   # - Do not run builds for anything that's cron triggered
@@ -96,6 +89,7 @@ ci_template: &CI_TEMPLATE
 
   init_external_repos_script: ./ci/init-external-repos.sh
 
+  pre_build_script: ./ci/pre-build.sh
   build_script: ./ci/build.sh
   test_script: ./ci/test.sh
 
@@ -159,23 +153,20 @@ env:
   # a solution for the mtime pruning above.
   ZEEK_CCACHE_EPOCH: 2
 
-  # Cache Spicy JIT results.
-  HILTI_CXX_COMPILER_LAUNCHER: ccache
-
 # Linux EOL timelines: https://linuxlifecycle.com/
 # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle
 
-fedora38_task:
+fedora39_task:
   container:
-    # Fedora 38 EOL: Around May 2024
-    dockerfile: ci/fedora-38/Dockerfile
+    # Fedora 39 EOL: Around Nov 2024
+    dockerfile: ci/fedora-39/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
-fedora37_task:
+fedora38_task:
   container:
-    # Fedora 37 EOL: Around Dec 2024
-    dockerfile: ci/fedora-37/Dockerfile
+    # Fedora 38 EOL: Around May 2024
+    dockerfile: ci/fedora-38/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
   << : *SKIP_TASK_ON_PR
@@ -202,39 +193,43 @@ centos7_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
   << : *SKIP_TASK_ON_PR
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG
 
 debian12_task:
   container:
-    # Debian 12 (bookworm) EOL: (not yet released)
+    # Debian 12 (bookworm) EOL: TBD
     dockerfile: ci/debian-12/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
 
-debian11_task:
-  container:
-    # Debian 11 EOL: June 2026
-    dockerfile: ci/debian-11/Dockerfile
+arm_debian12_task:
+  arm_container:
+    # Debian 12 (bookworm) EOL: TBD
+    dockerfile: ci/debian-12/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG
 
-arm_debian11_task:
-  arm_container:
-    # Debian 11 EOL: June 2026
-    dockerfile: ci/debian-11/Dockerfile
+debian12_static_task:
+  container:
+    # Just use a recent/common distro to run a static compile test.
+    # Debian 12 (bookworm) EOL: TBD
+    dockerfile: ci/debian-12/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
+  << : *SKIP_TASK_ON_PR
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG
 
-debian11_static_task:
+debian11_task:
   container:
-    # Just use a recent/common distro to run a static compile test.
     # Debian 11 EOL: June 2026
     dockerfile: ci/debian-11/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
   << : *SKIP_TASK_ON_PR
-  env:
-    ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG
 
 debian10_task:
   container:
@@ -243,10 +238,10 @@ debian10_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
-opensuse_leap_15_4_task:
+opensuse_leap_15_5_task:
   container:
-    # Opensuse Leap 15.4 EOL: ~Nov 2023
-    dockerfile: ci/opensuse-leap-15.4/Dockerfile
+    # Opensuse Leap 15.5 EOL: ~Dec 2024
+    dockerfile: ci/opensuse-leap-15.5/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
@@ -259,6 +254,13 @@ opensuse_tumbleweed_task:
   << : *CI_TEMPLATE
 #  << : *SKIP_TASK_ON_PR
 
+ubuntu23_task:
+  container:
+    # Ubuntu 23.10 EOL: July 2024
+    dockerfile: ci/ubuntu-23.10/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+
 ubuntu22_task:
   container:
     # Ubuntu 22.04 EOL: April 2027
@@ -298,28 +300,25 @@ alpine_task:
 
 # Apple doesn't publish official long-term support timelines.
 # We aim to support both the current and previous macOS release.
-macos_ventura_task:
+macos_sonoma_task:
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-base:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-base:latest
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   << : *MACOS_ENVIRONMENT
 
-macos_monterey_task:
+macos_ventura_task:
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-monterey-base:latest
+    image: ghcr.io/cirruslabs/macos-ventura-base:latest
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   << : *MACOS_ENVIRONMENT
-  << : *SKIP_TASK_ON_PR
 
 # FreeBSD EOL timelines: https://www.freebsd.org/security/#sup
 freebsd14_task:
   freebsd_instance:
-    # We don't support FreeBSD 14 yet, this is a purely informative task
-    image_family: freebsd-14-0-snap
-    allow_failures: true
-    skip_notification: true
+    # FreeBSD 14 EOL: Nov 30 2028
+    image_family: freebsd-14-0
     << : *FREEBSD_RESOURCES_TEMPLATE
 
   prepare_script: ./ci/freebsd/prepare.sh
@@ -341,38 +340,36 @@ asan_sanitizer_task:
   container:
     # Just uses a recent/common distro to run memory error/leak checks.
     dockerfile: ci/ubuntu-20.04/Dockerfile
-    << : *SANITIZERS_RESOURCE_TEMPLATE
+    << : *RESOURCES_TEMPLATE
 
   << : *CI_TEMPLATE
   test_fuzzers_script: ./ci/test-fuzzers.sh
   coverage_script: ./ci/upload-coverage.sh
   env:
     CXXFLAGS: -DZEEK_DICT_DEBUG
     ZEEK_CI_CONFIGURE_FLAGS: *ASAN_SANITIZER_CONFIG
-    ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1
     ASAN_OPTIONS: detect_leaks=1
 
 ubsan_sanitizer_task:
   container:
     # Just uses a recent/common distro to run undefined behavior checks.
     dockerfile: ci/ubuntu-20.04/Dockerfile
-    << : *SANITIZERS_RESOURCE_TEMPLATE
+    << : *RESOURCES_TEMPLATE
 
   << : *CI_TEMPLATE
   << : *SKIP_TASK_ON_PR
   test_fuzzers_script: ./ci/test-fuzzers.sh
   env:
     CXXFLAGS: -DZEEK_DICT_DEBUG
     ZEEK_CI_CONFIGURE_FLAGS: *UBSAN_SANITIZER_CONFIG
-    ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1
     ZEEK_TAILORED_UB_CHECKS: 1
     UBSAN_OPTIONS: print_stacktrace=1
 
 # tsan_sanitizer_task:
 #   container:
 #     # Just uses a recent/common distro to run memory error/leak checks.
 #     dockerfile: ci/ubuntu-22.04/Dockerfile
-#     << : *SANITIZERS_RESOURCE_TEMPLATE
+#     << : *RESOURCES_TEMPLATE
 
 #   << : *CI_TEMPLATE
 #   << : *SKIP_TASK_ON_PR
@@ -432,6 +429,7 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE
     CIRRUS_LOG_TIMESTAMP: true
     BUILDER_IMAGE_CACHE_DIR: /tmp/builder-image-cache
     ZEEK_IMAGE_CACHE_DIR: /tmp/zeek-image-cache-${CIRRUS_ARCH}
+    BUILDKIT_PROGRESS: plain
 
   always:
     ccache_cache:
@@ -630,7 +628,7 @@ cluster_testing_docker_builder:
   test_script:
     # Invoke btest directly here. This mirrors ci/test.sh, ensures we don't
     # accidentally build a Docker image, and enables console-level output:
-    - cd testing/external/zeek-testing-cluster && ../../../auxil/btest/btest -d -b -j ${ZEEK_CI_BTEST_JOBS}
+    - cd testing/external/zeek-testing-cluster && ../../../auxil/btest/btest -A -d -b -j ${ZEEK_CI_BTEST_JOBS}
   on_failure:
     upload_cluster_testing_artifacts:
       path: "testing/external/zeek-testing-cluster/.tmp/**"

CMakeLists.txt

@@ -525,7 +525,16 @@ file(
     "setenv HILTI_CXX_INCLUDE_DIRS \"`${cmake_binary_dir}/hilti-cxx-include-dirs`\"\n"
     "setenv ZEEK_SPICY_LIBRARY_PATH \"${cmake_source_dir}/scripts/spicy\"\n")
 
+if (CMAKE_CXX_COMPILER_LAUNCHER)
+    file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh
+         "export HILTI_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}\n")
+
+    file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh
+         "setenv HILTI_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}\n")
+endif ()
+
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
+
 execute_process(
     COMMAND grep "^constexpr int PLUGIN_API_VERSION"
     INPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/src/plugin/Plugin.h

README

@@ -1,3 +1,4 @@
+
 =================================
 The Zeek Network Security Monitor
 =================================
@@ -101,4 +102,4 @@ others.
 [4] https://www.zeek.org/community/index.html
 [5] https://clang.llvm.org/extra/clang-tidy/
 [6] https://scan.coverity.com/projects/bro
-[7] https://pvs-studio.com/en/pvs-studio/?utm_source=github&utm_medium=organic&utm_campaign=open_source
+[7] https://pvs-studio.com/en/pvs-studio/?utm_source=github&utm_medium=organic&utm_campaign=open_source

ci/fedora-37/Dockerfile → ci/fedora-39/Dockerfile

@@ -1,8 +1,8 @@
-FROM fedora:37
+FROM fedora:39
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20230816
+ENV DOCKERFILE_VERSION 20231208
 
 RUN dnf -y install \
     bison \

ci/init-external-repos.sh

@@ -51,8 +51,8 @@ if [[ -n "${CIRRUS_CI}" ]] && [[ "${CIRRUS_REPO_OWNER}" == "zeek" ]] && [[ ! -d
 
     banner "Trying to clone zeek-testing-private git repo"
     echo "${ZEEK_TESTING_PRIVATE_SSH_KEY}" >cirrus_key.b64
-    if [ "${CIRRUS_TASK_NAME}" == "macos_ventura" ]; then
-        # The base64 command provided with macOS Ventura requires an argument
+    if [ "${CIRRUS_TASK_NAME}" == "macos_ventura" -o "${CIRRUS_TASK_NAME}" == "macos_sonoma" ]; then
+        # The base64 command provided with macOS Ventura/Sonoma requires an argument
         # to pass the input filename
         base64 -d -i cirrus_key.b64 >cirrus_key
     else

ci/macos/prepare.sh

@@ -7,7 +7,7 @@ set -x
 
 brew update
 brew upgrade cmake
-brew install openssl@3 swig bison flex ccache
+brew install openssl@3 swig bison flex ccache libmaxminddb
 python3 -m pip install --user websockets
 
 # Brew doesn't create the /opt/homebrew/opt/openssl symlink if you install

ci/opensuse-leap-15.4/Dockerfile → ci/opensuse-leap-15.5/Dockerfile

@@ -1,34 +1,39 @@
-FROM opensuse/leap:15.4
+FROM opensuse/leap:15.5
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20230816
+ENV DOCKERFILE_VERSION 20230905
 
-RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.4:Update/standard/openSUSE:Leap:15.4:Update.repo \
+RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.5:Update/standard/openSUSE:Leap:15.5:Update.repo \
  && zypper refresh \
  && zypper in -y \
     bison \
     ccache \
     cmake \
     curl \
     flex \
-    gcc10 \
-    gcc10-c++ \
+    gcc12 \
+    gcc12-c++ \
     git \
     gzip \
     libopenssl-devel \
     libpcap-devel \
     make \
-    python3 \
-    python3-devel \
-    python3-pip \
+    openssh \
+    python311 \
+    python311-devel \
+    python311-pip \
     swig \
     tar \
     which \
     zlib-devel \
   && rm -rf /var/cache/zypp
 
+RUN update-alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.11 100
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 100
+RUN update-alternatives --install /usr/bin/python3-config python3-config /usr/bin/python3.11-config 100
+
 RUN pip3 install websockets junit2html
 
-RUN update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-10 100
-RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-10 100
+RUN update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100
+RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100

ci/opensuse-tumbleweed/Dockerfile

@@ -25,14 +25,16 @@ RUN zypper refresh \
     libopenssl-devel \
     libpcap-devel \
     make \
+    openssh \
     python3 \
     python3-devel \
     python3-pip \
+    python3-websockets \
     swig \
     tar \
     util-linux \
     which \
     zlib-devel \
   && rm -rf /var/cache/zypp
 
-RUN pip3 install --break-system-packages websockets junit2html
+RUN pip3 install --break-system-packages junit2html

ci/pre-build.sh

@@ -0,0 +1,8 @@
+#/usr/bin/env bash
+#
+# If ZEEK_CI_PREBUILD_COMMAND is not empty, execute it via bash -c. That's it.
+set -ex
+
+if [ -n "$ZEEK_CI_PREBUILD_COMMAND" ]; then
+    bash -c "$ZEEK_CI_PREBUILD_COMMAND"
+fi