Merge branch 'fluent:master' into out-es-add-cloud-apikey

Author: soedar

.github/workflows/build-legacy-branch.yaml

@@ -80,7 +80,7 @@ jobs:
 
       - name: Build the legacy x86_64 debug image
         if: matrix.arch == 'amd64'
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./Dockerfile.x86_64.debug
           context: .
@@ -102,7 +102,7 @@ jobs:
             raw,${{ matrix.suffix }}-${{ inputs.ref }}
 
       - name: Build the legacy ${{ matrix.arch }} image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./Dockerfile.${{ matrix.suffix }}
           context: .

.github/workflows/call-build-images.yaml

@@ -114,7 +114,7 @@ jobs:
 
       - name: Build the production images
         id: build_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./dockerfiles/Dockerfile
           context: .
@@ -141,7 +141,7 @@ jobs:
 
       - name: Build the debug multi-arch images
         id: debug_build_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./dockerfiles/Dockerfile
           context: .
@@ -314,7 +314,7 @@ jobs:
           docker push ${{ inputs.registry }}/${{ inputs.image }}:windows-${{ matrix.windows-base-version }}-${{ inputs.version }}
 
         # We cannot use this action as it requires privileged mode
-        # uses: docker/build-push-action@v5
+        # uses: docker/build-push-action@v6
         # with:
         #   file: ./dockerfiles/Dockerfile.windows
         #   context: .

.github/workflows/call-integration-image-build.yaml

@@ -62,7 +62,7 @@ jobs:
             raw,${{ inputs.image-tag }}
 
       - name: Build the AMD64 image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./dockerfiles/Dockerfile
           context: .
@@ -83,7 +83,7 @@ jobs:
             raw,${{ inputs.image-tag }}-debug
 
       - name: Build the AMD64 debug image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./dockerfiles/Dockerfile
           context: .

.github/workflows/pr-compile-check.yaml

@@ -22,7 +22,7 @@ jobs:
       uses: docker/setup-buildx-action@v3
 
     - name: Attempt to build current source for CentOS 7
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
         context: .
         file: ./dockerfiles/Dockerfile.centos7

.github/workflows/pr-image-tests.yaml

@@ -36,7 +36,7 @@ jobs:
 
         - name: Build the multi-arch images
           id: build
-          uses: docker/build-push-action@v5
+          uses: docker/build-push-action@v6
           with:
             file: ./dockerfiles/Dockerfile
             context: .
@@ -55,7 +55,7 @@ jobs:
           shell: bash
 
         - name: Build the debug multi-arch images
-          uses: docker/build-push-action@v5
+          uses: docker/build-push-action@v6
           with:
             file: ./dockerfiles/Dockerfile
             context: .

.github/workflows/staging-release.yaml

@@ -67,14 +67,6 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v4
 
-    # Check we can download the AppVeyor build which confirms it matches the version to release as well as being a successful build
-    - name: Get Appveyor binaries
-      run: |
-        ./packaging/appveyor-download.sh
-      shell: bash
-      env:
-        TAG: v${{ github.event.inputs.version }}
-
   staging-release-generate-package-matrix:
     name: Get package matrix
     runs-on: ubuntu-latest
@@ -356,14 +348,6 @@ jobs:
         AWS_REGION: "us-east-1"
       shell: bash
 
-    - name: Get Appveyor binaries
-      run: |
-        ./packaging/appveyor-download.sh
-      shell: bash
-      env:
-        TAG: v${{ github.event.inputs.version }}
-        OUTPUT_DIR: appveyor
-
     - name: Move components from staging and setup
       run: |
         ./packaging/update-source-packages.sh
@@ -518,8 +502,8 @@ jobs:
           TAG: ${{ steps.get-tag.outputs.tag }}
 
   staging-release-images-latest-tags:
-    # Only update latest tags for 3.0 releases
-    if: startsWith(github.event.inputs.version, '3.0')
+    # Only update latest tags for 3.1 releases
+    if: startsWith(github.event.inputs.version, '3.1')
     name: Release latest Linux container images
     runs-on: ubuntu-latest
     needs:
@@ -807,21 +791,21 @@ jobs:
           target_commitish: '2.1'
           make_latest: false
 
-      - name: Release 2.2 - not latest
+      - name: Release 3.0 - not latest
         uses: softprops/action-gh-release@v2
-        if: startsWith(inputs.version, '2.2')
+        if: startsWith(inputs.version, '3.0')
         with:
           body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
           draft: false
           generate_release_notes: true
           name: "Fluent Bit ${{ inputs.version }}"
           tag_name: v${{ inputs.version }}
-          target_commitish: '2.2'
+          target_commitish: '3.0'
           make_latest: false
 
-      - name: Release 3.0 and latest
+      - name: Release 3.1 and latest
         uses: softprops/action-gh-release@v2
-        if: startsWith(inputs.version, '3.0')
+        if: startsWith(inputs.version, '3.1')
         with:
           body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
           draft: false
@@ -914,13 +898,20 @@ jobs:
           ref: 2.2
           token: ${{ secrets.GH_PA_TOKEN }}
 
-      - name: Release 3.0 and latest
+      - name: Release 3.0 - not latest
         if: startsWith(inputs.version, '3.0')
         uses: actions/checkout@v4
         with:
           repository: fluent/fluent-bit-docs
           token: ${{ secrets.GH_PA_TOKEN }}
 
+      - name: Release 3.1 and latest
+        if: startsWith(inputs.version, '3.1')
+        uses: actions/checkout@v4
+        with:
+          repository: fluent/fluent-bit-docs
+          token: ${{ secrets.GH_PA_TOKEN }}
+
       - name: Ensure we have the script we need
         run: |
           if [[ ! -f update-release-version-docs.sh ]] ; then
@@ -993,9 +984,15 @@ jobs:
         with:
           ref: 2.2
 
-      - name: Release 3.0 and latest
+      - name: Release 3.0 not latest
         if: startsWith(inputs.version, '3.0')
         uses: actions/checkout@v4
+        with:
+          ref: 3.0
+
+      - name: Release 3.1 latest
+        if: startsWith(inputs.version, '3.1')
+        uses: actions/checkout@v4
 
       # Get the new version to use
       - name: 'Get next minor version'

CMakeLists.txt

@@ -8,7 +8,7 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 # Fluent Bit Version
 set(FLB_VERSION_MAJOR  3)
 set(FLB_VERSION_MINOR  1)
-set(FLB_VERSION_PATCH  0)
+set(FLB_VERSION_PATCH  4)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -110,6 +110,8 @@ option(FLB_RELEASE             "Build with release mode (-O2 -g -DNDEBUG)"   No)
 set(FLB_IPO "ReleaseOnly" CACHE STRING "Build with interprocedural optimization")
 set_property(CACHE FLB_IPO PROPERTY STRINGS "On;Off;ReleaseOnly")
 option(FLB_SMALL               "Optimise for small size"       No)
+set(FLB_SECURITY "ReleaseOnly" CACHE STRING "Build with security optimizations")
+set_property(CACHE FLB_SECURITY PROPERTY STRINGS "On;Off;ReleaseOnly")
 option(FLB_COVERAGE            "Build with code-coverage"      No)
 option(FLB_JEMALLOC            "Build with Jemalloc support"   No)
 option(FLB_REGEX               "Build with Regex support"     Yes)
@@ -283,7 +285,7 @@ endif ()
 find_package(Git)
 # If we do not have Git or this is not a Git repo or another error this just is ignored and we have no output at runtime.
 execute_process(COMMAND
-  "${GIT_EXECUTABLE}" log -1 --format=%H
+  "${GIT_EXECUTABLE}" -c log.showSignature=false log -1 --format=%H
   WORKING_DIRECTORY "${FLB_ROOT}"
   OUTPUT_VARIABLE FLB_GIT_HASH
   ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -332,6 +334,21 @@ if(FLB_IPO STREQUAL "On" OR (FLB_IPO STREQUAL "ReleaseOnly" AND FLB_RELEASE))
   endif()
 endif()
 
+# Harden release binary against security vulnerabilities
+if(FLB_SECURITY STREQUAL "On" OR (FLB_SECURITY STREQUAL "ReleaseOnly" AND FLB_RELEASE))
+  if (NOT MSVC)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,relro,-z,now")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,noexecstack")
+    if(NOT FLB_SMALL)
+      set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector")
+      # Fortify requires optimization
+      if(CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FORTIFY_SOURCE=1")
+      endif()
+    endif()
+  endif()
+endif()
+
 if(FLB_PARSER)
   FLB_DEFINITION(FLB_HAVE_PARSER)
   message(STATUS "Enabling FLB_REGEX since FLB_PARSER requires")

cmake/libraries.cmake

@@ -10,7 +10,7 @@ set(FLB_PATH_LIB_MSGPACK     "lib/msgpack-c")
 set(FLB_PATH_LIB_NGHTTP2     "lib/nghttp2")
 set(FLB_PATH_LIB_AVRO        "lib/avro")
 set(FLB_PATH_LIB_CHUNKIO     "lib/chunkio")
-set(FLB_PATH_LIB_LUAJIT      "lib/luajit-3065c9")
+set(FLB_PATH_LIB_LUAJIT      "lib/luajit-04dca791")
 set(FLB_PATH_LIB_MONKEY      "lib/monkey")
 set(FLB_PATH_LIB_JSMN        "lib/jsmn")
 set(FLB_PATH_LIB_SQLITE      "lib/sqlite-amalgamation-3450200")
@@ -19,8 +19,8 @@ set(FLB_PATH_LIB_ONIGMO      "lib/onigmo")
 set(FLB_PATH_LIB_MPACK       "lib/mpack-amalgamation-1.1.1")
 set(FLB_PATH_LIB_MINIZ       "lib/miniz")
 set(FLB_PATH_LIB_TUTF8E      "lib/tutf8e")
-set(FLB_PATH_LIB_CARES       "lib/c-ares-1.24.0")
+set(FLB_PATH_LIB_CARES       "lib/c-ares-1.32.0")
 set(FLB_PATH_LIB_SNAPPY      "lib/snappy-fef67ac")
-set(FLB_PATH_LIB_RDKAFKA     "lib/librdkafka-2.3.0")
+set(FLB_PATH_LIB_RDKAFKA     "lib/librdkafka-2.4.0")
 set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb")
 set(FLB_PATH_LIB_WASM_MICRO_RUNTIME  "lib/wasm-micro-runtime-WAMR-1.3.0")

cmake/luajit.cmake

@@ -1,4 +1,5 @@
 # luajit cmake
 option(LUAJIT_DIR "Path of LuaJIT 2.1 source dir" ON)
+option(LUAJIT_SETUP_INCLUDE_DIR "Setup include dir if parent is present" OFF)
 set(LUAJIT_DIR ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_LUAJIT})
 add_subdirectory("lib/luajit-cmake")

cmake/plugins_options.cmake

@@ -47,6 +47,7 @@ DEFINE_OPTION(FLB_IN_PROMETHEUS_SCRAPE        "Enable Prometheus Scrape input pl
 DEFINE_OPTION(FLB_IN_RANDOM                   "Enable random input plugin"                   ON)
 DEFINE_OPTION(FLB_IN_SERIAL                   "Enable Serial input plugin"                   ON)
 DEFINE_OPTION(FLB_IN_SPLUNK                   "Enable Splunk HTTP HEC input plugin"          ON)
+DEFINE_OPTION(FLB_IN_STATSD                   "Enable StatsD input plugin"                   ON)
 DEFINE_OPTION(FLB_IN_STDIN                    "Enable Standard input plugin"                 ON)
 DEFINE_OPTION(FLB_IN_STORAGE_BACKLOG          "Enable storage backlog input plugin"          ON)
 DEFINE_OPTION(FLB_IN_SYSLOG                   "Enable Syslog input plugin"                   ON)
@@ -67,7 +68,7 @@ DEFINE_OPTION(FLB_PROCESSOR_CONTENT_MODIFIER  "Enable content modifier processor
 DEFINE_OPTION(FLB_PROCESSOR_LABELS            "Enable metrics label manipulation processor"  ON)
 DEFINE_OPTION(FLB_PROCESSOR_METRICS_SELECTOR  "Enable metrics selector processor"            ON)
 DEFINE_OPTION(FLB_PROCESSOR_SQL               "Enable SQL processor"                         ON)
-
+DEFINE_OPTION(FLB_PROCESSOR_OPENTELEMETRY_ENVELOPE "Enable OpenTelemetry envelope processor"  ON)
 
 # Filters
 # =======
@@ -87,12 +88,14 @@ DEFINE_OPTION(FLB_FILTER_MULTILINE            "Enable multiline filter"
 DEFINE_OPTION(FLB_FILTER_NEST                 "Enable nest filter"                           ON)
 DEFINE_OPTION(FLB_FILTER_NIGHTFALL            "Enable Nightfall filter"                      ON)
 DEFINE_OPTION(FLB_FILTER_PARSER               "Enable parser filter"                         ON)
+DEFINE_OPTION(FLB_FILTER_RECORD_MODIFIER      "Enable record_modifier filter"                ON)
 DEFINE_OPTION(FLB_FILTER_REWRITE_TAG          "Enable tag rewrite filter"                    ON)
 DEFINE_OPTION(FLB_FILTER_STDOUT               "Enable stdout filter"                         ON)
 DEFINE_OPTION(FLB_FILTER_SYSINFO              "Enable sysinfo filter"                        ON)
 DEFINE_OPTION(FLB_FILTER_THROTTLE             "Enable throttle filter"                       ON)
 DEFINE_OPTION(FLB_FILTER_THROTTLE_SIZE        "Enable throttle size filter"                  OFF)
 DEFINE_OPTION(FLB_FILTER_TYPE_CONVERTER       "Enable type converter filter"                 ON)
+DEFINE_OPTION(FLB_FILTER_TENSORFLOW           "Enable tensorflow filter"                     OFF)
 DEFINE_OPTION(FLB_FILTER_WASM                 "Enable WASM filter"                           ON)
 
 # Outputs (destinations)
@@ -119,8 +122,11 @@ DEFINE_OPTION(FLB_OUT_KAFKA                   "Enable Kafka output plugin"
 DEFINE_OPTION(FLB_OUT_KAFKA_REST              "Enable Kafka Rest output plugin"              ON)
 DEFINE_OPTION(FLB_OUT_KINESIS_FIREHOSE        "Enable AWS Firehose output plugin"            ON)
 DEFINE_OPTION(FLB_OUT_KINESIS_STREAMS         "Enable AWS Kinesis output plugin"             ON)
+DEFINE_OPTION(FLB_OUT_LIB                     "Enable library mode output plugin"            ON)
 DEFINE_OPTION(FLB_OUT_LOGDNA                  "Enable LogDNA output plugin"                  ON)
 DEFINE_OPTION(FLB_OUT_LOKI                    "Enable Loki output plugin"                    ON)
+DEFINE_OPTION(FLB_OUT_NATS                    "Enable NATS output plugin"                    ON)
+DEFINE_OPTION(FLB_OUT_NRLOGS                  "Enable New Relic output plugin"               ON)
 DEFINE_OPTION(FLB_OUT_NULL                    "Enable dev null output plugin"                ON)
 DEFINE_OPTION(FLB_OUT_OPENSEARCH              "Enable OpenSearch output plugin"              ON)
 DEFINE_OPTION(FLB_OUT_OPENTELEMETRY           "Enable OpenTelemetry plugin"                  ON)