deps: migrate zstd to BCR-compatible native Bazel build for bzlmod

MODULE.bazel

@@ -3,9 +3,13 @@ module(
     version = "1.37.0-dev",
 )
 
+bazel_dep(name = "aspect_bazel_lib", version = "2.21.2")
 bazel_dep(name = "envoy_api", version = "1.37.0-dev")
 bazel_dep(name = "envoy_build_config", version = "1.37.0-dev")
 bazel_dep(name = "envoy_mobile", version = "1.37.0-dev")
+bazel_dep(name = "rules_python", version = "1.6.3")
+bazel_dep(name = "rules_shell", version = "0.6.1")
+bazel_dep(name = "zstd", version = "1.5.7")
 
 local_path_override(
     module_name = "envoy_api",

bazel/external/zstd.BUILD

@@ -0,0 +1,162 @@
+""" Builds zstd.
+"""
+
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "common_sources",
+    srcs = glob([
+        "lib/common/*.c",
+        "lib/common/*.h",
+    ]),
+)
+
+filegroup(
+    name = "compress_sources",
+    srcs = glob([
+        "lib/compress/*.c",
+        "lib/compress/*.h",
+    ]),
+)
+
+filegroup(
+    name = "decompress_sources",
+    srcs = glob([
+        "lib/decompress/*.c",
+        "lib/decompress/*.h",
+    ]) + select({
+        "@platforms//os:windows": [],
+        "//conditions:default": glob(["lib/decompress/*.S"]),
+    }),
+)
+
+filegroup(
+    name = "dictbuilder_sources",
+    srcs = glob([
+        "lib/dictBuilder/*.c",
+        "lib/dictBuilder/*.h",
+    ]),
+)
+
+cc_library(
+    name = "zstd",
+    srcs = [
+        ":common_sources",
+        ":compress_sources",
+        ":decompress_sources",
+        ":dictbuilder_sources",
+    ],
+    hdrs = [
+        "lib/zdict.h",
+        "lib/zstd.h",
+        "lib/zstd_errors.h",
+    ],
+    includes = ["lib"],
+    linkopts = ["-pthread"],
+    linkstatic = True,
+    local_defines = [
+        "XXH_NAMESPACE=ZSTD_",
+        "ZSTD_MULTITHREAD",
+        "ZSTD_BUILD_SHARED=OFF",
+        "ZSTD_BUILD_STATIC=ON",
+    ] + select({
+        "@platforms//os:windows": ["ZSTD_DISABLE_ASM"],
+        "//conditions:default": [],
+    }),
+)
+
+cc_binary(
+    name = "zstd_cli",
+    srcs = glob(
+        include = [
+            "programs/*.c",
+            "programs/*.h",
+        ],
+        exclude = [
+            "programs/datagen.c",
+            "programs/datagen.h",
+            "programs/platform.h",
+            "programs/util.h",
+        ],
+    ),
+    deps = [
+        ":datagen",
+        ":util",
+        ":zstd",
+    ],
+)
+
+cc_library(
+    name = "util",
+    srcs = [
+        "programs/platform.h",
+        "programs/util.c",
+    ],
+    hdrs = [
+        "lib/common/compiler.h",
+        "lib/common/debug.h",
+        "lib/common/mem.h",
+        "lib/common/portability_macros.h",
+        "lib/common/zstd_deps.h",
+        "programs/util.h",
+    ],
+)
+
+cc_library(
+    name = "datagen",
+    srcs = [
+        "programs/datagen.c",
+        "programs/platform.h",
+    ],
+    hdrs = ["programs/datagen.h"],
+    deps = [":util"],
+)
+
+cc_binary(
+    name = "datagen_cli",
+    srcs = [
+        "programs/lorem.c",
+        "programs/lorem.h",
+        "tests/datagencli.c",
+        "tests/loremOut.c",
+        "tests/loremOut.h",
+    ],
+    includes = [
+        "programs",
+        "tests",
+    ],
+    deps = [":datagen"],
+)
+
+cc_test(
+    name = "fullbench",
+    srcs = [
+        "lib/decompress/zstd_decompress_internal.h",
+        "programs/benchfn.c",
+        "programs/benchfn.h",
+        "programs/benchzstd.c",
+        "programs/benchzstd.h",
+        "programs/lorem.c",
+        "programs/lorem.h",
+        "programs/timefn.c",
+        "programs/timefn.h",
+        "tests/fullbench.c",
+        "tests/loremOut.c",
+        "tests/loremOut.h",
+    ],
+    copts = select({
+        "@platforms//os:windows": [],
+        "//conditions:default": ["-Wno-deprecated-declarations"],
+    }),
+    includes = [
+        "lib/common",
+        "programs",
+        "tests",
+    ],
+    deps = [
+        ":datagen",
+        ":zstd",
+    ],
+)

bazel/foreign_cc/BUILD

@@ -551,42 +551,6 @@ envoy_cmake(
     }),
 )
 
-envoy_cmake(
-    name = "zstd",
-    build_args = select({
-        "//bazel/foreign_cc:parallel_builds_enabled": ["-j"],
-        "//bazel:engflow_rbe_x86_64": ["-j"],
-        "//bazel:engflow_rbe_aarch64": ["-j"],
-        "//conditions:default": ["-j1"],
-    }),
-    build_data = ["@com_github_facebook_zstd//:all"],
-    cache_entries = {
-        "CMAKE_BUILD_TYPE": "Release",
-        "CMAKE_INSTALL_LIBDIR": "lib",
-        "ZSTD_BUILD_SHARED": "off",
-        "ZSTD_BUILD_STATIC": "on",
-    },
-    exec_properties = select({
-        "//bazel:engflow_rbe_x86_64": {
-            "Pool": "linux_x64_large",
-        },
-        "//bazel:engflow_rbe_aarch64": {
-            "Pool": "linux_arm64_small",
-        },
-        "//conditions:default": {},
-    }),
-    generate_args = [
-        "-G",
-        "Ninja",
-    ],
-    lib_source = "@com_github_facebook_zstd//:all",
-    out_static_libs = select({
-        "//bazel:windows_x86_64": ["zstd_static.lib"],
-        "//conditions:default": ["libzstd.a"],
-    }),
-    working_directory = "build/cmake",
-)
-
 envoy_cmake(
     name = "maxmind",
     build_args = select({

bazel/repositories.bzl

@@ -205,7 +205,7 @@ def envoy_dependencies(skip_targets = []):
     _com_github_zlib_ng_zlib_ng()
     _org_boost()
     _org_brotli()
-    _com_github_facebook_zstd()
+    _zstd()
     _re2()
     _proxy_wasm_cpp_sdk()
     _proxy_wasm_cpp_host()
@@ -503,10 +503,10 @@ def _org_brotli():
         name = "org_brotli",
     )
 
-def _com_github_facebook_zstd():
+def _zstd():
     external_http_archive(
-        name = "com_github_facebook_zstd",
-        build_file_content = BUILD_ALL_CONTENT,
+        name = "zstd",
+        build_file = "@envoy//bazel/external:zstd.BUILD",
     )
 
 def _com_google_cel_cpp():

bazel/repository_locations.bzl

@@ -798,7 +798,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         license = "MIT",
         license_url = "https://github.com/google/brotli/blob/v{version}/LICENSE",
     ),
-    com_github_facebook_zstd = dict(
+    zstd = dict(
         project_name = "zstd",
         project_desc = "zstd compression library",
         project_url = "https://facebook.github.io/zstd",

ci/do_ci.sh

@@ -542,12 +542,12 @@ case $CI_TARGET in
         fi
 
         bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
-              //tools/zstd \
+              @zstd//:zstd_cli \
               -- --stdout \
                  -d "$ENVOY_RELEASE_TARBALL" \
             | tar xfO - envoy > distribution/custom/envoy
         bazel run "${BAZEL_BUILD_OPTIONS[@]}" \
-              //tools/zstd \
+              @zstd//:zstd_cli \
               -- --stdout \
                  -d "$ENVOY_RELEASE_TARBALL" \
             | tar xfO - envoy-contrib > distribution/custom/envoy-contrib

contrib/qat/compression/qatzstd/compressor/source/BUILD

@@ -15,10 +15,12 @@ make(
     build_data = ["@com_github_qat_zstd//:all"],
     env = select({
         "//bazel:clang_build": {
-            "CFLAGS": "-Wno-error=unused-parameter -Wno-error=unused-command-line-argument -I$$EXT_BUILD_DEPS/qatlib/include -I$$EXT_BUILD_DEPS/zstd/include",
+            "CFLAGS": "-Wno-error=unused-parameter -Wno-error=unused-command-line-argument -I$$EXT_BUILD_DEPS/qatlib/include",
+            "ZSTDLIB": "$$EXT_BUILD_ROOT/external/zstd/lib",
         },
         "//conditions:default": {
-            "CFLAGS": "-I$$EXT_BUILD_DEPS/qatlib/include -I$$EXT_BUILD_DEPS/zstd/include",
+            "CFLAGS": "-I$$EXT_BUILD_DEPS/qatlib/include",
+            "ZSTDLIB": "$$EXT_BUILD_ROOT/external/zstd/lib",
         },
     }),
     includes = [],
@@ -34,8 +36,8 @@ make(
         "install",
     ],
     deps = [
-        "//bazel/foreign_cc:zstd",
         "//contrib/qat:qatlib",
+        "@zstd",
     ],
 )
 

distribution/binary/BUILD

@@ -36,7 +36,7 @@ pkg_tar(
     srcs = [
         ":release_files",
     ],
-    compressor = "//tools/zstd",
+    compressor = "@zstd//:zstd_cli",
     compressor_args = "-T0",
     extension = "tar.zst",
 )

source/common/compression/zstd/common/BUILD

@@ -13,7 +13,7 @@ envoy_cc_library(
     srcs = ["base.cc"],
     hdrs = ["base.h"],
     deps = [
-        "//bazel/foreign_cc:zstd",
         "//source/common/buffer:buffer_lib",
+        "@zstd",
     ],
 )

source/extensions/compression/zstd/common/BUILD

@@ -12,9 +12,9 @@ envoy_cc_library(
     name = "zstd_dictionary_manager_lib",
     hdrs = ["dictionary_manager.h"],
     deps = [
-        "//bazel/foreign_cc:zstd",
         "//envoy/event:dispatcher_interface",
         "//envoy/thread_local:thread_local_interface",
         "//source/common/config:datasource_lib",
+        "@zstd",
     ],
 )

test/run_envoy_bazel_coverage.sh

@@ -97,7 +97,7 @@ unpack_coverage_report() {
     mkdir -p "${COVERAGE_DIR}"
     rm -f bazel-out/_coverage/_coverage_report.tar.zst
     mv bazel-out/_coverage/_coverage_report.dat bazel-out/_coverage/_coverage_report.tar.zst
-    bazel run "${BAZEL_BUILD_OPTIONS[@]}" --nobuild_tests_only @envoy//tools/zstd -- -d -c "${PWD}/bazel-out/_coverage/_coverage_report.tar.zst" \
+    bazel run "${BAZEL_BUILD_OPTIONS[@]}" --nobuild_tests_only @zstd//:zstd_cli -- -d -c "${PWD}/bazel-out/_coverage/_coverage_report.tar.zst" \
         | tar -xf - -C "${COVERAGE_DIR}"
     COVERAGE_JSON="${COVERAGE_DIR}/coverage.json"
 }

tools/coverage/BUILD

@@ -76,8 +76,8 @@ sh_binary(
         ":templates/base.html",
         ":templates/index.html",
         ":templates/macros.html",
-        "//tools/zstd",
         "@jq_toolchains//:resolved_toolchain",
+        "@zstd//:zstd_cli",
     ],
     toolchains = ["@jq_toolchains//:resolved_toolchain"],
     visibility = ["//visibility:public"],

tools/coverage/report_generator.sh.template

@@ -22,14 +22,19 @@ done
 find_file() {
     # kinda wierd to use PYTHON_RUNFILES but that is what is available in the
     # bazel generator customization
-    find "${PYTHON_RUNFILES}" -type f -o -type l -name "${1}" -path "*/tools/coverage/**/*" | head -1
+    find "${PYTHON_RUNFILES}" \( -type f -o -type l \) -name "${1}" -path "*/tools/coverage/**/*" | head -1
+}
+
+find_external_file() {
+    # Find files in external repositories (not under tools/coverage)
+    find "${PYTHON_RUNFILES}" \( -type f -o -type l \) -name "${1}" | head -1
 }
 
 GRCOV="$(find_file grcov_bin)"
 GRCOV_CONFIG="$(find_file grcov_config.json)"
 COVERAGE_CONFIG="$(find_file "*coverage_config.json")"
 JQ_BIN="$(find_file jq)"
-ZSTD_BIN="$(find_file zstd)"
+ZSTD_BIN="$(find_external_file zstd_cli)"
 TEMPLATES_DIR="$(dirname "$(find_file base.html)")"
 JQ_FILTER="$(find_file "filter.jq")"
 INFO_FILES=()

tools/tarball/BUILD

@@ -4,5 +4,5 @@ licenses(["notice"])  # Apache 2
 
 unpacker(
     name = "unpack",
-    zstd = "//tools/zstd",
+    zstd = "@zstd//:zstd_cli",
 )