feat: add --output_json to s3_sync()

MODULE.bazel

@@ -20,6 +20,16 @@ bazel_dep(name = "aspect_rules_py", version = "0.7.3", dev_dependency = True)
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 bazel_dep(name = "container_structure_test", version = "1.16.0", dev_dependency = True)
 
+bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
+bazel_lib_toolchains.jq()
+bazel_lib_toolchains.coreutils()
+use_repo(bazel_lib_toolchains, "coreutils_toolchains", "jq_toolchains")
+
+register_toolchains(
+    "@jq_toolchains//:all",
+    "@coreutils_toolchains//:all",
+)
+
 aws = use_extension("//aws:extensions.bzl", "aws")
 aws.toolchain(aws_cli_version = "2.13.0")
 use_repo(aws, "aws", "aws_darwin", "aws_linux-aarch64", "aws_linux-x86_64", "aws_toolchains")

aws/private/s3_sync.bzl

@@ -38,8 +38,11 @@ _ATTRS = {
 }
 
 def _s3_sync_impl(ctx):
+    coreutils = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"]
+    jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"]
+
     executable = ctx.actions.declare_file("{}/s3_sync.sh".format(ctx.label.name))
-    runfiles = [executable] + ctx.files.srcs
+    runfiles = [executable, coreutils.coreutils_info.bin, jq.jqinfo.bin] + ctx.files.srcs
     vars = []
     if int(bool(ctx.attr.bucket)) + int(bool(ctx.attr.bucket_file)) + int(bool(ctx.attr.destination_uri_file)) != 1:
         fail("Exactly one of 'bucket', 'bucket_file', or 'destination_uri_file' must be set")
@@ -61,6 +64,8 @@ def _s3_sync_impl(ctx):
         is_executable = True,
         substitutions = {
             "$aws": ctx.attr.aws[DefaultInfo].default_runfiles.files.to_list()[0].short_path,
+            "$coreutils": coreutils.coreutils_info.bin.short_path,
+            "$jq": jq.jqinfo.bin.short_path,
             "artifacts=()": "artifacts=({})".format(" ".join([s.short_path for s in ctx.files.srcs])),
             "# Collect Args": "\n".join(vars),
         },
@@ -76,4 +81,8 @@ s3_sync = rule(
     executable = True,
     attrs = _ATTRS,
     doc = _DOC,
+    toolchains = [
+        "@aspect_bazel_lib//lib:coreutils_toolchain_type",
+        "@aspect_bazel_lib//lib:jq_toolchain_type",
+    ],
 )

aws/private/s3_sync.sh

@@ -74,6 +74,7 @@ Options:
   --bucket_file <file>    The path to a file that contains the name of the S3 bucket.
   --[no]dry_run           Toggles whether the utility will run in dry-run mode.
                           Default: false
+  --output_json <file>    Collect SHA256 sums/S3 destination for every file and write as JSON to the specified file.
 
 Arguments:
   <artifact>              The path to a file or directory which will be copied to the S3 bucket.
@@ -91,6 +92,11 @@ s3_cp() {
     else
         warn "[DRY RUN] Would copy ${src} to ${dst}"
     fi
+
+    if [[ -n "${output_json_file}" ]]; then
+        local sha256_sum=$("$coreutils" sha256sum "${src}" | cut -d' ' -f1)
+        sha256_results+=('{"file":"'"${src}"'","sha256":"'"${sha256_sum}"'","s3_path":"'"${dst}"'"}')
+    fi
 }
 
 cp_artifact() {
@@ -107,10 +113,18 @@ cp_artifact() {
     fi
 }
 
+output_json_results() {
+    local output_file="${1}"
+    printf '%s\n' "${sha256_results[@]}" | "$jq" -s . > "${output_file}"
+    msg "JSON output written to ${output_file}"
+}
+
 # Collect Args
 
 dry_run=false
+output_json_file=""
 artifacts=()
+sha256_results=()
 
 while (("$#")); do
     case "${1}" in
@@ -137,6 +151,10 @@ while (("$#")); do
         dry_run="false"
         shift 1
         ;;
+    "--output_json")
+        output_json_file="${2}"
+        shift 2
+        ;;
     "--role")
         role="${2}"
         shift 2
@@ -213,6 +231,10 @@ else
     done
 fi
 
+if [[ -n "${output_json_file}" ]]; then
+    output_json_results "${output_json_file}"
+fi
+
 # shellcheck disable=SC2236
 if [[ ! -z "${role:-}" && "${dry_run}" == "false" ]]; then
     unset AWS_ACCESS_KEY_ID

examples/release_to_s3/BUILD.bazel

@@ -18,6 +18,8 @@ expand_template(
 #  bazel run //examples/release_to_s3 -- --dry_run
 # Use a different profile:
 #  bazel run //examples/release_to_s3 -- --profile=prod
+# Output a metadata JSON file:
+#  bazel run //examples/release_to_s3 -- --output_json $PWD/out.json
 s3_sync(
     name = "release_to_s3",
     srcs = ["my_file.txt"],