Merge pull request #16616 from github/redsun82/fix-pkg

Reinstate bazel packaging library with a backward compatibility fix

Author: criemen

.bazelrc

@@ -10,15 +10,18 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
-build:linux --cxxopt=-std=c++20
+build:linux --cxxopt=-std=c++20 --host_cxxopt=-std=c++20
 # we currently cannot built the swift extractor for ARM
-build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
-build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
+build:macos --cxxopt=-std=c++20 --host_cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
+build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor --host_cxxopt=/std:c++20 --host_cxxopt=/Zc:preprocessor
 
 # this requires developer mode, but is required to have pack installer functioning
 startup --windows_enable_symlinks
 common --enable_runfiles
 
+# with the above, we can avoid building python zips which is the default on windows as that's expensive
+build --nobuild_python_zip
+
 common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 

.gitattributes

@@ -80,3 +80,9 @@ csharp/paket.lock linguist-generated=true
 csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
 csharp/paket.main.bzl linguist-generated=true
 csharp/paket.main_extension.bzl linguist-generated=true
+
+# ripunzip tool
+/misc/bazel/internal/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text
+
+# swift prebuilt resources
+/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text

.github/workflows/zipmerge-test.yml

@@ -0,0 +1,23 @@
+name: "Test zipmerge code"
+
+on:
+  pull_request:
+    paths:
+      - "misc/bazel/internal/zipmerge/**"
+      - "MODULE.bazel"
+      - ".bazelrc*"
+    branches:
+      - main
+      - "rc/*"
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          bazel test //misc/bazel/internal/zipmerge:test --test_output=all

MODULE.bazel

@@ -24,6 +24,7 @@ bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
 bazel_dep(name = "gazelle", version = "0.36.0")
 bazel_dep(name = "rules_dotnet", version = "0.15.1")
+bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
@@ -67,6 +68,36 @@ use_repo(node, "nodejs", "nodejs_toolchains")
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
 go_sdk.download(version = "1.22.2")
 
+lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files")
+
+lfs_files(
+    name = "ripunzip-linux",
+    srcs = ["//misc/bazel/internal/ripunzip:ripunzip-linux"],
+    executable = True,
+)
+
+lfs_files(
+    name = "ripunzip-windows",
+    srcs = ["//misc/bazel/internal/ripunzip:ripunzip-windows.exe"],
+    executable = True,
+)
+
+lfs_files(
+    name = "ripunzip-macos",
+    srcs = ["//misc/bazel/internal/ripunzip:ripunzip-macos"],
+    executable = True,
+)
+
+lfs_files(
+    name = "swift-resource-dir-linux",
+    srcs = ["//swift/third_party/resource-dir:resource-dir-linux.zip"],
+)
+
+lfs_files(
+    name = "swift-resource-dir-macos",
+    srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"],
+)
+
 register_toolchains(
     "@nodejs_toolchains//:all",
 )

misc/bazel/internal/BUILD.bazel

@@ -0,0 +1 @@
+exports_files(["install.py"])

misc/bazel/internal/install.py

@@ -0,0 +1,55 @@
+"""
+Helper script for installing `codeql_pack` targets.
+
+This mainly wraps around a `pkg_install` script from `rules_pkg` adding:
+* resolving destination directory with respect to a provided `--build-file`
+* clean-up of target destination directory before a reinstall
+* installing imported zip files using a provided `--ripunzip`
+"""
+
+import argparse
+import pathlib
+import shutil
+import subprocess
+from python.runfiles import runfiles
+
+runfiles = runfiles.Create()
+assert runfiles, "Installer should be run with `bazel run`"
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument("--destdir", type=pathlib.Path, required=True,
+                    help="Desination directory, relative to `--build-file`")
+parser.add_argument("--pkg-install-script", required=True,
+                    help="The wrapped `pkg_install` installation script rlocation")
+parser.add_argument("--build-file", required=True,
+                    help="BUILD.bazel rlocation relative to which the installation should take place")
+parser.add_argument("--ripunzip",
+                    help="ripunzip executable rlocation. Must be provided if `--zip-manifest` is.")
+parser.add_argument("--zip-manifest",
+                    help="The rlocation of a file containing newline-separated `prefix:zip_file` entries")
+parser.add_argument("--cleanup", action=argparse.BooleanOptionalAction, default=True,
+                    help="Whether to wipe the destination directory before installing (true by default)")
+opts = parser.parse_args()
+if opts.zip_manifest and not opts.ripunzip:
+    parser.error("Provide `--ripunzip` when specifying `--zip-manifest`")
+
+build_file = runfiles.Rlocation(opts.build_file)
+script = runfiles.Rlocation(opts.pkg_install_script)
+destdir = pathlib.Path(build_file).resolve().parent / opts.destdir
+
+if destdir.exists() and opts.cleanup:
+    shutil.rmtree(destdir)
+
+destdir.mkdir(parents=True, exist_ok=True)
+subprocess.run([script, "--destdir", destdir], check=True)
+
+if opts.zip_manifest:
+    ripunzip = runfiles.Rlocation(opts.ripunzip)
+    zip_manifest = runfiles.Rlocation(opts.zip_manifest)
+    with open(zip_manifest) as manifest:
+        for line in manifest:
+            prefix, _, zip = line.partition(":")
+            assert zip, f"missing prefix for {prefix}, you should use prefix:zip format"
+            dest = destdir / prefix
+            dest.mkdir(parents=True, exist_ok=True)
+            subprocess.run([ripunzip, "unzip-file", zip, "-d", dest], check=True)

misc/bazel/internal/ripunzip/BUILD.bazel

@@ -0,0 +1,8 @@
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
+
+native_binary(
+    name = "ripunzip",
+    src = select({"@platforms//os:" + os: "@ripunzip-" + os for os in ("linux", "windows", "macos")}),
+    out = "ripunzip.exe",
+    visibility = ["//visibility:public"],
+)