[WIP] Add llvm-installer clang builds on OS X targets

.github/workflows/ci.yml

@@ -184,6 +184,24 @@ jobs:
             build_thirdparty_args: --enforce_arch=arm64
             architecture: arm64
 
+          - name: macos-x86_64-clang19
+            runs_on: macos-13
+            docker_image:
+            build_thirdparty_args: >-
+              --enforce_arch=x86_64
+              --toolchain=llvm19
+            architecture: x86_64
+
+          - name: macos-arm64-clang19
+            # According to https://github.com/orgs/community/discussions/69211, this is the runner
+            # type that corresponds to Apple Silicon.
+            runs_on: macos-13-xlarge
+            docker_image:
+            build_thirdparty_args: >-
+              --enforce_arch=arm64
+              --toolchain=llvm19
+            architecture: arm64
+
           # ---------------------------------------------------------------------------------------
           # aarch64 - almalinux8 builds
           # ---------------------------------------------------------------------------------------

.github/workflows/macos_build.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 brew install autoconf automake pkg-config shellcheck hub
-dirs=( /opt/yb-build/{thirdparty,brew,tmp} )
+dirs=( /opt/yb-build/{thirdparty,brew,tmp,llvm} )
 sudo mkdir -p "${dirs[@]}"
 sudo chmod 777 "${dirs[@]}"
 

patches/bid-crlf.patch

@@ -16,6 +16,15 @@ diff -au -ruN a/LIBRARY/makefile b/LIBRARY/makefile
 
  $(SRC_DIR)/bid_trans.h : $(SRC_DIR)/bid_internal.h
  	touch $@
+@@ -399,7 +399,7 @@
+ F128_PLATFORM_FLAGS := $(foreach n, IML_HOST_ARCH IML_HOST_OS CC_NAME \
+                         ,-D$(call ToLower,$($n)))
+
+-F128_CFLAGS := $(CFLAGS_OPT) -DUSE_NATIVE_QUAD_TYPE=0 $(F128_PLATFORM_FLAGS)
++F128_CFLAGS := $(CFLAGS) -DUSE_NATIVE_QUAD_TYPE=0 $(F128_PLATFORM_FLAGS)
+
+ F128_HDR_NAMES := bessel cons int  lgamma powi sqrt     bid  erf inv_hyper \
+                   log    pow  trig cbrt   exp  inv_trig mod
 @@ -430,7 +430,7 @@
  $(F128_HDR_OBJS) :: $(OBJ_DIR)/dpml_ux_%.$O : $(F128_DIR)/dpml_%_x.h
 

python/build_definitions/bison.py

@@ -28,10 +28,11 @@ def __init__(self) -> None:
     def get_additional_compiler_flags(self, builder: BuilderInterface) -> List[str]:
         llvm_major_version = builder.compiler_choice.get_llvm_major_version()
         flags = []
-        linux_llvm16_or_later = (
-            is_linux() and llvm_major_version is not None and llvm_major_version >= 16)
+        llvm_installer_16_or_later = (
+            builder.compiler_choice.is_llvm_installer_clang() and
+            llvm_major_version is not None and llvm_major_version >= 16)
 
-        if (is_macos() or linux_llvm16_or_later):
+        if (is_macos() or llvm_installer_16_or_later):
             # To avoid this error in Bison 3.4.1 build:
             # lib/obstack.c:351:31: error: incompatible function pointer types initializing
             # 'void (*)(void) __attribute__((noreturn))' with an expression of type 'void (void)'
@@ -41,3 +42,6 @@ def get_additional_compiler_flags(self, builder: BuilderInterface) -> List[str]:
 
     def build(self, builder: BuilderInterface) -> None:
         builder.build_with_configure(dep=self, extra_configure_args=['--with-pic'])
+
+    def use_cppflags_env_var(self) -> bool:
+        return True

python/build_definitions/cassandra_cpp_driver.py

@@ -62,7 +62,7 @@ def get_additional_cxx_flags(self, builder: 'BuilderInterface') -> List[str]:
         extra_cxx_flags: List[str] = []
         builder.add_checked_flag(extra_cxx_flags, '-Wno-error=implicit-fallthrough')
         builder.add_checked_flag(extra_cxx_flags, '-Wno-error=class-memaccess')
-        if builder.compiler_choice.is_linux_clang():
+        if builder.compiler_choice.is_llvm_installer_clang():
             builder.add_checked_flag(extra_cxx_flags, '-Wno-error=deprecated-declarations')
         gcc_major_version = builder.compiler_choice.get_gcc_major_version()
         if gcc_major_version is not None and gcc_major_version >= 11:

python/build_definitions/crcutil.py

@@ -43,3 +43,6 @@ def build(self, builder: BuilderInterface) -> None:
         log_prefix = builder.log_prefix(self)
         builder.log_output(log_prefix, ['./autogen.sh'])
         builder.build_with_configure(dep=self)
+
+    def use_cppflags_env_var(self) -> bool:
+        return True

python/build_definitions/gettext.py

@@ -23,7 +23,7 @@ def __init__(self) -> None:
             'gettext',
             '0.21',
             'https://ftp.gnu.org/pub/gnu/gettext/gettext-{0}.tar.gz',
-            BuildGroup.POTENTIALLY_INSTRUMENTED)
+            BuildGroup.COMMON)
         self.copy_sources = True
 
     def get_compiler_wrapper_ld_flags_to_remove(self, builder: BuilderInterface) -> Set[str]:
@@ -54,3 +54,6 @@ def build(self, builder: BuilderInterface) -> None:
                 '--without-cvs',
                 '--without-xz',
             ])
+
+    def use_cppflags_env_var(self) -> bool:
+        return True

python/build_definitions/icu4c.py

@@ -121,3 +121,6 @@ def build(self, builder: BuilderInterface) -> None:
         )
 
         fix_shared_library_references(builder.prefix, 'libicu')
+
+    def use_cppflags_env_var(self) -> bool:
+        return True

python/build_definitions/krb5.py

@@ -79,3 +79,6 @@ def build(self, builder: BuilderInterface) -> None:
                     extra_configure_args=extra_args,
                 )
                 log(f"Finished building krb5 with {linking_type} linking in {os.getcwd()}")
+
+    def use_cppflags_env_var(self) -> bool:
+        return True

python/build_definitions/libedit.py

@@ -38,3 +38,6 @@ def get_additional_compiler_flags(self, builder: BuilderInterface) -> List[str]:
 
     def build(self, builder: BuilderInterface) -> None:
         builder.build_with_configure(dep=self, extra_configure_args=['--with-pic'])
+
+    def use_cppflags_env_var(self) -> bool:
+        return True

python/build_definitions/libkeyutils.py

@@ -40,7 +40,8 @@ def build(self, builder: BuilderInterface) -> None:
     def get_additional_ld_flags(self, builder: 'BuilderInterface') -> List[str]:
         flags = []
         if (builder.compiler_choice.is_clang() and
-                builder.compiler_choice.is_llvm_major_version_at_least(17)):
+                builder.compiler_choice.is_llvm_major_version_at_least(17) and
+                not is_macos()):
             # Workaround for https://github.com/madler/zlib/issues/856
             flags.append('-Wl,--undefined-version')
         return flags

python/build_definitions/libunistring.py

@@ -23,7 +23,7 @@ def __init__(self) -> None:
             'libunistring',
             '1.0',
             'https://ftp.gnu.org/gnu/libunistring/libunistring-{0}.tar.gz',
-            BuildGroup.POTENTIALLY_INSTRUMENTED)
+            BuildGroup.COMMON)
         self.copy_sources = True
 
     def get_compiler_wrapper_ld_flags_to_remove(self, builder: BuilderInterface) -> Set[str]:
@@ -33,3 +33,6 @@ def get_compiler_wrapper_ld_flags_to_remove(self, builder: BuilderInterface) ->
 
     def build(self, builder: BuilderInterface) -> None:
         builder.build_with_configure(dep=self)
+
+    def use_cppflags_env_var(self) -> bool:
+        return True

python/build_definitions/libunwind.py

@@ -19,9 +19,9 @@ class LibUnwindDependency(Dependency):
     def __init__(self) -> None:
         super(LibUnwindDependency, self).__init__(
             name='libunwind',
-            version='1.5.0',
+            version='1.8.1',
             url_pattern='https://github.com/libunwind/libunwind/releases/download/'
-                        'v1.5/libunwind-{0}.tar.gz',
+                        'v{0}/libunwind-{0}.tar.gz',
             build_group=BuildGroup.COMMON)
         self.copy_sources = True
 

python/build_definitions/libuuid.py

@@ -26,10 +26,11 @@ def __init__(self) -> None:
 
     def get_additional_compiler_flags(self, builder: BuilderInterface) -> List[str]:
         llvm_major_version = builder.compiler_choice.get_llvm_major_version()
-        linux_llvm15_or_later = (
-            is_linux() and llvm_major_version is not None and llvm_major_version >= 15)
+        llvm_installer_15_or_later = (
+            builder.compiler_choice.is_llvm_installer_clang() and
+            llvm_major_version is not None and llvm_major_version >= 15)
         flags = []
-        if linux_llvm15_or_later:
+        if llvm_installer_15_or_later:
             # https://gist.githubusercontent.com/mbautin/9ae79d6c81adaa68746287458cac4d10/raw
             flags.append('-Wno-error=implicit-function-declaration')
 

python/build_definitions/llvm_libcxx.py

@@ -31,7 +31,7 @@ def postprocess_ninja_build_file(
             builder: BuilderInterface,
             ninja_build_file_path: str) -> None:
         super().postprocess_ninja_build_file(builder, ninja_build_file_path)
-        if not builder.compiler_choice.is_linux_clang():
+        if not builder.compiler_choice.is_llvm_installer_clang():
             return
 
         if builder.build_type not in [BuildType.ASAN, BuildType.TSAN]:
@@ -46,8 +46,7 @@ def postprocess_ninja_build_file(
             num_lines_modified, os.path.abspath(ninja_build_file_path), removed_string)
 
     def get_additional_ld_flags(self, builder: BuilderInterface) -> List[str]:
-        # This workaround is needed for both LLVM 10 and LLVM 11.
-        if (builder.compiler_choice.is_linux_clang() and
+        if (builder.compiler_choice.is_llvm_installer_clang() and
                 builder.build_type in [BuildType.ASAN, BuildType.TSAN]):
             # We need to link with these libraries in ASAN because otherwise libc++ CMake
             # configuration step fails and some C standard library functions cannot be found.

python/build_definitions/openldap.py

@@ -29,11 +29,12 @@ def __init__(self) -> None:
 
     def get_additional_compiler_flags(self, builder: BuilderInterface) -> List[str]:
         llvm_major_version = builder.compiler_choice.get_llvm_major_version()
+        llvm_installer_15_or_later = (
+            builder.compiler_choice.is_llvm_installer_clang() and
+            llvm_major_version is not None and llvm_major_version >= 15)
         flags = []
-        linux_llvm15_or_later = (
-            is_linux() and llvm_major_version is not None and llvm_major_version >= 15)
 
-        if is_macos() or linux_llvm15_or_later:
+        if is_macos() or llvm_installer_15_or_later:
             # To avoid this error with Clang 15 on Linux:
             # https://gist.githubusercontent.com/mbautin/a9ca659ec5955ecb0e3d469376659c2b/raw
             flags.append('-Wno-error=implicit-function-declaration')

python/build_definitions/protobuf.py

@@ -50,3 +50,6 @@ def build(self, builder: BuilderInterface) -> None:
             ],
             run_autogen=True
         )
+
+    def use_cppflags_env_var(self) -> bool:
+        return True

python/build_definitions/zlib_dependency.py

@@ -34,7 +34,8 @@ def build(self, builder: BuilderInterface) -> None:
     def get_additional_ld_flags(self, builder: 'BuilderInterface') -> List[str]:
         flags = []
         if (builder.compiler_choice.is_clang() and
-                builder.compiler_choice.is_llvm_major_version_at_least(17)):
+                builder.compiler_choice.is_llvm_major_version_at_least(17) and
+                not is_macos()):
             # Workaround for https://github.com/madler/zlib/issues/856
             flags.append('-Wl,--undefined-version')
         return flags

python/yugabyte_db_thirdparty/builder.py

@@ -90,7 +90,7 @@
     get_clang_include_dir,
     create_llvm_tool_dir,
 )
-from yugabyte_db_thirdparty.macos import get_min_supported_macos_version
+from yugabyte_db_thirdparty.macos import get_min_supported_macos_version, get_macos_sysroot
 from yugabyte_db_thirdparty import (
     compile_commands,
     constants,
@@ -319,6 +319,7 @@ def populate_dependencies(self) -> None:
                 get_build_def_module('libuuid').LibUuidDependency(),
             ]
 
+        if is_linux() or self.compiler_choice.is_llvm_installer_clang():
             llvm_major_version: Optional[int] = self.compiler_choice.get_llvm_major_version()
             if (self.compiler_choice.is_clang() and
                     llvm_major_version is not None and llvm_major_version >= 10):
@@ -503,17 +504,22 @@ def init_compiler_independent_flags(self, dep: Dependency) -> None:
         elif is_macos():
             self.shared_lib_suffix = "dylib"
 
-            # YugaByte builds with C++11, which on OS X requires using libc++ as the standard
-            # library implementation. Some of the dependencies do not compile against libc++ by
-            # default, so we specify it explicitly.
-            self.cxx_flags.append("-stdlib=libc++")
-            self.ld_flags += ["-lc++", "-lc++abi"]
+            if not self.compiler_choice.is_llvm_installer_clang():
+                # YugaByte builds with C++11, which on OS X requires using libc++ as the standard
+                # library implementation. Some of the dependencies do not compile against libc++ by
+                # default, so we specify it explicitly.
+                self.cxx_flags.append("-stdlib=libc++")
+                self.ld_flags += ["-lc++", "-lc++abi"]
 
             # Build for macOS Mojave or later. See https://bit.ly/37myHbk
             extend_lists(
                 [self.compiler_flags, self.ld_flags, self.assembler_flags],
                 ["-mmacosx-version-min=%s" % get_min_supported_macos_version()])
 
+            extend_lists(
+                [self.preprocessor_flags, self.ld_flags],
+                ['-isysroot', get_macos_sysroot()])
+
             self.ld_flags.append("-Wl,-headerpad_max_install_names")
         else:
             fatal("Unsupported platform: {}".format(platform.system()))
@@ -983,12 +989,12 @@ def init_flags(self, dep: Dependency) -> None:
         """
         self.init_compiler_independent_flags(dep)
 
-        if not is_macos() and self.compiler_choice.using_clang():
-            # Special setup for Clang on Linux.
+        if self.compiler_choice.is_llvm_installer_clang():
+            # Special setup for LLVM installer Clang.
             compiler_choice = self.compiler_choice
             llvm_major_version: Optional[int] = compiler_choice.get_llvm_major_version()
             if llvm_major_version is not None and llvm_major_version >= 10:
-                self.init_linux_clang_flags(dep)
+                self.init_clang_flags(dep)
             else:
                 raise ValueError(f"Unknown or unsupproted LLVM major version: {llvm_major_version}")
 
@@ -1006,7 +1012,7 @@ def get_libcxx_dirs(self, libcxx_installed_suffix: str) -> Tuple[str, str]:
         libcxx_installed_lib = os.path.join(libcxx_installed_path, 'lib')
         return libcxx_installed_include, libcxx_installed_lib
 
-    def init_linux_clang_flags(self, dep: Dependency) -> None:
+    def init_clang_flags(self, dep: Dependency) -> None:
         """
         Flags for Clang. We are using LLVM-supplied libunwind, and in most cases, compiler-rt in
         this configuration.

python/yugabyte_db_thirdparty/compiler_choice.py

@@ -181,6 +181,9 @@ def is_gcc_major_version_at_least(self, expected_major_version: int) -> bool:
     def is_linux_clang(self) -> bool:
         return is_linux() and self.is_clang()
 
+    def is_llvm_installer_clang(self) -> bool:
+        return self.expected_major_compiler_version is not None and self.is_clang()
+
     def set_compiler(self, use_compiler_wrapper: Optional[bool] = None) -> None:
         if use_compiler_wrapper is not None:
             self.use_compiler_wrapper = use_compiler_wrapper

python/yugabyte_db_thirdparty/dependency.py

@@ -128,7 +128,7 @@ def get_compiler_wrapper_ld_flags_to_append(self,  builder: 'BuilderInterface')
         compiler wrapper command line.
         """
         llvm_major_version: Optional[int] = builder.compiler_choice.get_llvm_major_version()
-        if (is_linux() and
+        if (builder.compiler_choice.is_llvm_installer_clang() and
                 llvm_major_version is not None and
                 llvm_major_version >= 12 and
                 builder.lto_type is not None):

python/yugabyte_db_thirdparty/dependency_selection.py

@@ -42,7 +42,12 @@
 
 
 def get_common_dependency_module_names() -> List[str]:
-    return list(DEFAULT_COMMON_DEPENDENCY_MODULE_NAMES)
+    names = list(DEFAULT_COMMON_DEPENDENCY_MODULE_NAMES)
+    if is_macos():
+        # On macOS, flex, bison, and krb5 depend on gettext, and we don't want to use gettext from
+        # Homebrew. libunistring is required by gettext.
+        names.extend(['libunistring', 'gettext'])
+    return names
 
 
 def get_final_dependency_module_names(compiler_choice: CompilerChoice) -> List[str]:
@@ -51,11 +56,6 @@ def get_final_dependency_module_names(compiler_choice: CompilerChoice) -> List[s
     """
     dep_names: List[str] = []
 
-    if is_macos():
-        # On macOS, flex, bison, and krb5 depend on gettext, and we don't want to use gettext from
-        # Homebrew. libunistring is required by gettext.
-        dep_names.extend(['libunistring', 'gettext'])
-
     dep_names.append('ncurses')
 
     if is_linux():

python/yugabyte_db_thirdparty/macos.py

@@ -11,6 +11,8 @@
 # under the License.
 #
 
+import glob
+
 from yugabyte_db_thirdparty.arch import get_target_arch
 from sys_detection import is_macos
 
@@ -27,3 +29,13 @@ def get_min_supported_macos_version() -> str:
         return MIN_SUPPORTED_MACOS_VERSION_ARM64
     raise ValueError("Could not determine minimum supported macOS version for target "
                      "architecture %s" % target_arch)
+
+
+def get_macos_sysroot() -> str:
+    candidates = list(glob.glob(
+        '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/'
+        'MacOSX*.sdk'))
+    if not candidates:
+        raise RuntimeError("No Xcode SDKs found")
+    candidates.sort()
+    return candidates[-1]

thirdparty_src_checksums.txt

@@ -37,6 +37,7 @@ be1d5d8a71404a43f9cd7d0709a586bacc930b05b418863657b28bd5e761b59c  libedit-201912
 ce3aff48059fb8f7d53935171945546a36d962fd25295d6b5723a66ba8b43683  libkeyutils-1.6.1-yb-1.tar.gz
 3c0184c0e492d7c208ce31d25dd1d2c58f0c3ed6cbbe032c5b248cddad318544  libunistring-1.0.tar.gz
 90337653d92d4a13de590781371c604f9031cdb50520366aa1e3a91e1efb1017  libunwind-1.5.0.tar.gz
+ddf0e32dd5fafe5283198d37e4bf9decf7ba1770b6e7e006c33e6df79e6a6157  libunwind-1.8.1.tar.gz
 39cfe3100eae207d0c8f8f921a3e5a7c12a686833761fcf0ba6805e38312813e  libuuid-1.0.3.tar.gz
 8c8eff0bec85306aae508392b395e8bb0e1e34daad2a4e3bec308ced0c92bfaa  libuv-1.23.0.tar.gz
 8d1756fd704f147549f606cd987050fb94b0b1ff621ea6aa4d6bf0b74450468a  libverto-0.3.2.tar.gz