Swift: enable bootstrapping

Author: Alex Denisov

CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 3.12.4)
+project(codeql-swift-artifacts C CXX)
+
+find_package(LLVM REQUIRED CONFIG)
+find_package(Clang REQUIRED CONFIG)
+find_package(Swift REQUIRED CONFIG)
+
+message("Using LLVM_CONFIG: ${Swift_CONFIG}")
+message("Using Clang_CONFIG: ${LLVM_CONFIG}")
+message("Using Swift_CONFIG: ${Clang_CONFIG}")
+
+add_executable(codeql-swift-artifacts empty.cpp)
+target_link_libraries(codeql-swift-artifacts PRIVATE LLVMSupport swiftFrontendTool swiftCompilerModules)
+
+if(APPLE)
+  execute_process(
+    COMMAND xcrun -find swiftc
+    OUTPUT_VARIABLE CODEQL_SWIFT_COMPILER
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  execute_process(
+    COMMAND xcrun -show-sdk-path
+    OUTPUT_VARIABLE CODEQL_MACOS_SDK_PATH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+
+  # Add in the toolchain directory so we can grab compatibility libraries
+  # Inspired by the Swift's CMakeLists
+  get_filename_component(TOOLCHAIN_BIN_DIR ${CODEQL_SWIFT_COMPILER} DIRECTORY)
+  get_filename_component(TOOLCHAIN_LIB_DIR "${TOOLCHAIN_BIN_DIR}/../lib/swift/macosx" ABSOLUTE)
+  target_link_directories(codeql-swift-artifacts PUBLIC ${TOOLCHAIN_LIB_DIR})
+  target_link_directories(codeql-swift-artifacts PUBLIC ${CODEQL_MACOS_SDK_PATH}/usr/lib/swift)
+endif()
+

empty.cpp

@@ -11,98 +11,33 @@
 import zlib
 from collections import namedtuple
 
-DEPS = {"llvm": ["LLVMSupport"],
-        "swift": ["swiftFrontendTool"]}
-
 
 def getoptions():
     parser = argparse.ArgumentParser(description="package swift for codeql compilation")
-    for p in DEPS:
-        parser.add_argument(f"--{p}", required=True, type=resolve,
-                            metavar="DIR", help=f"path to {p} build root")
+    parser.add_argument(f"--llvm-build-tree", required=True, type=resolve,
+                        metavar="DIR", help=f"path to LLVM build tree")
+    parser.add_argument(f"--swift-build-tree", required=True, type=resolve,
+                        metavar="DIR", help=f"path to Swift build tree")
+    parser.add_argument(f"--swift-source-tree", required=True, type=resolve,
+                        metavar="DIR", help=f"path to Swift source tree")
+
     default_output = f"swift-prebuilt-{get_platform()}"
-    parser.add_argument("--keep-tmp-dir", "-K", action="store_true",
-                        help="do not clean up the temporary directory")
     parser.add_argument("--output", "-o", type=pathlib.Path, metavar="DIR_OR_ZIP",
                         help="output zip file or directory "
                              f"(by default the filename is {default_output})")
-    update_help_fmt = "Only update the {} library in DIR, triggering rebuilds of required files"
-    parser.add_argument("--update-shared", "-u", metavar="DIR", type=pathlib.Path,
-                        help=update_help_fmt.format("shared"))
-    parser.add_argument("--update-static", "-U", metavar="DIR", type=pathlib.Path,
-                        help=update_help_fmt.format("static"))
+
     opts = parser.parse_args()
-    if opts.output and (opts.update_shared or opts.update_static):
-        parser.error("provide --output or one of --update-*, not both")
     if opts.output is None:
         opts.output = pathlib.Path()
     opts.output = get_tgt(opts.output, default_output)
-    return opts
-
 
-Libs = namedtuple("Libs", ("archive", "static", "shared"))
-
-DEPLIST = [x for d in DEPS.values() for x in d]
-
-CMAKELISTS_DUMMY = f"""
-cmake_minimum_required(VERSION 3.12.4)
-
-project(dummy C CXX)
+    return opts
 
-find_package(LLVM REQUIRED CONFIG PATHS ${{LLVM_ROOT}}/lib/cmake/llvm NO_DEFAULT_PATH)
-find_package(Clang REQUIRED CONFIG PATHS ${{LLVM_ROOT}}/lib/cmake/clang NO_DEFAULT_PATH)
-find_package(Swift REQUIRED CONFIG PATHS ${{SWIFT_ROOT}}/lib/cmake/swift NO_DEFAULT_PATH)
 
-add_executable(dummy empty.cpp)
-target_link_libraries(dummy PRIVATE {" ".join(DEPLIST)})
-"""
+Libs = namedtuple("Libs", ("archive", "static", "shared", "linker_flags"))
 
 EXPORTED_LIB = "CodeQLSwiftFrontendTool"
 
-CMAKELISTS_EXPORTED_FMT = """
-add_library({exported} INTERFACE)
-
-if (BUILD_SHARED_LIBS)
-    if (APPLE)
-        set(EXT "dylib")
-    else()
-        set(EXT "so")
-    endif()
-else()
-    set(EXT "a")
-endif()
-
-set (SwiftLLVMWrapperLib libCodeQLSwiftFrontendTool.${{EXT}})
-set (input ${{CMAKE_CURRENT_LIST_DIR}}/${{SwiftLLVMWrapperLib}})
-set (output ${{CMAKE_BINARY_DIR}}/${{SwiftLLVMWrapperLib}})
-
-add_custom_command(OUTPUT ${{output}}
-        COMMAND ${{CMAKE_COMMAND}} -E copy_if_different ${{input}} ${{output}}
-        DEPENDS ${{input}})
-add_custom_target(copy-llvm-swift-wrapper DEPENDS ${{output}})
-
-target_include_directories({exported} INTERFACE ${{CMAKE_CURRENT_LIST_DIR}}/include)
-target_link_libraries({exported} INTERFACE
-    ${{output}}
-    {libs}
-)
-add_dependencies(swiftAndLlvmSupport copy-llvm-swift-wrapper)
-"""
-
-
-class TempDir:
-    def __init__(self, cleanup=True):
-        self.path = None
-        self.cleanup = cleanup
-
-    def __enter__(self):
-        self.path = pathlib.Path(tempfile.mkdtemp())
-        return self.path
-
-    def __exit__(self, *args):
-        if self.cleanup:
-            shutil.rmtree(self.path)
-
 
 def resolve(p):
     return pathlib.Path(p).resolve()
@@ -118,53 +53,27 @@ def run(prog, *, cwd, env=None, input=None):
     subprocess.run(prog, cwd=cwd, env=runenv, input=input, text=True)
 
 
-def build(dir, targets):
-    print(f"building {' '.join(targets)} in {dir}")
-    cmd = ["cmake", "--build", ".", "--"]
-    cmd.extend(targets)
-    run(cmd, cwd=dir)
-
-
 def get_platform():
     return "linux" if platform.system() == "Linux" else "macos"
 
 
-def create_empty_cpp(path):
-    with open(path / "empty.cpp", "w"):
-        pass
-
-
-def install(tmp, opts):
-    print("installing dependencies")
-    tgt = tmp / "install"
-    for p in DEPS:
-        builddir = getattr(opts, p)
-        run(["cmake", "--build", ".", "--", "install"], cwd=builddir, env={"DESTDIR": tgt})
-    if sys.platform != 'linux':
-        return tgt / "Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain"
-    return tgt
-
-
-def configure_dummy_project(tmp, *, llvm=None, swift=None, installed=None):
+def configure_dummy_project(tmp, *, llvm=None, swift=None):
     print("configuring dummy cmake project")
-    if installed is not None:
-        swift = llvm = installed / "usr"
-    with open(tmp / "CMakeLists.txt", "w") as out:
-        out.write(CMAKELISTS_DUMMY)
-    create_empty_cpp(tmp)
+    shutil.copy("CMakeLists.txt", tmp / "CMakeLists.txt")
+    shutil.copy("empty.cpp", tmp / "empty.cpp")
     tgt = tmp / "build"
     tgt.mkdir()
-    run(["cmake", f"-DLLVM_ROOT={llvm}", f"-DSWIFT_ROOT={swift}", "-DBUILD_SHARED_LIBS=OFF", ".."],
+    run(["cmake", f"-DCMAKE_PREFIX_PATH={llvm};{swift}", "-DBUILD_SHARED_LIBS=OFF", ".."],
         cwd=tgt)
     return tgt
 
 
 def get_libs(configured):
     print("extracting linking information from dummy project")
-    with open(configured / "CMakeFiles" / "dummy.dir" / "link.txt") as link:
+    with open(configured / "CMakeFiles" / "codeql-swift-artifacts.dir" / "link.txt") as link:
         libs = link.read().split()
-        libs = libs[libs.index('dummy')+1:] # skip up to -o dummy
-    ret = Libs([], [], [])
+        libs = libs[libs.index('codeql-swift-artifacts')+1:] # skip up to -o dummy
+    ret = Libs([], [], [], [])
     for l in libs:
         if l.endswith(".a"):
             ret.static.append(str((configured / l).resolve()))
@@ -173,11 +82,10 @@ def get_libs(configured):
             ret.shared.append(f"-l{l[3:]}")  # drop 'lib' prefix and '.so' suffix
         elif l.startswith("-l"):
             ret.shared.append(l)
+        elif l.startswith("-L") or l.startswith("-Wl"):
+            ret.linker_flags.append(l)
         else:
             raise ValueError(f"cannot understand link.txt: " + l)
-    # move direct dependencies into archive
-    ret.archive[:] = ret.static[:len(DEPLIST)]
-    ret.static[:len(DEPLIST)] = []
     return ret
 
 
@@ -195,7 +103,6 @@ def create_static_lib(tgt, libs):
         mriscript = f"create {tgt}\n{includedlibs}\nsave\nend"
         run(["ar", "-M"], cwd=tgt.parent, input=mriscript)
     else:
-        includedlibs = " ".join(f"{l}" for l in libs.archive + libs.static)
         libtool_args = ["libtool", "-static"]
         libtool_args.extend(libs.archive)
         libtool_args.extend(libs.static)
@@ -214,6 +121,7 @@ def create_shared_lib(tgt, libs):
     print(f"packaging {libname}")
     compiler = os.environ.get("CC", "clang")
     cmd = [compiler, "-shared"]
+    cmd.extend(libs.linker_flags)
 
     if sys.platform == 'linux':
         cmd.append("-Wl,--whole-archive")
@@ -239,17 +147,17 @@ def create_shared_lib(tgt, libs):
 def copy_includes(src, tgt):
     print("copying includes")
     for dir, exts in (("include", ("h", "def", "inc")), ("stdlib", ("h",))):
-        srcdir = src / "usr" / dir
+        srcdir = src / dir
         for ext in exts:
             for srcfile in srcdir.rglob(f"*.{ext}"):
                 tgtfile = tgt / dir / srcfile.relative_to(srcdir)
                 tgtfile.parent.mkdir(parents=True, exist_ok=True)
                 shutil.copy(srcfile, tgtfile)
 
 
-def create_sdk(installed, tgt):
+def export_sdk(tgt, swift_source_tree, swift_build_tree):
     print("assembling sdk")
-    srcdir = installed / "usr" / "lib" / "swift"
+    srcdir = swift_build_tree/ "lib" / "swift"
     tgtdir = tgt / "usr" / "lib" / "swift"
     if get_platform() == "linux":
         srcdir /= "linux"
@@ -258,24 +166,30 @@ def create_sdk(installed, tgt):
         srcdir /= "macosx"
     for mod in srcdir.glob("*.swiftmodule"):
         shutil.copytree(mod, tgtdir / mod.name)
-    shutil.copytree(installed / "usr" / "stdlib" / "public" / "SwiftShims",
-                    tgt / "usr" / "include" / "SwiftShims")
+    shutil.copytree(swift_source_tree / "stdlib" / "public" / "SwiftShims",
+                    tgt / "usr" / "include" / "SwiftShims",
+                    ignore=shutil.ignore_patterns('CMakeLists.txt'))
 
 
-def create_export_dir(tmp, installed, libs):
-    print("assembling prebuilt directory")
-    exportedlibs = [create_static_lib(tmp, libs), create_shared_lib(tmp, libs)]
-    tgt = tmp / "exported"
-    tgt.mkdir()
+def export_libs(exported_dir, libs):
+    print("exporting libraries")
+    exportedlibs = [
+            create_static_lib(exported_dir, libs),
+            create_shared_lib(exported_dir, libs)
+            ]
     for l in exportedlibs:
-        l.rename(tgt / l.name)
-    with open(tgt / "swift_llvm_prebuilt.cmake", "w") as out:
-        # drop -l prefix here
-        sharedlibs = " ".join(l[2:] for l in libs.shared)
-        out.write(CMAKELISTS_EXPORTED_FMT.format(exported=EXPORTED_LIB, libs=sharedlibs))
-    copy_includes(installed, tgt)
-    create_sdk(installed, tgt / "sdk")
-    return tgt
+        l.rename(exported_dir / l.name)
+
+
+def export_headers(exported_dir, swift_source_tree, llvm_build_tree, swift_build_tree):
+    print("exporting headers")
+    # Assuming default checkout where LLVM sources are placed next to Swift sources
+    llvm_source_tree = swift_source_tree.parent / 'llvm-project/llvm'
+    clang_source_tree = swift_source_tree.parent / 'llvm-project/clang'
+    clang_tools_build_tree = llvm_build_tree / 'tools/clang'
+    header_dirs = [ llvm_source_tree, clang_source_tree, swift_source_tree, llvm_build_tree, swift_build_tree, clang_tools_build_tree ]
+    for h in header_dirs:
+        copy_includes(h, exported_dir)
 
 
 def zip_dir(src, tgt):
@@ -296,27 +210,17 @@ def main(opts):
     if os.path.exists(tmp):
         shutil.rmtree(tmp)
     os.mkdir(tmp)
-    if opts.update_shared or opts.update_static:
-        for project, deps in DEPS.items():
-            build(getattr(opts, project), deps)
-        configured = configure_dummy_project(tmp, llvm=opts.llvm, swift=opts.swift)
-        libs = get_libs(configured)
-        if opts.update_shared:
-            create_shared_lib(opts.update_shared, libs)
-        if opts.update_static:
-            create_static_lib(opts.update_static, libs)
-    else:
-        installed = install(tmp, opts)
-        swift_syntax_build = opts.swift / "include/swift/Syntax/"
-        swift_syntax_install = installed / "usr/include/swift/Syntax/"
-        for header in os.listdir(swift_syntax_build):
-            if header.endswith('.h') or header.endswith('.def'):
-                shutil.copy(swift_syntax_build / header, swift_syntax_install / header)
-        configured = configure_dummy_project(tmp, installed=installed)
-        libs = get_libs(configured)
-        exported = create_export_dir(tmp, installed, libs)
-        zip_dir(exported, opts.output)
-        tar_dir(exported, opts.output)
+    configured = configure_dummy_project(tmp, llvm=opts.llvm_build_tree, swift=opts.swift_build_tree)
+    libs = get_libs(configured)
+
+    exported = tmp / "exported"
+    exported.mkdir()
+    export_libs(exported, libs)
+    export_headers(exported, opts.swift_source_tree, opts.llvm_build_tree, opts.swift_build_tree)
+    export_sdk(exported / "sdk", opts.swift_source_tree, opts.swift_build_tree)
+
+    zip_dir(exported, opts.output)
+    tar_dir(exported, opts.output)
 
 
 if __name__ == "__main__":

pkg_swift_llvm.py

@@ -1,5 +1,5 @@
 [preset: codeql-baseline]
-bootstrapping=off
+bootstrapping=hosttools
 
 llvm-cmake-options=-DLLVM_ENABLE_TERMINFO=OFF -DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64