Linux arm cross compile build (#1018)

## Changelog

1. Use GCC to cross-compile Linux arm64 binaries in CI
    * Fixed compilation of remill with GCC
2. Use new Docker image as a base with installed cross-compilation dependencies
3. Patch LLVM 15 and 16 to compile for target architecture (`ports/llvm-{15,16}/portfile.cmake`)
4. Patch LLVM 15 and 16 to pass the correct compiler for cross-compiling and native binary builds (`ports/llvm-{15,16}/portfile.cmake`)
5. Update to LLVM 16.0.4
6. Patch protobuf port to always build target executable (`ports/protobuf/portfile.cmake`)
7. Patch XED to remove usage of `-m` architecture flags that broke during ARM cross-compilation (`ports/xed/0001-mbuild-remove-m64.patch`)
8. Patch glog to fix linker error between GCC and Clang (`ports/glog/fix_logstream_linker_error.patch`)
9. Patch vcpkg to correctly find compatible host executables to run
10. Use GCC to compile compatible lifting tools
11. No longer force clang as the default compiler when using `build_dependencies.sh` script

Author: ekilmer

.github/workflows/vcpkg_ci_amd64.yml

@@ -5,6 +5,7 @@ env:
   VCPKG_BINARY_SOURCES: 'clear;nuget,Source,readwrite;nugettimeout,3601'
   # Self-hosted runners can hopefully reuse already downloaded packages
   VCPKG_USE_NUGET_CACHE: 1
+  VCPKG_DEFAULT_HOST_TRIPLET: 'x64-linux-rel'
 
 on:
   release:
@@ -36,29 +37,25 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        host:
-          - { arch: 'amd64', triplet: 'x64-linux-rel' }
         image:
-          - { name: 'ubuntu', tag: '20.04' }
-          - { name: 'ubuntu', tag: '22.04' }
+          - { name: 'ubuntu-v2', tag: '20.04' }
+          - { name: 'ubuntu-v2', tag: '22.04' }
         llvm: [ 'llvm-15', 'llvm-16[pasta]', 'llvm-16' ]
+        target_arch: [ 'x64', 'arm64' ]
 
     container:
       image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }}
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    env:
-      VCPKG_DEFAULT_HOST_TRIPLET: ${{ matrix.host.triplet }}
-
-    name: ${{matrix.image.name}}-${{matrix.image.tag}} ${{matrix.llvm}} ${{matrix.host.arch}}
+    name: ${{matrix.image.name}}-${{matrix.image.tag}} ${{matrix.llvm}} ${{matrix.target_arch}}
     runs-on: ubuntu-latest # run the job on the newly created runner
     steps:
       - name: Set Artifact Name
         run: |
           # Need to fix because paths with brackets cause issues
-          NAME="$(echo 'vcpkg_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.llvm }}_${{ matrix.host.arch }}' | tr '[' '-' | tr -d ']')"
+          NAME="$(echo 'vcpkg_${{ matrix.image.name }}-${{ matrix.image.tag }}_${{ matrix.llvm }}_${{ matrix.target_arch == 'x64' && 'amd64' || matrix.target_arch }}' | tr '[' '-' | tr -d ']')"
           echo "ARTIFACT_NAME=${NAME}" >> "$GITHUB_ENV"
 
       - uses: actions/checkout@v3
@@ -116,10 +113,10 @@ jobs:
             -source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \
             "${{ secrets.GITHUB_TOKEN }}"
 
-          ./build_dependencies.sh --release --export-dir ./${{ env.ARTIFACT_NAME }} ${{ matrix.llvm }} --clean-after-build --debug
+          ./build_dependencies.sh --release --target-arch ${{ matrix.target_arch }} --export-dir ./${{ env.ARTIFACT_NAME }} ${{ matrix.llvm }} --clean-after-build --debug
 
           echo "VCPKG_ROOT=$(pwd)/${{ env.ARTIFACT_NAME }}" >> $GITHUB_ENV
-          echo "TARGET_TRIPLET=${{ matrix.host.triplet }}" >> $GITHUB_ENV
+          echo "TARGET_TRIPLET=${{ matrix.target_arch }}-linux-rel" >> $GITHUB_ENV
 
       - name: Upload CMake logs on error
         if: failure()
@@ -165,9 +162,9 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ${{  github.workspace }}/.ccache
-          key: ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.host.arch }}-${{ matrix.llvm }}-${{ steps.ccache_prep.outputs.timestamp }}
+          key: ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.target_arch }}-${{ matrix.llvm }}-${{ steps.ccache_prep.outputs.timestamp }}
           restore-keys: |
-            ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.host.arch }}-${{ matrix.llvm }}-
+            ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.target_arch }}-${{ matrix.llvm }}-
 
       - name: ccache Initial stats
         shell: bash
@@ -177,26 +174,30 @@ jobs:
       - name: 'Rellic build'
         shell: 'bash'
         working-directory: rellic
-        if: contains(matrix.llvm, 'llvm-15')
+        if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
         run: |
+          # Does not compile with gcc
+          export CC="$(which clang)"
+          export CXX="$(which clang++)"
           cmake -G Ninja \
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             "-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
             -DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET} \
-            -DVCPKG_HOST_TRIPLET=${TARGET_TRIPLET} \
+            -DVCPKG_HOST_TRIPLET=${VCPKG_DEFAULT_HOST_TRIPLET} \
             "-DCMAKE_INSTALL_PREFIX=$(pwd)/install" \
             -S . -B build
           cmake --build build
           cmake --build build --target install
       - name: 'Rellic test'
         shell: 'bash'
         working-directory: rellic/build
-        if: contains(matrix.llvm, 'llvm-15')
+        if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
         run: |
           ../scripts/roundtrip.py ./tools/rellic-decomp ../tests/tools/decomp "${VCPKG_ROOT}/installed/${TARGET_TRIPLET}/tools/llvm/clang"
 
       - name: 'Remill dependencies'
         shell: 'bash'
+        if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
         working-directory: remill
         run: |
           python3 -m pip install poetry
@@ -206,13 +207,19 @@ jobs:
 
       - name: 'Remill build'
         shell: 'bash'
+        if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
         working-directory: remill
         run: |
+          if [[ '${{ matrix.image.tag }}' == '20.04' ]]; then
+            # Remill uses C++20 concepts that aren't supported by gcc-9 in 20.04
+            export CC="$(which clang)"
+            export CXX="$(which clang++)"
+          fi
           cmake -G Ninja \
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             "-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
             -DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET} \
-            -DVCPKG_HOST_TRIPLET=${TARGET_TRIPLET} \
+            -DVCPKG_HOST_TRIPLET=${VCPKG_DEFAULT_HOST_TRIPLET} \
             "-DCMAKE_INSTALL_PREFIX=$(pwd)/install" \
             -S . -B build
           cmake --build build
@@ -221,21 +228,26 @@ jobs:
       - name: 'Remill test'
         shell: 'bash'
         working-directory: remill/build
+        if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
         run: |
           # Some tests fail on ubuntu 22.04
           env CTEST_OUTPUT_ON_FAILURE=1 ctest . || true
 
       - name: 'Anvill build'
         shell: 'bash'
         working-directory: anvill
-        if: contains(matrix.llvm, 'llvm-15')
+        if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
         run: |
+          # TODO: Does not support compilation with gcc
+          export CC="$(which clang)"
+          export CXX="$(which clang++)"
+
           cmake -G Ninja \
             -DCMAKE_VERBOSE_MAKEFILE=ON \
             "-DCMAKE_INSTALL_PREFIX=$(pwd)/install" \
             "-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
             -DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET} \
-            -DVCPKG_HOST_TRIPLET=${TARGET_TRIPLET} \
+            -DVCPKG_HOST_TRIPLET=${VCPKG_DEFAULT_HOST_TRIPLET} \
             -DANVILL_ENABLE_TESTS=true \
             -DANVILL_ENABLE_INSTALL=true \
             -DANVILL_ENABLE_PYTHON3_LIBS=OFF \
@@ -249,7 +261,7 @@ jobs:
       #- name: 'Anvill test'
       #  shell: 'bash'
       #  working-directory: anvill
-      #  if: contains(matrix.llvm, 'llvm-15')
+      #  if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
       #  run: |
       #    ./install/bin/anvill-decompile-spec --spec ../bin/Decompile/tests/specs/ret0.json --bc_out ./ret0.bc --ir_out ret0.ir
 

.github/workflows/vcpkg_ci_mac.yml

@@ -78,10 +78,14 @@ jobs:
           echo "Selecting XCode Version ${{ matrix.os.xcode }}"
           sudo xcode-select -s /Applications/Xcode_${{ matrix.os.xcode }}.app/Contents/Developer
 
+      # Crosscompiling LLVM 16 requires a newer clang than available in XCode 14.2
       - name: Install latest LLVM
+        if: contains(matrix.llvm, 'llvm-16') && matrix.target_arch == 'arm64'
         run: |
           brew install llvm
           echo "/usr/local/opt/llvm/bin" >> "$GITHUB_PATH"
+          echo "CC=/usr/local/opt/llvm/bin/clang" >> "$GITHUB_ENV"
+          echo "CXX=/usr/local/opt/llvm/bin/clang++" >> "$GITHUB_ENV"
 
       - name: Initialize vcpkg
         shell: bash

build_dependencies.sh

@@ -96,30 +96,6 @@ do
   die_if_not_installed ${pkg}
 done
 
-# check if CC is not set or a null string
-if [[ -z "${CC+unset}" || -z "${CC}" ]]; then
-  if type clang &>/dev/null; then
-    export CC="${CC:-$(which clang)}"
-    msg "Using default clang as CC=${CC}"
-  else
-    msg "Using default C compiler"
-  fi
-else
-  msg "Using custom CC=${CC}"
-fi
-
-# check if CXX is not set or a null string
-if [[ -z "${CXX+unset}" || -z "${CXX}" ]]; then
-  if type clang++ &>/dev/null; then
-    export CXX="${CXX:-$(which clang++)}"
-    msg "Using default clang++ as CXX=${CXX}"
-  else
-    msg "Using default C++ compiler"
-  fi
-else
-  msg "Using custom CXX=${CXX}"
-fi
-
 if [[ "$(uname -m)" = "aarch64" ]]; then
   export VCPKG_FORCE_SYSTEM_BINARIES=1
 fi

ports/glog/fix_cplusplus_macro.patch

@@ -0,0 +1,15 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index d145517..e8e1c90 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -669,6 +669,10 @@ if (CYGWIN OR WIN32)
+   target_compile_definitions (glog PUBLIC GLOG_NO_ABBREVIATED_SEVERITIES)
+ endif (CYGWIN OR WIN32)
+ 
++if ((MSVC) AND (MSVC_VERSION GREATER_EQUAL 1914))
++    target_compile_options(glog INTERFACE "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/Zc:__cplusplus>")
++endif()
++
+ if (WITH_CUSTOM_PREFIX)
+   target_compile_definitions (glog PUBLIC GLOG_CUSTOM_PREFIX_SUPPORT)
+ endif (WITH_CUSTOM_PREFIX)

ports/glog/fix_crosscompile_symbolize.patch

@@ -0,0 +1,22 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index d314abe..d145517 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -450,6 +450,9 @@ endif (HAVE_CXX11_ATOMIC)
+ 
+ if (WITH_SYMBOLIZE)
+   if (WIN32 OR CYGWIN)
++    if(CMAKE_CROSSCOMPILING)
++      set(HAVE_SYMBOLIZE 0)
++    else()
+     cmake_push_check_state (RESET)
+     set (CMAKE_REQUIRED_LIBRARIES DbgHelp)
+ 
+@@ -480,6 +483,7 @@ if (WITH_SYMBOLIZE)
+     ]=] HAVE_SYMBOLIZE)
+ 
+     cmake_pop_check_state ()
++    endif()
+ 
+     if (HAVE_SYMBOLIZE)
+       set (HAVE_STACKTRACE 1)

ports/glog/fix_glog_CMAKE_MODULE_PATH.patch

@@ -0,0 +1,33 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 846b444..20441d1 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -1027,7 +1027,7 @@ write_basic_package_version_file (
+ 
+ export (TARGETS glog NAMESPACE glog:: FILE glog-targets.cmake)
+ export (PACKAGE glog)
+-
++if(0)
+ get_filename_component (_PREFIX "${CMAKE_INSTALL_PREFIX}" ABSOLUTE)
+ 
+ # Directory containing the find modules relative to the config install
+@@ -1063,6 +1063,7 @@ file (INSTALL
+ "
+   COMPONENT Development
+ )
++endif()
+ 
+ install (FILES
+   ${CMAKE_CURRENT_BINARY_DIR}/glog-config.cmake
+diff --git a/glog-config.cmake.in b/glog-config.cmake.in
+index 5c5c9c0..31fac52 100644
+--- a/glog-config.cmake.in
++++ b/glog-config.cmake.in
+@@ -5,7 +5,6 @@ endif (CMAKE_VERSION VERSION_LESS @glog_CMake_VERSION@)
+ @PACKAGE_INIT@
+ 
+ include (CMakeFindDependencyMacro)
+-include (${CMAKE_CURRENT_LIST_DIR}/glog-modules.cmake)
+ 
+ @gflags_DEPENDENCY@
+ @Unwind_DEPENDENCY@

ports/glog/fix_logstream_linker_error.patch

@@ -0,0 +1,33 @@
+From 674d45068454718433aa46a40bd553bcbb142f71 Mon Sep 17 00:00:00 2001
+From: Eric Kilmer <[email protected]>
+Date: Tue, 16 May 2023 14:34:53 -0400
+Subject: [PATCH] logging: Prevent LogStream constructor from being discarded
+
+Fixes linker error reported in https://github.com/google/glog/issues/922
+---
+ src/glog/logging.h.in | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+
+diff --git a/src/glog/logging.h.in b/src/glog/logging.h.in
+index e8e6c41..ad25a65 100644
+--- a/src/glog/logging.h.in
++++ b/src/glog/logging.h.in
+@@ -1344,6 +1344,15 @@ GLOG_MSVC_PUSH_DISABLE_WARNING(4275)
+   class GLOG_EXPORT LogStream : public std::ostream {
+ GLOG_MSVC_POP_WARNING()
+   public:
++#if defined __has_attribute
++#  if __has_attribute (used)
++    // In some cases, like when compiling glog as a static library with GCC and
++    // linking against a Clang-built executable, this constructor will be
++    // removed by the linker. We use this attribute to prevent the linker from
++    // discarding it.
++    __attribute__ ((used))
++#  endif
++#endif
+     LogStream(char *buf, int len, int64 ctr)
+         : std::ostream(NULL),
+           streambuf_(buf, len),
+-- 
+2.40.1
+

ports/glog/glog_disable_debug_postfix.patch

@@ -0,0 +1,13 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 20441d1..d314abe 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -65,7 +65,7 @@ set (CMAKE_CXX_VISIBILITY_PRESET hidden)
+ set (CMAKE_POSITION_INDEPENDENT_CODE ON)
+ set (CMAKE_VISIBILITY_INLINES_HIDDEN ON)
+ 
+-set (CMAKE_DEBUG_POSTFIX d)
++#set (CMAKE_DEBUG_POSTFIX d)
+ set (CMAKE_THREAD_PREFER_PTHREAD 1)
+ 
+ find_package (GTest NO_MODULE)

ports/glog/portfile.cmake

@@ -0,0 +1,39 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO google/glog
+    REF v0.6.0
+    SHA512 fd2c42583d0dd72c790a8cf888f328a64447c5fb9d99b2e2a3833d70c102cb0eb9ae874632c2732424cc86216c8a076a3e24b23a793eaddb5da8a1dc52ba9226
+    HEAD_REF master
+    PATCHES
+      fix_glog_CMAKE_MODULE_PATH.patch
+      glog_disable_debug_postfix.patch
+      fix_crosscompile_symbolize.patch
+      fix_cplusplus_macro.patch
+      fix_logstream_linker_error.patch
+)
+
+vcpkg_check_features(
+    OUT_FEATURE_OPTIONS FEATURE_OPTIONS
+    FEATURES
+        unwind          WITH_UNWIND
+        customprefix    WITH_CUSTOM_PREFIX
+)
+file(REMOVE "${SOURCE_PATH}/glog-modules.cmake.in")
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        -DBUILD_TESTING=OFF
+        ${FEATURE_OPTIONS}
+)
+
+vcpkg_cmake_install()
+vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/glog)
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share")
+
+vcpkg_copy_pdbs()
+vcpkg_fixup_pkgconfig()
+
+file(INSTALL "${SOURCE_PATH}/COPYING" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright)

ports/glog/vcpkg.json

@@ -0,0 +1,28 @@
+{
+  "name": "glog",
+  "version": "0.6.0",
+  "port-version": 3,
+  "description": "C++ implementation of the Google logging module",
+  "homepage": "https://github.com/google/glog",
+  "license": "BSD-3-Clause",
+  "dependencies": [
+    "gflags",
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ],
+  "features": {
+    "customprefix": {
+      "description": "Enable support for user-generated message prefixes"
+    },
+    "unwind": {
+      "description": "Enable libunwind support",
+      "supports": "linux"
+    }
+  }
+}