ci: Update build script for upgrading libwebrtc version m107 (#842)

* upgrade libwebrtc M96

* fix

* replace python to python3

* use clang provided by chromium

* apply patch to enable bitcode

* Revert "use clang provided by chromium"

This reverts commit ba4ce00.

* Fix build errors due to interface changes in M96.
NOTE:
The webrtc.zip for the M96 version also needs to be updated to include these missing files. (For testing, I used the files locally from the M92 version)
api/video_track_source_proxy.h
api/proxy.h
media/base/h264_profile_level_id.h

* Fix typo

* update libwebrtc m107

* fix build error when generating license file

* fix

* fix

* fix build error on android

* fix

* fix

* fix

* fix

* use_custom_libcxx=false

* test

* fix error

* use_custom_libcxx=false

* test

* fix

* revert

* update image

* fix

* revert

* wip

* change image for linux build

* fix build error

* add patch file

* disable use_custom_libcxx for android

* remove is_java_debug

* revert

* downgrade JDK version

* add is_java_debug

* revert

* revert

* revert

* revert

Co-authored-by: Nishant Naveen <[email protected]>

Author: karasusan

.yamato/package.metafile

@@ -2,7 +2,7 @@ upm:
   registry_url: https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
   package_version: stable
 webrtc_version:
-  name: M92
+  name: M107
 
 editors:
   - version: 2019.4

.yamato/upm-ci-libwebrtc.yml

@@ -3,17 +3,17 @@
 platforms:
   - name: win
     type: Unity::VM
-    image: renderstreaming/win10:v0.3.13-1084239
+    image: renderstreaming/win10:v0.4.0-1250406
     flavor: b1.xlarge
     build_command: BuildScripts~/build_libwebrtc_win.cmd
   - name: macos
     type: Unity::VM::osx
-    image: package-ci/mac:v1.20.0-1079282
+    image: package-ci/macos-12:v4.3.0-1248181
     flavor: m1.mac  
     build_command: BuildScripts~/build_libwebrtc_macos.sh
   - name: linux
     type: Unity::VM
-    image: cds-ops/ubuntu-18.04-base:v1.6.0-546186
+    image: package-ci/ubuntu-20.04:v1.4.0-1235639
     flavor: b1.xlarge
     build_command: BuildScripts~/build_libwebrtc_linux.sh
   - name: android
@@ -23,7 +23,7 @@ platforms:
     build_command: BuildScripts~/build_libwebrtc_android.sh
   - name: ios
     type: Unity::VM::osx
-    image: package-ci/mac:v1.20.0-1079282
+    image: package-ci/macos-12:v4.3.0-1248181
     flavor: m1.mac  
     build_command: BuildScripts~/build_libwebrtc_ios.sh
 projects:

BuildScripts~/build_libwebrtc_android.sh

@@ -7,9 +7,10 @@ fi
 
 export COMMAND_DIR=$(cd $(dirname $0); pwd)
 export PATH="$(pwd)/depot_tools:$PATH"
-export WEBRTC_VERSION=4515
+export WEBRTC_VERSION=5304
 export OUTPUT_DIR="$(pwd)/out"
 export ARTIFACTS_DIR="$(pwd)/artifacts"
+export PYTHON3_BIN="$(pwd)/depot_tools/python-bin/python3"
 
 if [ ! -e "$(pwd)/src" ]
 then
@@ -19,12 +20,25 @@ then
   sudo git config --system core.longpaths true
   git checkout "refs/remotes/branch-heads/$WEBRTC_VERSION"
   cd ..
-  gclient sync -f
+  gclient sync -D --force --reset
 fi
 
-# add jsoncpp
+# Add jsoncpp
 patch -N "src/BUILD.gn" < "$COMMAND_DIR/patches/add_jsoncpp.patch"
 
+# Add visibility libunwind
+patch -N "src/buildtools/third_party/libunwind/BUILD.gn" < "$COMMAND_DIR/patches/add_visibility_libunwind.patch"
+
+# Add deps libunwind
+patch -N "src/build/config/BUILD.gn" < "$COMMAND_DIR/patches/add_deps_libunwind.patch"
+
+# downgrade JDK11 to JDK8 because Unity supports OpenJDK version 1.8.
+# https://docs.unity3d.com/Manual/android-sdksetup.html
+pushd "src/build"
+git apply "$COMMAND_DIR/patches/downgrade_JDK.patch"
+popd
+
+
 mkdir -p "$ARTIFACTS_DIR/lib"
 
 for target_cpu in "arm64"
@@ -34,16 +48,19 @@ do
   for is_debug in "true" "false"
   do
     # generate ninja files
+    # use `treat_warnings_as_errors` option to avoid deprecation warnings
     gn gen "$OUTPUT_DIR" --root="src" \
       --args="is_debug=${is_debug} \
+      is_java_debug=${is_debug} \
       target_os=\"android\" \
       target_cpu=\"${target_cpu}\" \
       rtc_use_h264=false \
       rtc_include_tests=false \
       rtc_build_examples=false \
       is_component_build=false \
       use_rtti=true \
-      use_custom_libcxx=false"
+      use_custom_libcxx=false \
+      treat_warnings_as_errors=false"
 
     # build static library
     ninja -C "$OUTPUT_DIR" webrtc
@@ -62,17 +79,20 @@ pushd src
 
 for is_debug in "true" "false"
 do
-  python tools_webrtc/android/build_aar.py \
+  # use `treat_warnings_as_errors` option to avoid deprecation warnings
+  "$PYTHON3_BIN" tools_webrtc/android/build_aar.py \
     --build-dir $OUTPUT_DIR \
     --output $OUTPUT_DIR/libwebrtc.aar \
     --arch arm64-v8a \
     --extra-gn-args "is_debug=${is_debug} \
+      is_java_debug=${is_debug} \
       rtc_use_h264=false \
       rtc_include_tests=false \
       rtc_build_examples=false \
       is_component_build=false \
       use_rtti=true \
-      use_custom_libcxx=false"
+      use_custom_libcxx=false \
+      treat_warnings_as_errors=false"
 
   filename="libwebrtc.aar"
   if [ $is_debug = "true" ]; then
@@ -84,12 +104,8 @@ done
 
 popd
 
-# fix error when generate license
-patch -N "./src/tools_webrtc/libs/generate_licenses.py" < \
-  "$COMMAND_DIR/patches/generate_licenses.patch"
-
-python "./src/tools_webrtc/libs/generate_licenses.py" \
-  --target //:default "$OUTPUT_DIR" "$OUTPUT_DIR"
+"$PYTHON3_BIN" "./src/tools_webrtc/libs/generate_licenses.py" \
+  --target :webrtc "$OUTPUT_DIR" "$OUTPUT_DIR"
 
 cd src
 find . -name "*.h" -print | cpio -pd "$ARTIFACTS_DIR/include"
@@ -98,4 +114,4 @@ cp "$OUTPUT_DIR/LICENSE.md" "$ARTIFACTS_DIR"
 
 # create zip
 cd "$ARTIFACTS_DIR"
-zip -r webrtc-android.zip lib include LICENSE.md
+zip -r webrtc-android.zip lib include LICENSE.md

BuildScripts~/build_libwebrtc_ios.sh

@@ -7,9 +7,10 @@ fi
 
 export COMMAND_DIR=$(cd $(dirname $0); pwd)
 export PATH="$(pwd)/depot_tools:$PATH"
-export WEBRTC_VERSION=4515
+export WEBRTC_VERSION=5304
 export OUTPUT_DIR="$(pwd)/out"
 export ARTIFACTS_DIR="$(pwd)/artifacts"
+export PYTHON3_BIN="$(pwd)/depot_tools/python-bin/python3"
 
 if [ ! -e "$(pwd)/src" ]
 then
@@ -19,7 +20,7 @@ then
   sudo git config --system core.longpaths true
   git checkout "refs/remotes/branch-heads/$WEBRTC_VERSION"
   cd ..
-  gclient sync -f
+  gclient sync -D --force --reset
 fi
 
 # add jsoncpp
@@ -44,18 +45,15 @@ do
     #       See below for details.
     #       https://bugs.chromium.org/p/webrtc/issues/detail?id=11729
     #
-    # note: `use_xcode_clang=true` is for using bitcode.
-    #
     gn gen "$OUTPUT_DIR" --root="src" \
-      --args="is_debug=${is_debug}    \
-      target_os=\"ios\"               \
-      target_cpu=\"${target_cpu}\"    \
-      rtc_use_h264=false              \
-      treat_warnings_as_errors=false  \
-      use_xcode_clang=true            \
-      enable_ios_bitcode=true         \
-      ios_enable_code_signing=false   \
-      rtc_include_tests=false         \
+      --args="is_debug=${is_debug} \
+      target_os=\"ios\" \
+      target_cpu=\"${target_cpu}\" \
+      rtc_use_h264=false \
+      use_custom_libcxx=false \
+      treat_warnings_as_errors=false \
+      ios_enable_code_signing=false \
+      rtc_include_tests=false \
       rtc_build_examples=false"
 
     # build static library
@@ -81,12 +79,8 @@ do
   rm -r "$ARTIFACTS_DIR/lib/x64"
 done
 
-# fix error when generate license
-patch -N "./src/tools_webrtc/libs/generate_licenses.py" < \
-  "$COMMAND_DIR/patches/generate_licenses.patch"
-
-vpython "./src/tools_webrtc/libs/generate_licenses.py" \
-  --target //:default "$OUTPUT_DIR" "$OUTPUT_DIR"
+"$PYTHON3_BIN" "./src/tools_webrtc/libs/generate_licenses.py" \
+  --target :webrtc "$OUTPUT_DIR" "$OUTPUT_DIR"
 
 cd src
 find . -name "*.h" -print | cpio -pd "$ARTIFACTS_DIR/include"

BuildScripts~/build_libwebrtc_linux.sh

@@ -7,9 +7,10 @@ fi
 
 export COMMAND_DIR=$(cd $(dirname $0); pwd)
 export PATH="$(pwd)/depot_tools:$PATH"
-export WEBRTC_VERSION=4515
+export WEBRTC_VERSION=5304
 export OUTPUT_DIR="$(pwd)/out"
 export ARTIFACTS_DIR="$(pwd)/artifacts"
+export PYTHON3_BIN="$(pwd)/depot_tools/python-bin/python3"
 
 if [ ! -e "$(pwd)/src" ]
 then
@@ -19,7 +20,7 @@ then
   sudo git config --system core.longpaths true
   git checkout "refs/remotes/branch-heads/$WEBRTC_VERSION"
   cd ..
-  gclient sync -f
+  gclient sync -D --force --reset
 fi
 
 # add jsoncpp
@@ -32,7 +33,19 @@ do
   mkdir -p "$ARTIFACTS_DIR/lib/${target_cpu}"
   for is_debug in "true" "false"
   do
-    args="is_debug=${is_debug} target_os=\"linux\" target_cpu=\"${target_cpu}\" rtc_include_tests=false rtc_build_examples=false rtc_use_h264=false symbol_level=0 enable_iterator_debugging=false is_component_build=false use_rtti=true rtc_use_x11=false libcxx_abi_unstable=false";
+    args="is_debug=${is_debug} \
+      target_os=\"linux\" \
+      target_cpu=\"${target_cpu}\" \
+      use_custom_libcxx=false \
+      rtc_include_tests=false \
+      rtc_build_examples=false \
+      rtc_use_h264=false \
+      symbol_level=0 \
+      enable_iterator_debugging=false \
+      is_component_build=false \
+      use_rtti=true \
+      rtc_use_x11=false"
+
     if [ $is_debug = "true" ]; then
       args="${args} is_asan=true is_lsan=true";
     fi
@@ -53,12 +66,8 @@ do
   done
 done
 
-# fix error when generate license
-patch -N "./src/tools_webrtc/libs/generate_licenses.py" < \
-  "$COMMAND_DIR/patches/generate_licenses.patch"
-
-vpython "./src/tools_webrtc/libs/generate_licenses.py" \
-  --target //:default "$OUTPUT_DIR" "$OUTPUT_DIR"
+"$PYTHON3_BIN" "./src/tools_webrtc/libs/generate_licenses.py" \
+  --target :webrtc "$OUTPUT_DIR" "$OUTPUT_DIR"
 
 cd src
 find . -name "*.h" -print | cpio -pd "$ARTIFACTS_DIR/include"

BuildScripts~/build_libwebrtc_macos.sh

@@ -7,9 +7,10 @@ fi
 
 export COMMAND_DIR=$(cd $(dirname $0); pwd)
 export PATH="$(pwd)/depot_tools:$PATH"
-export WEBRTC_VERSION=4515
+export WEBRTC_VERSION=5304
 export OUTPUT_DIR="$(pwd)/out"
 export ARTIFACTS_DIR="$(pwd)/artifacts"
+export PYTHON3_BIN="$(pwd)/depot_tools/python-bin/python3"
 
 if [ ! -e "$(pwd)/src" ]
 then
@@ -19,7 +20,7 @@ then
   sudo git config --system core.longpaths true
   git checkout "refs/remotes/branch-heads/$WEBRTC_VERSION"
   cd ..
-  gclient sync -f
+  gclient sync -D --force --reset
 fi
 
 # add jsoncpp
@@ -44,15 +45,15 @@ do
       --args="is_debug=${is_debug} \
       target_os=\"mac\"  \
       target_cpu=\"${target_cpu}\" \
+      use_custom_libcxx=false \
       rtc_include_tests=false \
       rtc_build_examples=false \
       rtc_use_h264=false \
       symbol_level=0 \
       enable_iterator_debugging=false \
       is_component_build=false \
       use_rtti=true \
-      rtc_use_x11=false \
-      libcxx_abi_unstable=false"
+      rtc_use_x11=false"
 
     # build static library
     ninja -C "$OUTPUT_DIR" webrtc
@@ -77,12 +78,8 @@ do
   rm -r "$ARTIFACTS_DIR/lib/x64"
 done
 
-# fix error when generate license
-patch -N "./src/tools_webrtc/libs/generate_licenses.py" < \
-  "$COMMAND_DIR/patches/generate_licenses.patch"
-
-vpython "./src/tools_webrtc/libs/generate_licenses.py" \
-  --target //:default "$OUTPUT_DIR" "$OUTPUT_DIR"
+"$PYTHON3_BIN" "./src/tools_webrtc/libs/generate_licenses.py" \
+  --target :webrtc "$OUTPUT_DIR" "$OUTPUT_DIR"
 
 cd src
 find . -name "*.h" -print | cpio -pd "$ARTIFACTS_DIR/include"

BuildScripts~/build_libwebrtc_win.cmd

@@ -6,9 +6,8 @@ if not exist depot_tools (
 
 set COMMAND_DIR=%~dp0
 set PATH=%cd%\depot_tools;%PATH%
-set WEBRTC_VERSION=4515
+set WEBRTC_VERSION=5304
 set DEPOT_TOOLS_WIN_TOOLCHAIN=0
-set CPPFLAGS=/WX-
 set GYP_GENERATORS=ninja,msvs-ninja
 set GYP_MSVS_VERSION=2019
 set OUTPUT_DIR=out
@@ -24,12 +23,18 @@ if not exist src (
   call git.bat config --system core.longpaths true
   call git.bat checkout  refs/remotes/branch-heads/%WEBRTC_VERSION%
   cd ..
-  call gclient.bat sync -f
+  call gclient.bat sync -D --force --reset
 )
 
 rem add jsoncpp
 patch -N "src\BUILD.gn" < "%COMMAND_DIR%\patches\add_jsoncpp.patch"
 
+rem fix towupper
+patch -N "src\modules\desktop_capture\win\full_screen_win_application_handler.cc" < "%COMMAND_DIR%\patches\fix_towupper.patch"
+
+rem fix abseil
+patch -N "src\third_party\abseil-cpp/absl/base/config.h" < "%COMMAND_DIR%\patches\fix_abseil.patch"
+
 rem install pywin32
 call "%cd%\depot_tools\bootstrap-3_8_0_chromium_8_bin\python\bin\python.exe" ^
   -m pip install pywin32 --index-url "%PYPI_URL%" --upgrade
@@ -44,10 +49,10 @@ for %%i in (x64) do (
 
     rem generate ninja for release
     call gn.bat gen %OUTPUT_DIR% --root="src" ^
-      --args="is_debug=%%j is_clang=false target_cpu=\"%%i\" rtc_include_tests=false rtc_build_examples=false rtc_use_h264=false symbol_level=0 enable_iterator_debugging=false"
+      --args="is_debug=%%j is_clang=true target_cpu=\"%%i\" use_custom_libcxx=false rtc_include_tests=false rtc_build_examples=false rtc_use_h264=false symbol_level=0 enable_iterator_debugging=false"
 
     rem build
-    ninja.exe -C %OUTPUT_DIR%
+    ninja.exe -C %OUTPUT_DIR% webrtc
 
     set filename=
     if true==%%j (
@@ -63,13 +68,9 @@ for %%i in (x64) do (
 
 endlocal
 
-rem fix error when generate license
-patch -N "%cd%\src\tools_webrtc\libs\generate_licenses.py" < ^
-  "%COMMAND_DIR%\patches\generate_licenses.patch"
-
 rem generate license
 call python.bat "%cd%\src\tools_webrtc\libs\generate_licenses.py" ^
-  --target //:default %OUTPUT_DIR% %OUTPUT_DIR%
+  --target :webrtc %OUTPUT_DIR% %OUTPUT_DIR%
 
 rem unescape license
 powershell -File "%COMMAND_DIR%\Unescape.ps1" "%OUTPUT_DIR%\LICENSE.md"