Merge #171

171: Improve android test support r=Dylan-DPC a=malbarbo

With this changes it is possible to run libstd tests and most of cargo tests (qemu segfault in some).

Co-authored-by: Marco A L Barbosa <[email protected]>

Author: bors[bot]

docker/aarch64-linux-android/Dockerfile

@@ -12,32 +12,35 @@ RUN apt-get update && \
 COPY xargo.sh /
 RUN bash /xargo.sh
 
-COPY qemu.sh /
-RUN bash /qemu.sh aarch64 android
-
 COPY android-ndk.sh /
 RUN bash /android-ndk.sh arm64 21
 ENV PATH=$PATH:/android-ndk/bin
 
+COPY android-system.sh /
+RUN bash /android-system.sh arm64
+
+COPY qemu.sh /
+RUN bash /qemu.sh aarch64 android
+
 COPY openssl.sh /
 RUN bash /openssl.sh linux-generic64 aarch64-linux-android- -mandroid -fomit-frame-pointer
 
-COPY android-system.sh /
-RUN bash /android-system.sh arm64
+RUN cp /android-ndk/sysroot/usr/lib/libz.so /system/lib/
 
 # Libz is distributed in the android ndk, but for some unknown reason it is not
 # found in the build process of some crates, so we explicit set the DEP_Z_ROOT
 ENV CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER=aarch64-linux-android-gcc \
     CARGO_TARGET_AARCH64_LINUX_ANDROID_RUNNER=qemu-aarch64 \
     CC_aarch64_linux_android=aarch64-linux-android-gcc \
     CXX_aarch64_linux_android=aarch64-linux-android-g++ \
-    DEP_Z_ROOT=/android-ndk/sysroot/usr/ \
+    DEP_Z_INCLUDE=/android-ndk/sysroot/usr/include/ \
     OPENSSL_STATIC=1 \
     OPENSSL_DIR=/openssl \
     OPENSSL_INCLUDE_DIR=/openssl/include \
     OPENSSL_LIB_DIR=/openssl/lib \
     RUST_TEST_THREADS=1 \
     HOME=/tmp/ \
     TMPDIR=/tmp/ \
-    ANDROID_ROOT=/system \
-    ANDROID_DATA=/
+    ANDROID_DATA=/ \
+    ANDROID_DNS_MODE=local \
+    ANDROID_ROOT=/system

docker/android-system.sh

@@ -52,15 +52,18 @@ EOF
     # this is the minimum set of modules that are need to build bionic
     # this was created by trial and error
     ./repo init -u https://android.googlesource.com/platform/manifest -b android-5.0.0_r1
-    ./repo sync bionic
-    ./repo sync build
-    ./repo sync external/compiler-rt
-    ./repo sync external/jemalloc
-    ./repo sync external/libcxx
-    ./repo sync external/libcxxabi
-    ./repo sync external/stlport
-    ./repo sync prebuilts/clang/linux-x86/host/3.5
-    ./repo sync system/core
+    ./repo sync -c bionic
+    ./repo sync -c build
+    ./repo sync -c external/compiler-rt
+    ./repo sync -c external/jemalloc
+    ./repo sync -c external/libcxx
+    ./repo sync -c external/libcxxabi
+    ./repo sync -c external/libselinux
+    ./repo sync -c external/mksh
+    ./repo sync -c external/openssl
+    ./repo sync -c external/stlport
+    ./repo sync -c prebuilts/clang/linux-x86/host/3.5
+    ./repo sync -c system/core
     case $arch in
         arm)
             ./repo sync prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.8
@@ -78,18 +81,36 @@ EOF
     esac
 
     # avoid build tests
-    rm bionic/linker/tests/Android.mk bionic/tests/Android.mk
+    rm bionic/linker/tests/Android.mk bionic/tests/Android.mk bionic/benchmarks/Android.mk
+
+    # patch the linker to avoid the error
+    # FATAL: kernel did not supply AT_SECURE
+    sed -i -e 's/if (!kernel_supplied_AT_SECURE)/if (false)/g' bionic/linker/linker_environ.cpp
 
     source build/envsetup.sh
     lunch aosp_$arch-user
     mmma bionic/
+    mmma external/mksh/
+    mmma system/core/toolbox/
 
     if [ $arch = "arm" ]; then
         mv out/target/product/generic/system/ /
     else
         mv out/target/product/generic_$arch/system/ /
     fi
 
+    # list from https://elinux.org/Android_toolbox
+    for tool in cat chmod chown cmp cp ctrlaltdel date dd df dmesg du getevent \
+        getprop grep hd id ifconfig iftop insmod ioctl ionice kill ln log ls \
+        lsmod lsof lsusb md5 mkdir mount mv nandread netstat newfs_msdos notify \
+        printenv ps reboot renice rm rmdir rmmod route schedtop sendevent \
+        setconsole setprop sleep smd start stop sync top touch umount \
+        uptime vmstat watchprops wipe; do
+        ln -s /system/bin/toolbox /system/bin/$tool
+    done
+
+    echo "127.0.0.1 localhost" > /system/etc/hosts
+
     # clean up
     apt-get purge --auto-remove -y ${purge_list[@]}
 

docker/arm-linux-androideabi/Dockerfile

@@ -12,32 +12,35 @@ RUN apt-get update && \
 COPY xargo.sh /
 RUN bash /xargo.sh
 
-COPY qemu.sh /
-RUN bash /qemu.sh arm android
-
 COPY android-ndk.sh /
 RUN bash /android-ndk.sh arm 21
 ENV PATH=$PATH:/android-ndk/bin
 
+COPY android-system.sh /
+RUN bash /android-system.sh arm
+
+COPY qemu.sh /
+RUN bash /qemu.sh arm android
+
 COPY openssl.sh /
 RUN bash /openssl.sh android arm-linux-androideabi-
 
-COPY android-system.sh /
-RUN bash /android-system.sh arm
+RUN cp /android-ndk/sysroot/usr/lib/libz.so /system/lib/
 
 # Libz is distributed in the android ndk, but for some unknown reason it is not
 # found in the build process of some crates, so we explicit set the DEP_Z_ROOT
 ENV CARGO_TARGET_ARM_LINUX_ANDROIDEABI_LINKER=arm-linux-androideabi-gcc \
     CARGO_TARGET_ARM_LINUX_ANDROIDEABI_RUNNER=qemu-arm \
     CC_arm_linux_androideabi=arm-linux-androideabi-gcc \
     CXX_arm_linux_androideabi=arm-linux-androideabi-g++ \
-    DEP_Z_ROOT=/android-ndk/sysroot/usr/ \
+    DEP_Z_INCLUDE=/android-ndk/sysroot/usr/include/ \
     OPENSSL_STATIC=1 \
     OPENSSL_DIR=/openssl \
     OPENSSL_INCLUDE_DIR=/openssl/include \
     OPENSSL_LIB_DIR=/openssl/lib \
     RUST_TEST_THREADS=1 \
     HOME=/tmp/ \
     TMPDIR=/tmp/ \
-    ANDROID_ROOT=/system \
-    ANDROID_DATA=/
+    ANDROID_DATA=/ \
+    ANDROID_DNS_MODE=local \
+    ANDROID_ROOT=/system

docker/armv7-linux-androideabi/Dockerfile

@@ -12,32 +12,35 @@ RUN apt-get update && \
 COPY xargo.sh /
 RUN bash /xargo.sh
 
-COPY qemu.sh /
-RUN bash /qemu.sh arm android
-
 COPY android-ndk.sh /
 RUN bash /android-ndk.sh arm 21
 ENV PATH=$PATH:/android-ndk/bin
 
+COPY android-system.sh /
+RUN bash /android-system.sh arm
+
+COPY qemu.sh /
+RUN bash /qemu.sh arm android
+
 COPY openssl.sh /
 RUN bash /openssl.sh android-armv7 arm-linux-androideabi-
 
-COPY android-system.sh /
-RUN bash /android-system.sh arm
+RUN cp /android-ndk/sysroot/usr/lib/libz.so /system/lib/
 
 # Libz is distributed in the android ndk, but for some unknown reason it is not
 # found in the build process of some crates, so we explicit set the DEP_Z_ROOT
 ENV CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER=arm-linux-androideabi-gcc \
     CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_RUNNER=qemu-arm \
     CC_armv7_linux_androideabi=arm-linux-androideabi-gcc \
     CXX_armv7_linux_androideabi=arm-linux-androideabi-g++ \
-    DEP_Z_ROOT=/android-ndk/sysroot/usr/ \
+    DEP_Z_INCLUDE=/android-ndk/sysroot/usr/include/ \
     OPENSSL_STATIC=1 \
     OPENSSL_DIR=/openssl \
     OPENSSL_INCLUDE_DIR=/openssl/include \
     OPENSSL_LIB_DIR=/openssl/lib \
     RUST_TEST_THREADS=1 \
     HOME=/tmp/ \
     TMPDIR=/tmp/ \
-    ANDROID_ROOT=/system \
-    ANDROID_DATA=/
+    ANDROID_DATA=/ \
+    ANDROID_DNS_MODE=local \
+    ANDROID_ROOT=/system

docker/i686-linux-android/Dockerfile

@@ -12,6 +12,13 @@ RUN apt-get update && \
 COPY xargo.sh /
 RUN bash /xargo.sh
 
+COPY android-ndk.sh /
+RUN bash /android-ndk.sh x86 21
+ENV PATH=$PATH:/android-ndk/bin
+
+COPY android-system.sh /
+RUN bash /android-system.sh x86
+
 # We could supposedly directly run i686 binaries like we do for x86_64, but
 # doing so generates an assertion failure:
 #   ... assertion failed: signal(libc::SIGPIPE, libc::SIG_IGN) != libc::SIG_ERR
@@ -22,30 +29,27 @@ RUN bash /xargo.sh
 COPY qemu.sh /
 RUN bash /qemu.sh i386 android
 
-COPY android-ndk.sh /
-RUN bash /android-ndk.sh x86 21
-ENV PATH=$PATH:/android-ndk/bin
-
 # Build with no-asm to make openssl linked binaries position-independent (PIE)
 COPY openssl.sh /
 RUN bash /openssl.sh android-x86 i686-linux-android- no-asm
 
-COPY android-system.sh /
-RUN bash /android-system.sh x86
+RUN cp /android-ndk/sysroot/usr/lib/libz.so /system/lib/
 
 # Libz is distributed in the android ndk, but for some unknown reason it is not
 # found in the build process of some crates, so we explicit set the DEP_Z_ROOT
 ENV CARGO_TARGET_I686_LINUX_ANDROID_LINKER=i686-linux-android-gcc \
     CARGO_TARGET_I686_LINUX_ANDROID_RUNNER="qemu-i386 -cpu n270" \
     CC_i686_linux_android=i686-linux-android-gcc \
     CXX_i686_linux_android=i686-linux-android-g++ \
-    DEP_Z_ROOT=/android-ndk/sysroot/usr/ \
+    DEP_Z_INCLUDE=/android-ndk/sysroot/usr/include/ \
+    LIBZ_SYS_STATIC=1 \
     OPENSSL_STATIC=1 \
     OPENSSL_DIR=/openssl \
     OPENSSL_INCLUDE_DIR=/openssl/include \
     OPENSSL_LIB_DIR=/openssl/lib \
     RUST_TEST_THREADS=1 \
     HOME=/tmp/ \
     TMPDIR=/tmp/ \
-    ANDROID_ROOT=/system \
-    ANDROID_DATA=/
+    ANDROID_DATA=/ \
+    ANDROID_DNS_MODE=local \
+    ANDROID_ROOT=/system

docker/qemu.sh

@@ -41,26 +41,6 @@ main() {
     # https://lists.nongnu.org/archive/html/qemu-trivial/2017-10/msg00023.html
     if [[ "$os" == "android" ]]; then
       patch -p1 <<'EOF'
-diff -ur qemu-2.10.0/linux-user/elfload.c qemu-2.10.0.new/linux-user/elfload.c
---- qemu-2.10.0/linux-user/elfload.c	2017-09-27 11:27:13.866595788 -0300
-+++ qemu-2.10.0.new/linux-user/elfload.c	2017-09-27 11:58:30.662613425 -0300
-@@ -1354,7 +1354,7 @@
-                                  ~(abi_ulong)(TARGET_ELF_EXEC_PAGESIZE-1))
- #define TARGET_ELF_PAGEOFFSET(_v) ((_v) & (TARGET_ELF_EXEC_PAGESIZE-1))
- 
--#define DLINFO_ITEMS 14
-+#define DLINFO_ITEMS 15
- 
- static inline void memcpy_fromfs(void * to, const void * from, unsigned long n)
- {
-@@ -1782,6 +1782,7 @@
-     NEW_AUX_ENT(AT_HWCAP, (abi_ulong) ELF_HWCAP);
-     NEW_AUX_ENT(AT_CLKTCK, (abi_ulong) sysconf(_SC_CLK_TCK));
-     NEW_AUX_ENT(AT_RANDOM, (abi_ulong) u_rand_bytes);
-+    NEW_AUX_ENT(AT_SECURE, (abi_ulong) (getuid() != geteuid() || getgid() != getegid()));
- 
- #ifdef ELF_HWCAP2
-     NEW_AUX_ENT(AT_HWCAP2, (abi_ulong) ELF_HWCAP2);
 diff -ur qemu-2.10.0/linux-user/ioctls.h qemu-2.10.0.new/linux-user/ioctls.h
 --- qemu-2.10.0/linux-user/ioctls.h	2017-09-27 11:27:13.858595669 -0300
 +++ qemu-2.10.0.new/linux-user/ioctls.h	2017-09-27 11:43:40.613299859 -0300

docker/x86_64-linux-android/Dockerfile

@@ -16,23 +16,32 @@ COPY android-ndk.sh /
 RUN bash /android-ndk.sh x86_64 21
 ENV PATH=$PATH:/android-ndk/bin
 
+COPY android-system.sh /
+RUN bash /android-system.sh x86_64
+
+# Using qemu allows older host cpus (without sse4) to execute the target binaries
+COPY qemu.sh /
+RUN bash /qemu.sh x86_64 android
+
 COPY openssl.sh /
 RUN bash /openssl.sh linux-x86_64 x86_64-linux-android- -mandroid -fomit-frame-pointer
 
-COPY android-system.sh /
-RUN bash /android-system.sh x86_64
+RUN cp /android-ndk/sysroot/usr/lib/libz.so /system/lib/
 
 # Libz is distributed in the android ndk, but for some unknown reason it is not
 # found in the build process of some crates, so we explicit set the DEP_Z_ROOT
 ENV CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER=x86_64-linux-android-gcc \
+    CARGO_TARGET_X86_64_LINUX_ANDROID_RUNNER="qemu-x86_64 -cpu Penryn" \
     CC_x86_64_linux_android=x86_64-linux-android-gcc \
     CXX_x86_64_linux_android=x86_64-linux-android-g++ \
-    DEP_Z_ROOT=/android-ndk/sysroot/usr/ \
+    DEP_Z_INCLUDE=/android-ndk/sysroot/usr/include/ \
     OPENSSL_STATIC=1 \
     OPENSSL_DIR=/openssl \
     OPENSSL_INCLUDE_DIR=/openssl/include \
     OPENSSL_LIB_DIR=/openssl/lib \
+    RUST_TEST_THREADS=1 \
     HOME=/tmp/ \
     TMPDIR=/tmp/ \
-    ANDROID_ROOT=/system \
-    ANDROID_DATA=/
+    ANDROID_DATA=/ \
+    ANDROID_DNS_MODE=local \
+    ANDROID_ROOT=/system