Skip to content

Commit e8f1d73

Browse files
sheetalarkadamsheetalarkadam
authored
Add Android QNN Browserstack test (microsoft#22434)
Add Android QNN Browserstack test ### Motivation and Context Real device test in CI
1 parent c9ed016 commit e8f1d73

File tree

10 files changed

+211
-35
lines changed

10 files changed

+211
-35
lines changed

‎java/src/test/android/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ Use the android's [build instructions](https://onnxruntime.ai/docs/build/android
2929

3030
Please note that you may need to set the `--android_abi=x86_64` (the default option is `arm64-v8a`). This is because android instrumentation test is run on an android emulator which requires an abi of `x86_64`.
3131

32+
#### QNN Builds
33+
We use two AndroidManifest.xml files to manage different runtime requirements for QNN support. In the [build configuration](app/build.gradle), we specify which manifest file to use based on the qnnVersion.
34+
In the [QNN manifest](app/src/main/AndroidManifestQnn.xml), we include the <uses-native-library> declaration for libcdsprpc.so, which is required for devices using QNN and Qualcomm DSP capabilities.
35+
For QNN builds, it is also necessary to set the `ADSP_LIBRARY_PATH` environment variable to the [native library directory](https://developer.android.com/reference/android/content/pm/ApplicationInfo#nativeLibraryDir) depending on the device. This ensures that any native libraries downloaded as dependencies such as QNN libraries are found by the application. This is conditionally added by using the BuildConfig field IS_QNN_BUILD set in the build.gradle file.
36+
3237
#### Build Output
3338

3439
The build will generate two apks which is required to run the test application in `$YOUR_BUILD_DIR/java/androidtest/android/app/build/outputs/apk`:

‎java/src/test/android/app/build.gradle‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ plugins {
44
}
55

66
def minSdkVer = System.properties.get("minSdkVer")?:24
7+
def qnnVersion = System.properties['qnnVersion']
78

89
android {
910
compileSdkVersion 32
@@ -16,6 +17,14 @@ android {
1617
versionName "1.0"
1718

1819
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
20+
21+
// Add BuildConfig field for qnnVersion
22+
if (qnnVersion != null) {
23+
buildConfigField "boolean", "IS_QNN_BUILD", "true"
24+
}
25+
else {
26+
buildConfigField "boolean", "IS_QNN_BUILD", "false"
27+
}
1928
}
2029

2130
buildTypes {
@@ -31,6 +40,24 @@ android {
3140
kotlinOptions {
3241
jvmTarget = '1.8'
3342
}
43+
// Conditional packagingOptions for QNN builds only
44+
if (qnnVersion != null) {
45+
packagingOptions {
46+
jniLibs {
47+
useLegacyPackaging = true
48+
}
49+
// Dsp is used in older QC devices and not supported by ORT
50+
// Gpu support isn't the target, we just want Npu support (Htp)
51+
exclude 'lib/arm64-v8a/libQnnGpu.so'
52+
exclude 'lib/arm64-v8a/libQnnDsp*.so'
53+
}
54+
55+
sourceSets {
56+
main {
57+
manifest.srcFile 'src/main/AndroidManifestQnn.xml' // Use QNN manifest
58+
}
59+
}
60+
}
3461
namespace 'ai.onnxruntime.example.javavalidator'
3562
}
3663

@@ -44,9 +71,18 @@ dependencies {
4471
testImplementation 'junit:junit:4.+'
4572
androidTestImplementation 'androidx.test.ext:junit:1.1.3'
4673
androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
47-
implementation(name: "onnxruntime-android", ext: "aar")
4874

4975
androidTestImplementation 'androidx.test:runner:1.4.0'
5076
androidTestImplementation 'androidx.test:rules:1.4.0'
5177
androidTestImplementation 'com.microsoft.appcenter:espresso-test-extension:1.4'
78+
79+
// dependencies for onnxruntime-android-qnn
80+
if (qnnVersion != null) {
81+
implementation(name: "onnxruntime-android-qnn", ext: "aar")
82+
implementation "com.qualcomm.qti:qnn-runtime:$qnnVersion"
83+
}
84+
else {
85+
implementation(name: "onnxruntime-android", ext: "aar")
86+
}
87+
5288
}

‎java/src/test/android/app/src/androidTest/java/ai/onnxruntime/example/javavalidator/SimpleTest.kt‎

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,18 @@ class SimpleTest {
3838
@Test
3939
fun runSigmoidModelTest() {
4040
for (intraOpNumThreads in 1..4) {
41-
runSigmoidModelTestImpl(intraOpNumThreads)
41+
runSigmoidModelTestImpl(intraOpNumThreads, OrtProvider.CPU)
4242
}
4343
}
4444

4545
@Test
4646
fun runSigmoidModelTestNNAPI() {
47-
runSigmoidModelTestImpl(1, true)
47+
runSigmoidModelTestImpl(1, OrtProvider.NNAPI)
48+
}
49+
50+
@Test
51+
fun runSigmoidModelTestQNN() {
52+
runSigmoidModelTestImpl(1, OrtProvider.QNN)
4853
}
4954

5055
@Throws(IOException::class)
@@ -54,22 +59,49 @@ class SimpleTest {
5459
}
5560

5661
@Throws(OrtException::class, IOException::class)
57-
fun runSigmoidModelTestImpl(intraOpNumThreads: Int, useNNAPI: Boolean = false) {
58-
reportHelper.label("Start Running Test with intraOpNumThreads=$intraOpNumThreads, useNNAPI=$useNNAPI")
62+
fun runSigmoidModelTestImpl(intraOpNumThreads: Int, executionProvider: OrtProvider) {
63+
reportHelper.label("Start Running Test with intraOpNumThreads=$intraOpNumThreads, executionProvider=$executionProvider")
5964
Log.println(Log.INFO, TAG, "Testing with intraOpNumThreads=$intraOpNumThreads")
60-
Log.println(Log.INFO, TAG, "Testing with useNNAPI=$useNNAPI")
65+
Log.println(Log.INFO, TAG, "Testing with executionProvider=$executionProvider")
66+
6167
val env = OrtEnvironment.getEnvironment(OrtLoggingLevel.ORT_LOGGING_LEVEL_VERBOSE)
6268
env.use {
6369
val opts = SessionOptions()
6470
opts.setIntraOpNumThreads(intraOpNumThreads)
65-
if (useNNAPI) {
66-
if (OrtEnvironment.getAvailableProviders().contains(OrtProvider.NNAPI)) {
67-
opts.addNnapi()
68-
} else {
69-
Log.println(Log.INFO, TAG, "NO NNAPI EP available, skip the test")
70-
return
71+
72+
when (executionProvider) {
73+
74+
OrtProvider.NNAPI -> {
75+
if (OrtEnvironment.getAvailableProviders().contains(OrtProvider.NNAPI)) {
76+
opts.addNnapi()
77+
} else {
78+
Log.println(Log.INFO, TAG, "NO NNAPI EP available, skip the test")
79+
return
80+
}
81+
}
82+
83+
OrtProvider.QNN -> {
84+
if (OrtEnvironment.getAvailableProviders().contains(OrtProvider.QNN)) {
85+
// Since this is running in an Android environment, we use the .so library
86+
val qnnLibrary = "libQnnHtp.so"
87+
val providerOptions = Collections.singletonMap("backend_path", qnnLibrary)
88+
opts.addQnn(providerOptions)
89+
} else {
90+
Log.println(Log.INFO, TAG, "NO QNN EP available, skip the test")
91+
return
92+
}
93+
}
94+
95+
OrtProvider.CPU -> {
96+
// No additional configuration is needed for CPU
97+
}
98+
99+
else -> {
100+
// Non exhaustive when statements on enum will be prohibited in future Gradle versions
101+
Log.println(Log.INFO, TAG, "Skipping test as OrtProvider is not implemented")
71102
}
72103
}
104+
73105
opts.use {
74106
val session = env.createSession(readModel("sigmoid.ort"), opts)
75107
session.use {
@@ -92,13 +124,15 @@ class SimpleTest {
92124
output.use {
93125
@Suppress("UNCHECKED_CAST")
94126
val rawOutput = output[0].value as Array<Array<FloatArray>>
127+
// QNN EP will run the Sigmoid float32 op with fp16 precision
128+
val precision = if (executionProvider == OrtProvider.QNN) 1e-3 else 1e-6
95129
for (i in 0..2) {
96130
for (j in 0..3) {
97131
for (k in 0..4) {
98132
Assert.assertEquals(
99133
rawOutput[i][j][k],
100134
expected[i][j][k],
101-
1e-6.toFloat()
135+
precision.toFloat()
102136
)
103137
}
104138
}

‎java/src/test/android/app/src/main/AndroidManifest.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@
1717
</activity>
1818
</application>
1919

20-
</manifest>
20+
</manifest>

‎java/src/test/android/app/src/main/AndroidManifestQnn.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
3+
4+
<application
5+
android:allowBackup="true"
6+
android:icon="@mipmap/ic_launcher"
7+
android:label="@string/app_name"
8+
android:roundIcon="@mipmap/ic_launcher_round"
9+
android:supportsRtl="true"
10+
android:theme="@style/Theme.JavaValidator">
11+
<activity android:name=".MainActivity" android:exported="true">
12+
<intent-filter>
13+
<action android:name="android.intent.action.MAIN" />
14+
15+
<category android:name="android.intent.category.LAUNCHER" />
16+
</intent-filter>
17+
</activity>
18+
<uses-native-library
19+
android:name="libcdsprpc.so"
20+
android:required="false" />
21+
</application>
22+
23+
</manifest>

‎java/src/test/android/app/src/main/java/ai/onnxruntime/example/javavalidator/MainActivity.kt‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,19 @@
11
package ai.onnxruntime.example.javavalidator
22

33
import android.os.Bundle
4+
import android.system.Os
45
import androidx.appcompat.app.AppCompatActivity
56

67
/*Empty activity app mainly used for testing*/
78
class MainActivity : AppCompatActivity() {
89
override fun onCreate(savedInstanceState: Bundle?) {
10+
if (BuildConfig.IS_QNN_BUILD) {
11+
val adspLibraryPath = applicationContext.applicationInfo.nativeLibraryDir
12+
// set the path variable to the native library directory
13+
// so that any native libraries downloaded as dependencies
14+
// (like qnn libs) are found
15+
Os.setenv("ADSP_LIBRARY_PATH", adspLibraryPath, true)
16+
}
917
super.onCreate(savedInstanceState)
1018
}
11-
}
19+
}

‎tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml‎

Lines changed: 48 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@ parameters:
1919
type: string
2020
default: ''
2121

22+
- name: QnnSDKVersion
23+
displayName: QNN SDK Version
24+
type: string
25+
default: '2.28.0.241029'
26+
2227
jobs:
2328
- job: Final_AAR_Testing_Android_${{ parameters.job_name_suffix }}
2429
workspace:
@@ -50,36 +55,61 @@ jobs:
5055

5156
- template: use-android-ndk.yml
5257

53-
- template: use-android-emulator.yml
54-
parameters:
55-
create: true
56-
start: true
57-
5858
- script: |
59-
set -e -x
60-
mkdir android_test
61-
cd android_test
62-
cp -av $(Build.SourcesDirectory)/java/src/test/android ./
63-
cd ./android
64-
mkdir -p app/libs
65-
cp $(Build.BinariesDirectory)/final-android-aar/${{parameters.packageName}}-$(OnnxRuntimeVersion)${{parameters.ReleaseVersionSuffix}}.aar app/libs/onnxruntime-android.aar
66-
$(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest --stacktrace
67-
displayName: Run E2E test using Emulator
59+
set -e -x
60+
mkdir -p android_test/android/app/libs
61+
cd android_test/android
62+
cp -av $(Build.SourcesDirectory)/java/src/test/android/* ./
63+
cp $(Build.BinariesDirectory)/final-android-aar/${{parameters.packageName}}-$(OnnxRuntimeVersion)${{parameters.ReleaseVersionSuffix}}.aar app/libs/${{parameters.packageName}}.aar
64+
displayName: Copy Android test files and AAR to android_test directory
6865
workingDirectory: $(Build.BinariesDirectory)
6966
70-
- template: use-android-emulator.yml
71-
parameters:
72-
stop: true
67+
# skip emulator tests for qnn package as there are no arm64-v8a emulators and no qnn libraries for x86
68+
- ${{ if not(contains(parameters.packageName, 'qnn')) }}:
69+
- template: use-android-emulator.yml
70+
parameters:
71+
create: true
72+
start: true
73+
74+
- script: |
75+
set -e -x
76+
cd android_test/android
77+
$(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest --stacktrace
78+
displayName: Run E2E test using Emulator
79+
workingDirectory: $(Build.BinariesDirectory)
80+
81+
- template: use-android-emulator.yml
82+
parameters:
83+
stop: true
84+
85+
- ${{ else }}:
86+
- script: |
87+
# QNN SDK version string, expected format: 2.28.0.241029
88+
# Extract the first three parts of the version string to get the Maven package version (e.g., 2.28.0)
89+
QnnMavenPackageVersion=$(echo ${{ parameters.QnnSDKVersion }} | cut -d'.' -f1-3)
90+
echo "QnnMavenPackageVersion: $QnnMavenPackageVersion"
91+
echo "##vso[task.setvariable variable=QnnMavenPackageVersion]$QnnMavenPackageVersion"
92+
displayName: Trim QNN SDK version to major.minor.patch
93+
94+
- script: |
95+
set -e -x
96+
# build apks for qnn package as they are not built in the emulator test step
97+
$(Build.SourcesDirectory)/java/gradlew --no-daemon clean assembleDebug assembleAndroidTest -DqnnVersion=$(QnnMavenPackageVersion) --stacktrace
98+
displayName: Build QNN APK
99+
workingDirectory: $(Build.BinariesDirectory)/android_test/android
73100
74101
# we run e2e tests on one older device (Pixel 3) and one newer device (Galaxy 23)
75102
- script: |
76103
set -e -x
77104
pip install requests
105+
78106
python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \
79107
--test_platform espresso \
80108
--app_path "debug/app-debug.apk" \
81109
--test_path "androidTest/debug/app-debug-androidTest.apk" \
82-
--devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0"
110+
--devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0" \
111+
--build_tag "${{ parameters.packageName }}"
112+
83113
displayName: Run E2E tests using Browserstack
84114
workingDirectory: $(Build.BinariesDirectory)/android_test/android/app/build/outputs/apk
85115
timeoutInMinutes: 15

‎tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,12 @@ stages:
9898
enable_code_sign: ${{ parameters.DoEsrp }}
9999
packageName: 'onnxruntime-android-qnn'
100100
ReleaseVersionSuffix: $(ReleaseVersionSuffix)
101-
#TODO: Add test job for QNN Android AAR
101+
102+
- template: android-java-api-aar-test.yml
103+
parameters:
104+
artifactName: 'onnxruntime-android-qnn-aar'
105+
job_name_suffix: 'QNN'
106+
packageName: 'onnxruntime-android-qnn'
102107

103108
- stage: iOS_Full_xcframework
104109
dependsOn: []

‎tools/ci_build/github/azure-pipelines/templates/jobs/download_linux_qnn_sdk.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,29 @@ steps:
1616
echo $(QnnSDKRootDir)
1717
displayName: 'Print QnnSDKRootDir after downloading QNN SDK'
1818
19+
- script: |
20+
set -x
21+
sdk_file="$(QnnSDKRootDir)/sdk.yaml"
22+
# Parse the sdk.yaml file to get the QNN SDK version downloaded
23+
downloaded_qnn_sdk_version=$(grep '^version:' "$sdk_file" | head -n 1 | cut -d':' -f2 | xargs | cut -d'.' -f1-3 | tr -d '\r')
24+
25+
# Extract major.minor.patch part from QnnSDKVersion passed as parameter
26+
expected_qnn_sdk_version=$(echo ${{ parameters.QnnSDKVersion }} | cut -d'.' -f1-3)
27+
28+
if [[ -z "$downloaded_qnn_sdk_version" ]]; then
29+
echo "QNN version not found in sdk.yaml."
30+
exit 1
31+
fi
32+
33+
# Compare provided version with version from sdk.yaml
34+
if [[ "$downloaded_qnn_sdk_version" == "$expected_qnn_sdk_version" ]]; then
35+
echo "Success: QnnSDKVersion matches sdk.yaml version ($downloaded_qnn_sdk_version)."
36+
else
37+
echo "Error: QnnSDKVersion ($expected_qnn_sdk_version) does not match sdk.yaml version ($downloaded_qnn_sdk_version) in the QNN SDK directory"
38+
exit 1
39+
fi
40+
displayName: "Sanity Check: QnnSDKVersion vs sdk.yaml version"
41+
1942
- script: |
2043
azcopy cp --recursive 'https://lotusscus.blob.core.windows.net/models/qnnsdk/Qualcomm AI Hub Proprietary License.pdf' $(QnnSDKRootDir)
2144
displayName: 'Download Qualcomm AI Hub license'

0 commit comments

Comments
 (0)