Skip to content

Commit 9a06cad

Browse files
authored
chore: use larger dataset for integration test of image_classification_v2 (#968)
* Update tiny dataset for IMAGENET * Update expected_accuracy for image_classification * Update expected_accuracy for image_classification * Update expected_accuracy * Update expected_accuracy
1 parent a248581 commit 9a06cad

File tree

2 files changed

+22
-9
lines changed

2 files changed

+22
-9
lines changed

Diff for: flutter/assets/tasks.pbtxt

+4-4
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,10 @@ task {
4242
}
4343
tiny {
4444
name: "Imagenet dataset for integration test"
45-
input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v0_7/imagenet_tiny.zip"
46-
input_checksum: "42cb83d80d0341d6719d68b32322adf2"
47-
groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_0/imagenet_tiny-groundtruth.txt"
48-
groundtruth_checksum: "cc6d83ca25daeb2475f36c079e53bc26"
45+
input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v5_0/imagenet_tiny.zip"
46+
input_checksum: "2cde6f1d741b9c1834db261ac6e43a6e"
47+
groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v5_0/imagenet_tiny_val.txt"
48+
groundtruth_checksum: "086f99eff3d1b93fd41e7f35498570a7"
4949
}
5050
}
5151
model {

Diff for: flutter/integration_test/expected_accuracy.dart

+18-5
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,29 @@ key: <accelerator> OR <accelerator>|<backendName>
1818
*/
1919

2020
const Map<String, Interval> _imageClassificationV2 = {
21+
'cpu': Interval(min: 0.82, max: 0.84),
22+
'npu': Interval(min: 0.80, max: 0.82),
23+
'tpu': Interval(min: 0.82, max: 0.84),
24+
'ane': Interval(min: 0.69, max: 0.91),
25+
'cpu&gpu&ane': Interval(min: 0.69, max: 0.91),
26+
// TODO (anhappdev): check expected accuracy for snpe_dsp, psnpe_dsp
27+
'snpe_dsp': Interval(min: 0.0, max: 1.0),
28+
'psnpe_dsp': Interval(min: 0.0, max: 1.0),
29+
'neuron-mdla': Interval(min: 0.81, max: 0.83),
30+
'samsung_npu': Interval(min: 0.83, max: 0.85),
31+
};
32+
33+
const Map<String, Interval> _imageClassificationOfflineV2 = {
2134
'cpu': Interval(min: 0.88, max: 0.91),
22-
'npu': Interval(min: 0.69, max: 0.91),
23-
'tpu': Interval(min: 0.88, max: 0.91),
35+
'npu': Interval(min: 0.69, max: 0.71),
36+
'tpu': Interval(min: 0.89, max: 0.91),
2437
'ane': Interval(min: 0.69, max: 0.91),
2538
'cpu&gpu&ane': Interval(min: 0.69, max: 0.91),
2639
// TODO (anhappdev): check expected accuracy for snpe_dsp, psnpe_dsp
2740
'snpe_dsp': Interval(min: 0.0, max: 1.0),
2841
'psnpe_dsp': Interval(min: 0.0, max: 1.0),
29-
'neuron-mdla': Interval(min: 0.79, max: 0.91),
30-
'samsung_npu': Interval(min: 0.99, max: 1.0),
42+
'neuron-mdla': Interval(min: 0.79, max: 0.81),
43+
'samsung_npu': Interval(min: 0.98, max: 1.0),
3144
};
3245

3346
const Map<String, Interval> _objectDetection = {
@@ -105,5 +118,5 @@ const benchmarkExpectedAccuracy = {
105118
BenchmarkId.naturalLanguageProcessing: _naturalLanguageProcessing,
106119
BenchmarkId.superResolution: _superResolution,
107120
BenchmarkId.stableDiffusion: _stableDiffusion,
108-
BenchmarkId.imageClassificationOfflineV2: _imageClassificationV2,
121+
BenchmarkId.imageClassificationOfflineV2: _imageClassificationOfflineV2,
109122
};

0 commit comments

Comments
 (0)