mlperf_task.proto

/* Copyright 2019 The MLPerf Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
syntax = "proto2";

package mlperf.mobile;

option cc_enable_arenas = true;
option java_multiple_files = true;
option java_package = "org.mlperf.proto";

// App config
//
// Next ID: 2
message MLPerfConfig {
  // All tasks will be run.
  repeated TaskConfig task = 1;
}

// Config of the mlperf tasks.
// A task is basically a combination of models and a dataset.
//
// Next ID: 13
message TaskConfig {
  // Must be unique in one task file. Ex: image_classification
  // used to match backend settings
  required string id = 1;
  // Human-readable name. Ex: Image classification.
  required string name = 2;
  // Max expected throughput score
  required float max_throughput = 5;
  // Max expected accuracy
  required float max_accuracy = 6;
  // LoadGen parameter. Allowed values: SingleStream, Offline
  required string scenario = 7;
  required RunConfig runs = 12;
  required DatasetConfig datasets = 8;
  required ModelConfig model = 9;
  repeated CustomConfig custom_config = 11;
}

// Run configurations
message RunConfig {
  required OneRunConfig normal = 1;
  required OneRunConfig quick = 2;
  required OneRunConfig rapid = 3;
}

// Config of one run
message OneRunConfig {
  // Minimum number of samples the test should run in the performance mode.
  required int32 min_query_count = 3;
  // Minimum duration the test should run in the performance mode, in seconds.
  required double min_duration = 4 [default = 60];
  // Maximum duration the test should run in the performance mode, in seconds.
  required double max_duration = 10 [default = 600];
}

// Datasets for a task
//
// Next ID: 5
message DatasetConfig {
  // Type of the dataset.
  enum DatasetType {
    NONE = 0;
    IMAGENET = 1;
    COCO = 2;
    SQUAD = 3;
    ADE20K = 4;
    SNUSR = 5;
    COCOGEN = 6;
  }
  required DatasetType type = 1;
  // Config of the dataset.
  // full is used for Accuracy run.
  required OneDatasetConfig full = 2;
  // lite is used for Performance run.
  required OneDatasetConfig lite = 3;
  // tiny is used for unit test.
  required OneDatasetConfig tiny = 4;
}

// Config of one dataset
//
// Next ID: 4
message OneDatasetConfig {
  // Human-readable name of the dataset
  required string name = 1;
  // URL or local path to dataset input file
  required string input_path = 2;
  // MD5 checksum to validate the input file
  required string input_checksum = 4;
  // URL or local path to dataset groundtruth file
  required string groundtruth_path = 3;
  // MD5 checksum to validate the groundtruth file
  required string groundtruth_checksum = 5;
}

// Config of a model.
//
// Next ID: 7
message ModelConfig {
  // unique ID used by UI to find matching description
  required string id = 1;
  // Human-readable name of the model.
  required string name = 2;
  // Offset value of the model if applicable.
  optional int32 offset = 3 [default = 0];
  // image width of the input (if model takes images)
  optional int32 image_width = 4;
  // image height of the input (if model takes images)
  optional int32 image_height = 5;
  // Number of detection classes if applicable
  optional int32 num_classes = 6;
}

// CustomConfig are task specific configuration.
// The TaskConfig.CustomConfig will be converted to
// BenchmarkSetting.CustomSetting and passed to the backend.
// To avoid name collision, the id should be prefixed with TaskConfig.id.
message CustomConfig {
  // Id of this config.
  required string id = 1;
  // Value of this config.
  required string value = 2;
}