Why does my tirton service response time keep increasing at high QPM?

[CarlLinGitHub]

The vertical axis is ms, QPM=2000. Model work on the service is a xlm-roberta-large.onnx with dynamic batch_size and squence_length.

Here is the config.pbtxt.

`name: "xlm-roberta-large-onnx"
backend: "onnxruntime"
dynamic_batching {
}
max_batch_size: 16

input [
{
name: "input_ids"
data_type: TYPE_INT64
dims: [-1]
},
{
name: "attention_mask"
data_type: TYPE_INT64
dims: [-1]
}
]

output [
{
name: "embeds"
data_type: TYPE_FP32
dims: [512]
}
]

instance_group [
{
count: 1
kind: KIND_GPU
}
]

parameters {
key: "execution_mode"
value: { string_value: "1" }
}`

[JCwow]

Hi @CarlLinGitHub, Please try this

name: "xlm-roberta-large-onnx"
backend: "onnxruntime"
max_batch_size: 32

# Enhanced Dynamic Batching Configuration
dynamic_batching {
  preferred_batch_size: [ 4, 8, 16, 32 ]
  max_queue_delay_microseconds: 500
  default_queue_policy {
    timeout_action: REJECT
    default_timeout_microseconds: 10000000
    max_queue_size: 64
  }
}

input [
{
  name: "input_ids"
  data_type: TYPE_INT64
  dims: [-1]
},
{
  name: "attention_mask"
  data_type: TYPE_INT64
  dims: [-1]
}
]

output [
{
  name: "embeds"
  data_type: TYPE_FP32
  dims: [512]
}
]

# Multiple Instances for Higher Throughput
instance_group [
{
  count: 2
  kind: KIND_GPU
}
]

# ONNX Runtime Optimizations
parameters {
  key: "execution_mode"
  value: { string_value: "0" }  # ORT_PARALLEL instead of ORT_SEQUENTIAL
}
parameters {
  key: "graph_optimization_level"
  value: { string_value: "1" }  # ORT_ENABLE_EXTENDED
}
parameters {
  key: "intra_op_num_threads"
  value: { string_value: "0" }  # Use all available threads
}
parameters {
  key: "inter_op_num_threads"
  value: { string_value: "0" }  # Use all available threads
}

# TensorRT Optimization
optimization {
  execution_accelerators {
    gpu_execution_accelerator : [ {
      name : "tensorrt"
      parameters { key: "precision_mode" value: "FP16" }
      parameters { key: "max_workspace_size_bytes" value: "2147483648" }
      parameters { key: "trt_engine_cache_enable" value: "1" }
    }]
  }
}

# Model Warmup to Avoid Cold Start Penalties
model_warmup [
{
  name : "random_sample_1"
  batch_size: 1
  inputs {
    key: "input_ids"
    value: {
      data_type: TYPE_INT64
      dims: [ 128 ]
      random_data: true
    }
  }
  inputs {
    key: "attention_mask"  
    value: {
      data_type: TYPE_INT64
      dims: [ 128 ]
      random_data: true
    }
  }
},
{
  name : "random_sample_8"
  batch_size: 8
  inputs {
    key: "input_ids"
    value: {
      data_type: TYPE_INT64
      dims: [ 128 ]
      random_data: true
    }
  }
  inputs {
    key: "attention_mask"
    value: {
      data_type: TYPE_INT64
      dims: [ 128 ]
      random_data: true
    }
  }
},
{
  name : "random_sample_16"
  batch_size: 16
  inputs {
    key: "input_ids"
    value: {
      data_type: TYPE_INT64
      dims: [ 128 ]
      random_data: true
    }
  }
  inputs {
    key: "attention_mask"
    value: {
      data_type: TYPE_INT64
      dims: [ 128 ]
      random_data: true
    }
  }
}
] 

Let me know if this works better — I’d be happy to make adjustments.