adjust number of user for multi-turn chat

* Number of user  = Number of group * Number of prompts per group

Author: huaxig

README.md

@@ -22,7 +22,7 @@ Inference Perf is a GenAI inference performance benchmarking tool that allows yo
 * Supports benchmarking large deployments with frameworks like [llm-d](https://llm-d.ai/), [Dynamo](https://docs.nvidia.com/dynamo/latest/) and [Inference Gateway](https://gateway-api-inference-extension.sigs.k8s.io/).
 * Supports specifying an exact input and output distribution to simulate different scenarios - Gaussian distribution, fixed length, min-max cases are all supported.
 * Generates different load patterns and can benchmark specific cases like burst traffic, scaling to saturation and other autoscaling / routing scenarios.
-* Supprots Multi-turn chat conversations, it can keep context of a series of messages to simulate a conversation. A request in each chat round will keep previouse messages as prefix. see example [config-multi-turn](examples/vllm/config-shared-prefix-multi-turn.yml)
+* Supports Multi-turn chat conversations, it can keep context of a series of messages to simulate a conversation. A request in each chat round will keep previouse messages as prefix. see example [config-multi-turn](examples/vllm/config-shared-prefix-multi-turn.yml)
 
 ## Roadmap
 

examples/vllm/config-shared-prefix-multi-turn.yml

@@ -3,8 +3,8 @@ load:
   num_workers: 2
   worker_max_concurrency: 10
   stages:
-  - rate: 5
-    duration: 10
+  - rate: 20                      # Send all 20 users' requests per second
+    duration: 5
 api:
   type: completion
 server:
@@ -17,12 +17,12 @@ tokenizer:
 data:
   type: shared_prefix
   shared_prefix:
-    num_groups: 2                 # Number of distinct users
-    num_prompts_per_group: 25     # Number of unique questions per user
+    num_groups: 2                 # Number of distinct prefix, Note: the number of users is num_groups * num_prompts_per_group
+    num_prompts_per_group: 10     # Number of unique questions per group (prefix)
     system_prompt_len: 100        # Length of the first prefix (in tokens), simulate initialization of a system prompt
     question_len: 50              # Length of the unique question part (in tokens)
     output_len: 50                # Target length for the model's generated output (in tokens)
-    enable_multi_turn_chat: true  # enable multi-turn chat, create user session for each group. The chat context will be appended for the each request in the group.
+    enable_multi_turn_chat: true  # enable multi-turn chat, it will create user session to keep the conversation. The chat context will be appended for the each request.
 metrics:
   type: prometheus
   prometheus:

inference_perf/apis/user_session.py

@@ -74,3 +74,8 @@ async def process_failure(
         self.update_inference_info(inference_info)
         self.user_session.update_context(self._session_context)
         return inference_info
+
+
+# TODO: UserSessionChatAPIData need to be implemented
+# class UserSessionChatAPIData(ChatCompletionAPIData):
+#     ...

inference_perf/datagen/shared_prefix_datagen.py

@@ -65,8 +65,8 @@ def is_prefered_worker_requested(self) -> bool:
     def load_lazy_data(self, data: LazyLoadInferenceAPIData) -> InferenceAPIData:
         i = data.data_index % len(self.prompts)
         if self.enable_multi_turn_chat:
-            user_id = data.data_index % self.num_groups
-            round = data.data_index // self.num_groups
+            user_id = data.data_index % len(self.user_sessions)
+            round = data.data_index // len(self.user_sessions)
             return UserSessionCompletionAPIData(
                 prompt=self.prompts[i],
                 max_tokens=self.output_len,
@@ -106,26 +106,24 @@ def _generate_prompts(self) -> None:
             shared_prefix_token_ids = self._generate_random_token_ids(self.system_prompt_len)
             shared_prefix_text = hf_tokenizer.decode(shared_prefix_token_ids, skip_special_tokens=True)
 
-            if self.enable_multi_turn_chat:
-                # Create user session and store prefix as context (system prompt)
-                self.user_sessions.append(
-                    LocalUserSession(user_session_id=f"user_session_{group_id}", context=shared_prefix_text)
-                )
-                for _ in range(self.num_prompts_per_group):
-                    question_token_ids = self._generate_random_token_ids(self.question_len)
-                    question_text = hf_tokenizer.decode(question_token_ids, skip_special_tokens=True)
-                    # store question only as each round's prompt
-                    self.prompts.append(question_text)
-            else:
-                for _ in range(self.num_prompts_per_group):
-                    # Generate a unique question
-                    question_token_ids = self._generate_random_token_ids(self.question_len)
-                    question_text = hf_tokenizer.decode(question_token_ids, skip_special_tokens=True)
-
-                    # Combine shared prefix and question
-                    full_prompt_text = shared_prefix_text + " " + question_text
-
-                    self.prompts.append(full_prompt_text)
+            for prompt_id in range(self.num_prompts_per_group):
+                # Generate a unique question
+                question_token_ids = self._generate_random_token_ids(self.question_len)
+                question_text = hf_tokenizer.decode(question_token_ids, skip_special_tokens=True)
+
+                if self.enable_multi_turn_chat:
+                    # multi turn chat, create user to keep conversation
+                    self.user_sessions.append(
+                        LocalUserSession(
+                            user_session_id=f"user_session_{self.num_prompts_per_group * group_id + prompt_id}",
+                            context=shared_prefix_text,
+                        )
+                    )
+                else:
+                    # Single turn chat, Combine shared prefix and question
+                    question_text = shared_prefix_text + " " + question_text
+
+                self.prompts.append(question_text)
 
         # Shuffle the generated prompts to ensure randomness if served sequentially by different workers
         random.shuffle(self.prompts)