perf: Fine-tune max tokens for lower latency conversation

clemlesne · clemlesne · commit 30ff57982019 · 2024-02-12T16:03:04.000+01:00
diff --git a/main.py b/main.py
@@ -1020,7 +1020,7 @@ async def _error_response() -> Tuple[CallModel, ActionModel]:
     tool_calls = {}
     try:
         async for delta in completion_stream(
-            max_tokens=400,
+            max_tokens=350,
             messages=messages,
             tools=tools,
         ):
