Skip to content

Commit a5ddc9d

Browse files
drbhdrbh
committed
feat: refactor and simplify chat stream more, bump tests and support stream_options
1 parent c4cb54c commit a5ddc9d

File tree

6 files changed

+264
-143
lines changed

6 files changed

+264
-143
lines changed

‎integration-tests/conftest.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,8 @@ def eq_chat_complete(response: ChatComplete, other: ChatComplete) -> bool:
269269
def eq_chat_complete_chunk(
270270
response: ChatCompletionChunk, other: ChatCompletionChunk
271271
) -> bool:
272+
if len(response.choices) == 0:
273+
return len(other.choices) == 0
272274
return response.choices[0].delta.content == other.choices[0].delta.content
273275

274276
def eq_response(response: Response, other: Response) -> bool:

‎integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_stream_usage.json‎

Lines changed: 39 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,11 @@
1212
"logprobs": null
1313
}
1414
],
15-
"created": 1726656043,
15+
"created": 1740516693,
1616
"id": "",
17-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
17+
"model": "meta-llama/Llama-3.1-8B-Instruct",
1818
"object": "chat.completion.chunk",
19-
"system_fingerprint": "2.2.1-dev0-native",
19+
"system_fingerprint": "3.1.1-dev0-native",
2020
"usage": null
2121
},
2222
{
@@ -32,11 +32,11 @@
3232
"logprobs": null
3333
}
3434
],
35-
"created": 1726656043,
35+
"created": 1740516693,
3636
"id": "",
37-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
37+
"model": "meta-llama/Llama-3.1-8B-Instruct",
3838
"object": "chat.completion.chunk",
39-
"system_fingerprint": "2.2.1-dev0-native",
39+
"system_fingerprint": "3.1.1-dev0-native",
4040
"usage": null
4141
},
4242
{
@@ -52,11 +52,11 @@
5252
"logprobs": null
5353
}
5454
],
55-
"created": 1726656043,
55+
"created": 1740516693,
5656
"id": "",
57-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
57+
"model": "meta-llama/Llama-3.1-8B-Instruct",
5858
"object": "chat.completion.chunk",
59-
"system_fingerprint": "2.2.1-dev0-native",
59+
"system_fingerprint": "3.1.1-dev0-native",
6060
"usage": null
6161
},
6262
{
@@ -72,11 +72,11 @@
7272
"logprobs": null
7373
}
7474
],
75-
"created": 1726656043,
75+
"created": 1740516694,
7676
"id": "",
77-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
77+
"model": "meta-llama/Llama-3.1-8B-Instruct",
7878
"object": "chat.completion.chunk",
79-
"system_fingerprint": "2.2.1-dev0-native",
79+
"system_fingerprint": "3.1.1-dev0-native",
8080
"usage": null
8181
},
8282
{
@@ -92,11 +92,11 @@
9292
"logprobs": null
9393
}
9494
],
95-
"created": 1726656043,
95+
"created": 1740516694,
9696
"id": "",
97-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
97+
"model": "meta-llama/Llama-3.1-8B-Instruct",
9898
"object": "chat.completion.chunk",
99-
"system_fingerprint": "2.2.1-dev0-native",
99+
"system_fingerprint": "3.1.1-dev0-native",
100100
"usage": null
101101
},
102102
{
@@ -112,11 +112,11 @@
112112
"logprobs": null
113113
}
114114
],
115-
"created": 1726656043,
115+
"created": 1740516694,
116116
"id": "",
117-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
117+
"model": "meta-llama/Llama-3.1-8B-Instruct",
118118
"object": "chat.completion.chunk",
119-
"system_fingerprint": "2.2.1-dev0-native",
119+
"system_fingerprint": "3.1.1-dev0-native",
120120
"usage": null
121121
},
122122
{
@@ -132,11 +132,11 @@
132132
"logprobs": null
133133
}
134134
],
135-
"created": 1726656044,
135+
"created": 1740516694,
136136
"id": "",
137-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
137+
"model": "meta-llama/Llama-3.1-8B-Instruct",
138138
"object": "chat.completion.chunk",
139-
"system_fingerprint": "2.2.1-dev0-native",
139+
"system_fingerprint": "3.1.1-dev0-native",
140140
"usage": null
141141
},
142142
{
@@ -152,11 +152,11 @@
152152
"logprobs": null
153153
}
154154
],
155-
"created": 1726656044,
155+
"created": 1740516694,
156156
"id": "",
157-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
157+
"model": "meta-llama/Llama-3.1-8B-Instruct",
158158
"object": "chat.completion.chunk",
159-
"system_fingerprint": "2.2.1-dev0-native",
159+
"system_fingerprint": "3.1.1-dev0-native",
160160
"usage": null
161161
},
162162
{
@@ -172,11 +172,11 @@
172172
"logprobs": null
173173
}
174174
],
175-
"created": 1726656044,
175+
"created": 1740516694,
176176
"id": "",
177-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
177+
"model": "meta-llama/Llama-3.1-8B-Instruct",
178178
"object": "chat.completion.chunk",
179-
"system_fingerprint": "2.2.1-dev0-native",
179+
"system_fingerprint": "3.1.1-dev0-native",
180180
"usage": null
181181
},
182182
{
@@ -192,11 +192,20 @@
192192
"logprobs": null
193193
}
194194
],
195-
"created": 1726656044,
195+
"created": 1740516694,
196196
"id": "",
197-
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
197+
"model": "meta-llama/Llama-3.1-8B-Instruct",
198198
"object": "chat.completion.chunk",
199-
"system_fingerprint": "2.2.1-dev0-native",
199+
"system_fingerprint": "3.1.1-dev0-native",
200+
"usage": null
201+
},
202+
{
203+
"choices": [],
204+
"created": 1740516694,
205+
"id": "",
206+
"model": "meta-llama/Llama-3.1-8B-Instruct",
207+
"object": "chat.completion.chunk",
208+
"system_fingerprint": "3.1.1-dev0-native",
200209
"usage": {
201210
"completion_tokens": 10,
202211
"prompt_tokens": 40,

‎integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_tool_reply_response.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,22 @@
55
"index": 0,
66
"logprobs": null,
77
"message": {
8-
"content": "I can't access real-time data, but I can provide you with current conditions and forecast for Paris, France:\n\nThe current conditions in Paris are mostly cloudy with a temperature of 6.7°C (44.1°F). \n\nPlease note that the actual weather may differ from this information, and I recommend checking the forecast on a reliable weather website for the most up-to-date information.",
8+
"content": "I can't access real-time data, but I can provide you with current conditions and forecast for Paris, France:\n\nThe current conditions in Paris are mostly cloudy with a temperature of 6.7°C (44.1°F). \n\nPlease note that the actual weather may differ from the provided information. For up-to-date information, I suggest checking a reliable weather website or app for the latest conditions and forecast.",
99
"name": null,
1010
"role": "assistant",
1111
"tool_calls": null
1212
},
1313
"usage": null
1414
}
1515
],
16-
"created": 1739932427,
16+
"created": 1740516945,
1717
"id": "",
1818
"model": "meta-llama/Llama-3.1-8B-Instruct",
1919
"object": "chat.completion",
2020
"system_fingerprint": "3.1.1-dev0-native",
2121
"usage": {
22-
"completion_tokens": 79,
23-
"prompt_tokens": 103,
24-
"total_tokens": 182
22+
"completion_tokens": 83,
23+
"prompt_tokens": 109,
24+
"total_tokens": 192
2525
}
2626
}

‎integration-tests/models/test_completion_prompts.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ async def test_flash_llama_completion_stream_usage(
9191
index = c["choices"][0]["index"]
9292
assert index == 0
9393
string += c["choices"][0]["delta"]["content"]
94-
94+
elif len(c["choices"]) == 0:
9595
has_usage = c["usage"] is not None
9696
assert not had_usage
9797
if has_usage:
@@ -142,7 +142,7 @@ async def test_flash_llama_completion_stream_usage(
142142
index = c["choices"][0]["index"]
143143
assert index == 0
144144
string += c["choices"][0]["delta"]["content"]
145-
145+
elif len(c["choices"]) == 0:
146146
has_usage = c["usage"] is not None
147147
assert not had_usage
148148
if has_usage:

‎integration-tests/models/test_tools_llama.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -497,7 +497,7 @@ async def test_flash_llama_tool_reply_response(
497497
assert responses.choices[0].message.tool_calls is None
498498
assert (
499499
responses.choices[0].message.content
500-
== "I can't access real-time data, but I can provide you with current conditions and forecast for Paris, France:\n\nThe current conditions in Paris are mostly cloudy with a temperature of 6.7°C (44.1°F). \n\nPlease note that the actual weather may differ from this information, and I recommend checking the forecast on a reliable weather website for the most up-to-date information."
500+
== "I can't access real-time data, but I can provide you with current conditions and forecast for Paris, France:\n\nThe current conditions in Paris are mostly cloudy with a temperature of 6.7°C (44.1°F). \n\nPlease note that the actual weather may differ from the provided information. For up-to-date information, I suggest checking a reliable weather website or app for the latest conditions and forecast."
501501
)
502502

503503
assert responses == response_snapshot

0 commit comments

Comments
 (0)