Skip to content

Pr 2954 ci branch #3006

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 22 commits into from
Closed

Pr 2954 ci branch #3006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
9a9a763
fix: Adapt function call response to return a json string for arguments
Trofleb Jan 24, 2025
8542e2b
feat: Make streaming for tool calling behave the same as the open ai api
Trofleb Jan 24, 2025
c1c4dfb
fix: Allow back arguments in function definition and the correspondin…
Trofleb Feb 4, 2025
983b967
fix: Functioncall is actually a bit different than the deprecated fun…
Trofleb Feb 4, 2025
0ca7af8
feat: serialize function definition with serialize_as_string
drbh Feb 7, 2025
5f03014
fix: bump openapi spec
drbh Feb 10, 2025
dbce04e
fix: adjust streaming tool response
drbh Feb 11, 2025
07c2090
fix: ensure wrapping curly is not included
drbh Feb 11, 2025
40f905d
fix: adjust stream, improve tests and add openai client test
drbh Feb 17, 2025
1529a67
fix: remove snap with incorrect naming
drbh Feb 17, 2025
c215c0d
fix: bump client test expected prefill
drbh Feb 17, 2025
7d17d7c
fix: bump client tests for api changes
drbh Feb 17, 2025
fd611f3
fix: bump integration test deps for openai
drbh Feb 17, 2025
68aa6b1
fix: bump requirements file too
drbh Feb 17, 2025
538456b
fix: only send function name on first stream event
drbh Feb 18, 2025
e1b6d5b
fix: clippy cleanup
drbh Feb 18, 2025
a416ddb
fix: adjust integration tests for openai client dep
drbh Feb 19, 2025
31a536d
feat: refactor chat stream to remove state machine and simplfy logic
drbh Feb 24, 2025
c4cb54c
fix: bump integrations requirements
drbh Feb 24, 2025
a5ddc9d
feat: refactor and simplify chat stream more, bump tests and support …
drbh Feb 25, 2025
330f2e4
feat: improve partial parsing types and add test for balancing and pa…
drbh Feb 26, 2025
efb2005
feat: consolidate streaming and event creation logic and add tests fo…
drbh Feb 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions clients/python/tests/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ def test_generate(llama_7b_url, hf_headers):
assert response.details.generated_tokens == 1
assert response.details.seed is None
assert len(response.details.prefill) == 0
# assert response.details.prefill[0] == InputToken(id=1, text="<s>", logprob=None)
assert len(response.details.tokens) == 1
assert response.details.tokens[0].id == 29918
assert response.details.tokens[0].text == "_"
Expand Down Expand Up @@ -84,10 +83,6 @@ async def test_generate_async(llama_7b_url, hf_headers):
assert response.details.generated_tokens == 1
assert response.details.seed is None
assert len(response.details.prefill) == 0
# assert response.details.prefill[0] == InputToken(id=1, text="<s>", logprob=None)
# assert response.details.prefill[1] == InputToken(
# id=1243, text="test", logprob=-10.96875
# )
assert len(response.details.tokens) == 1
assert response.details.tokens[0].id == 29918
assert response.details.tokens[0].text == "_"
Expand Down
4 changes: 2 additions & 2 deletions clients/python/text_generation/types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ class ChoiceDeltaToolCall(BaseModel):
class ChoiceDelta(BaseModel):
role: str
content: Optional[str] = None
tool_calls: Optional[ChoiceDeltaToolCall] = None
tool_calls: Optional[List[ChoiceDeltaToolCall]] = None


class Choice(BaseModel):
Expand Down Expand Up @@ -464,4 +464,4 @@ class DeployedModel(BaseModel):
# https://github.com/pydantic/pydantic/issues/9177
model_config = ConfigDict(protected_namespaces=())
model_id: str
sha: str
task: str
5 changes: 4 additions & 1 deletion docs/openapi.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2302,7 +2302,10 @@
"example": "assistant"
},
"tool_calls": {
"$ref": "#/components/schemas/DeltaToolCall"
"type": "array",
"items": {
"$ref": "#/components/schemas/DeltaToolCall"
}
}
}
},
Expand Down
2 changes: 2 additions & 0 deletions integration-tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,8 @@ def eq_chat_complete(response: ChatComplete, other: ChatComplete) -> bool:
def eq_chat_complete_chunk(
response: ChatCompletionChunk, other: ChatCompletionChunk
) -> bool:
if len(response.choices) == 0:
return len(other.choices) == 0
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we to tool_call check instead?

return response.choices[0].delta.content == other.choices[0].delta.content

def eq_response(response: Response, other: Response) -> bool:
Expand Down
69 changes: 39 additions & 30 deletions ...odels/__snapshots__/test_completion_prompts/test_flash_llama_completion_stream_usage.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
"logprobs": null
}
],
"created": 1726656043,
"created": 1740516693,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
Expand All @@ -32,11 +32,11 @@
"logprobs": null
}
],
"created": 1726656043,
"created": 1740516693,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
Expand All @@ -52,11 +52,11 @@
"logprobs": null
}
],
"created": 1726656043,
"created": 1740516693,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
Expand All @@ -72,11 +72,11 @@
"logprobs": null
}
],
"created": 1726656043,
"created": 1740516694,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
Expand All @@ -92,11 +92,11 @@
"logprobs": null
}
],
"created": 1726656043,
"created": 1740516694,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
Expand All @@ -112,11 +112,11 @@
"logprobs": null
}
],
"created": 1726656043,
"created": 1740516694,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
Expand All @@ -132,11 +132,11 @@
"logprobs": null
}
],
"created": 1726656044,
"created": 1740516694,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
Expand All @@ -152,11 +152,11 @@
"logprobs": null
}
],
"created": 1726656044,
"created": 1740516694,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
Expand All @@ -172,11 +172,11 @@
"logprobs": null
}
],
"created": 1726656044,
"created": 1740516694,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
Expand All @@ -192,11 +192,20 @@
"logprobs": null
}
],
"created": 1726656044,
"created": 1740516694,
"id": "",
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.2.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
},
{
"choices": [],
"created": 1740516694,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.1-dev0-native",
"usage": {
"completion_tokens": 10,
"prompt_tokens": 40,
Expand Down
29 changes: 29 additions & 0 deletions integration-tests/models/__snapshots__/test_openai_llama_tools/test_openai_llama_tools.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"choices": [
{
"delta": {
"role": "assistant",
"tool_calls": [
{
"function": {
"arguments": "\"}",
"name": "get_current_weather"
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1739910558,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
}
10 changes: 5 additions & 5 deletions ...snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,22 @@
"index": 0,
"logprobs": null,
"message": {
"content": "I am an AI assistant",
"content": "I am a helpful assistant!",
"name": null,
"role": "assistant",
"tool_calls": null
},
"usage": null
}
],
"created": 1728497062,
"created": 1739357385,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion",
"system_fingerprint": "2.4.2-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": {
"completion_tokens": 23,
"prompt_tokens": 604,
"total_tokens": 627
"prompt_tokens": 494,
"total_tokens": 517
}
}
6 changes: 3 additions & 3 deletions ...ts__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information_stream.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"choices": [
{
"delta": {
"content": " assistant",
"content": "!",
"role": "assistant",
"tool_calls": null
},
Expand All @@ -11,10 +11,10 @@
"logprobs": null
}
],
"created": 1728497531,
"created": 1740432006,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.4.2-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
}
6 changes: 3 additions & 3 deletions ...s/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"choices": [
{
"delta": {
"content": " fans",
"content": ".",
"role": "assistant",
"tool_calls": null
},
Expand All @@ -11,10 +11,10 @@
"logprobs": null
}
],
"created": 1728497461,
"created": 1740432012,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.4.2-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
}
26 changes: 14 additions & 12 deletions ...test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream_function_object.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,27 @@
{
"delta": {
"role": "assistant",
"tool_calls": {
"function": {
"arguments": "<|eot_id|>",
"name": null
},
"id": "",
"index": 0,
"type": "function"
}
"tool_calls": [
{
"function": {
"arguments": "}",
"name": "get_n_day_weather_forecast"
},
"id": "",
"index": 0,
"type": "function"
}
]
},
"finish_reason": "stop",
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 1732293254,
"created": 1739910826,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.4.1-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
}
4 changes: 2 additions & 2 deletions ...napshots__/test_tools_llama/test_flash_llama_grammar_tools_sea_creatures_stream_none.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
"logprobs": null
}
],
"created": 1729262528,
"created": 1740433572,
"id": "",
"model": "meta-llama/Llama-3.1-8B-Instruct",
"object": "chat.completion.chunk",
"system_fingerprint": "2.3.2-dev0-native",
"system_fingerprint": "3.1.1-dev0-native",
"usage": null
}
Loading
Loading