A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Streamed tool calls now more strictly follow OpenAI's format; ensures Vercel AI SDK compatibility (#8272)
Kyle Mistele committed
Sep 9, 2024 at 09:45 UTC
08287ef6751e79a89bf4f060f5f9545560a6de12
6 files changed
+19
-44
tests/tool_use/utils.py
+1
-1
@@ -19,7 +19,7 @@ ARGS: List[str] = ["--enable-auto-tool-choice", "--max-model-len", "8096"]
19
CONFIGS: Dict[str, ServerConfig] = {
20
"hermes": {
21
"model":
22
- "NousResearch/Hermes-2-Pro-Llama-3-8B",
22
+ "NousResearch/Hermes-3-Llama-3.1-8B",
23
"arguments": [
24
"--tool-call-parser", "hermes", "--chat-template",
25
str(VLLM_PATH / "examples/tool_chat_template_hermes.jinja")
vllm/entrypoints/openai/protocol.py
-7
@@ -713,13 +713,6 @@ class DeltaToolCall(OpenAIBaseModel):
713
function: Optional[DeltaFunctionCall] = None
714
715
716
-# the initial delta that gets sent once a new tool call is started;
717
-class InitialDeltaToolCall(DeltaToolCall):
718
- id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
719
- type: Literal["function"] = "function"
720
- index: int
721
-
722
-
716
class ExtractedToolCallInformation(BaseModel):
717
# indicate if tools were called
718
tools_called: bool
vllm/entrypoints/openai/serving_chat.py
+5
-1
@@ -271,9 +271,13 @@ class OpenAIServingChat(OpenAIServing):
271
# NOTE num_choices defaults to 1 so this usually executes
272
# once per request
273
for i in range(num_choices):
274
+
275
choice_data = ChatCompletionResponseStreamChoice(
276
index=i,
276
- delta=DeltaMessage(role=role),
277
+ delta=DeltaMessage(
278
+ role=role,
279
+ content="",
280
+ ),
281
logprobs=None,
282
finish_reason=None)
283
chunk = ChatCompletionStreamResponse(
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
-1
@@ -20,7 +20,6 @@ class ToolParser:
20
# the index of the tool call that is currently being parsed
21
self.current_tool_id: int = -1
22
self.current_tool_name_sent: bool = False
23
- self.current_tool_initial_sent: bool = False
23
self.streamed_args_for_tool: List[str] = []
24
25
self.model_tokenizer = tokenizer
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+5
-15
@@ -8,14 +8,14 @@ from partial_json_parser.core.options import Allow
8
from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
9
DeltaToolCall,
10
ExtractedToolCallInformation,
11
- FunctionCall,
12
- InitialDeltaToolCall, ToolCall)
11
+ FunctionCall, ToolCall)
12
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
13
ToolParser)
14
from vllm.entrypoints.openai.tool_parsers.utils import (
15
extract_intermediate_diff)
16
from vllm.logger import init_logger
17
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18
+from vllm.utils import random_uuid
19
20
logger = init_logger(__name__)
21
@@ -34,7 +34,6 @@ class Hermes2ProToolParser(ToolParser):
34
self.prev_tool_call_arr: List[Dict] = []
35
self.current_tool_id: int = -1
36
self.current_tool_name_sent = False
37
- self.current_tool_initial_sent: bool = False
37
self.streamed_args_for_tool: List[str] = [
38
] # map what has been streamed for each tool so far to a list
39
@@ -168,7 +167,6 @@ class Hermes2ProToolParser(ToolParser):
167
# set cursors and state appropriately
168
self.current_tool_id += 1
169
self.current_tool_name_sent = False
171
- self.current_tool_initial_sent = False
170
self.streamed_args_for_tool.append("")
171
logger.debug("Starting on a new tool %s", self.current_tool_id)
172
@@ -218,24 +216,16 @@ class Hermes2ProToolParser(ToolParser):
216
logger.debug('not enough tokens to parse into JSON yet')
217
return None
218
221
- # case - we haven't sent the initial delta with the tool call ID
222
- # (it will be sent)
223
- if not self.current_tool_initial_sent:
224
- self.current_tool_initial_sent = True
225
- return DeltaMessage(tool_calls=[
226
- InitialDeltaToolCall(
227
- index=self.current_tool_id).model_dump(
228
- exclude_none=True)
229
- ])
230
-
219
# case - we haven't sent the tool name yet. If it's available, send
220
# it. otherwise, wait until it's available.
233
- elif not self.current_tool_name_sent:
221
+ if not self.current_tool_name_sent:
222
function_name: Union[str, None] = current_tool_call.get("name")
223
if function_name:
224
self.current_tool_name_sent = True
225
return DeltaMessage(tool_calls=[
226
DeltaToolCall(index=self.current_tool_id,
227
+ type="function",
228
+ id=f"chatcmpl-tool-{random_uuid()}",
229
function=DeltaFunctionCall(
230
name=function_name).model_dump(
231
exclude_none=True))
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+8
-19
@@ -8,14 +8,14 @@ from partial_json_parser.core.options import Allow
8
from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
9
DeltaToolCall,
10
ExtractedToolCallInformation,
11
- FunctionCall,
12
- InitialDeltaToolCall, ToolCall)
11
+ FunctionCall, ToolCall)
12
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
13
ToolParser)
14
from vllm.entrypoints.openai.tool_parsers.utils import (
15
extract_intermediate_diff)
16
from vllm.logger import init_logger
17
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18
+from vllm.utils import random_uuid
19
20
logger = init_logger(__name__)
21
@@ -25,7 +25,7 @@ class MistralToolParser(ToolParser):
25
Tool call parser for Mistral 7B Instruct v0.3, intended for use with the
26
examples/tool_chat_template_mistral.jinja template.
27
28
- Used when --enable-auto-tool-choice --tool-call-parser gmistral are all set
28
+ Used when --enable-auto-tool-choice --tool-call-parser mistral are all set
29
"""
30
31
def __init__(self, tokenizer: AnyTokenizer):
@@ -42,7 +42,6 @@ class MistralToolParser(ToolParser):
42
self.prev_tool_call_arr: List[Dict] = []
43
self.current_tool_id: int = -1
44
self.current_tool_name_sent: bool = False
45
- self.current_tool_initial_sent: bool = False
45
self.streamed_args_for_tool: List[str] = [
46
] # map what has been streamed for each tool so far to a list
47
self.bot_token = "[TOOL_CALLS]"
@@ -91,7 +90,6 @@ class MistralToolParser(ToolParser):
90
91
except Exception as e:
92
logger.error("Error in extracting tool call from response: %s", e)
94
- print("ERROR", e)
93
# return information to just treat the tool call as regular JSON
94
return ExtractedToolCallInformation(tools_called=False,
95
tool_calls=[],
@@ -109,7 +107,7 @@ class MistralToolParser(ToolParser):
107
108
# if the tool call token is not in the tokens generated so far, append
109
# output to contents since it's not a tool
112
- if self.bot_token_id not in current_token_ids:
110
+ if self.bot_token not in current_text:
111
return DeltaMessage(content=delta_text)
112
113
# if the tool call token ID IS in the tokens generated so far, that
@@ -134,7 +132,7 @@ class MistralToolParser(ToolParser):
132
# replace BOT token with empty string, and convert single quotes
133
# to double to allow parsing as JSON since mistral uses single
134
# quotes instead of double for tool calls
137
- parsable_arr = current_text.split(self.bot_token)[1]
135
+ parsable_arr = current_text.split(self.bot_token)[-1]
136
137
# tool calls are generated in an array, so do partial JSON
138
# parsing on the entire array
@@ -186,31 +184,22 @@ class MistralToolParser(ToolParser):
184
# re-set stuff pertaining to progress in the current tool
185
self.current_tool_id = len(tool_call_arr) - 1
186
self.current_tool_name_sent = False
189
- self.current_tool_initial_sent = False
187
self.streamed_args_for_tool.append("")
188
logger.debug("starting on new tool %d", self.current_tool_id)
189
return delta
190
191
# case: update an existing tool - this is handled below
192
196
- # if the current tool initial data incl. the id, type=function
197
- # and idx not sent, send that
198
- if not self.current_tool_initial_sent:
199
- self.current_tool_initial_sent = True
200
- delta = DeltaMessage(tool_calls=[
201
- InitialDeltaToolCall(
202
- index=self.current_tool_id).model_dump(
203
- exclude_none=True)
204
- ])
205
-
193
# if the current tool name hasn't been sent, send if available
194
# - otherwise send nothing
208
- elif not self.current_tool_name_sent:
195
+ if not self.current_tool_name_sent:
196
function_name = current_tool_call.get("name")
197
if function_name:
198
199
delta = DeltaMessage(tool_calls=[
200
DeltaToolCall(index=self.current_tool_id,
201
+ type="function",
202
+ id=f"chatcmpl-tool-{random_uuid()}",
203
function=DeltaFunctionCall(
204
name=function_name).model_dump(
205
exclude_none=True))