@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 08287ef675
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Streamed tool calls now more strictly follow OpenAI's format; ensures Vercel AI SDK compatibility (#8272)

Kyle Mistele committed Sep 9, 2024 at 09:45 UTC 08287ef6751e79a89bf4f060f5f9545560a6de12
6 files changed +19 -44
tests/tool_use/utils.py
+1 -1
@@ -19,7 +19,7 @@ ARGS: List[str] = ["--enable-auto-tool-choice", "--max-model-len", "8096"]
19 CONFIGS: Dict[str, ServerConfig] = {
20 "hermes": {
21 "model":
22 - "NousResearch/Hermes-2-Pro-Llama-3-8B",
22 + "NousResearch/Hermes-3-Llama-3.1-8B",
23 "arguments": [
24 "--tool-call-parser", "hermes", "--chat-template",
25 str(VLLM_PATH / "examples/tool_chat_template_hermes.jinja")
vllm/entrypoints/openai/protocol.py
-7
@@ -713,13 +713,6 @@ class DeltaToolCall(OpenAIBaseModel):
713 function: Optional[DeltaFunctionCall] = None
714
715
716 -# the initial delta that gets sent once a new tool call is started;
717 -class InitialDeltaToolCall(DeltaToolCall):
718 - id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
719 - type: Literal["function"] = "function"
720 - index: int
721 -
722 -
716 class ExtractedToolCallInformation(BaseModel):
717 # indicate if tools were called
718 tools_called: bool
vllm/entrypoints/openai/serving_chat.py
+5 -1
@@ -271,9 +271,13 @@ class OpenAIServingChat(OpenAIServing):
271 # NOTE num_choices defaults to 1 so this usually executes
272 # once per request
273 for i in range(num_choices):
274 +
275 choice_data = ChatCompletionResponseStreamChoice(
276 index=i,
276 - delta=DeltaMessage(role=role),
277 + delta=DeltaMessage(
278 + role=role,
279 + content="",
280 + ),
281 logprobs=None,
282 finish_reason=None)
283 chunk = ChatCompletionStreamResponse(
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
-1
@@ -20,7 +20,6 @@ class ToolParser:
20 # the index of the tool call that is currently being parsed
21 self.current_tool_id: int = -1
22 self.current_tool_name_sent: bool = False
23 - self.current_tool_initial_sent: bool = False
23 self.streamed_args_for_tool: List[str] = []
24
25 self.model_tokenizer = tokenizer
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+5 -15
@@ -8,14 +8,14 @@ from partial_json_parser.core.options import Allow
8 from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
9 DeltaToolCall,
10 ExtractedToolCallInformation,
11 - FunctionCall,
12 - InitialDeltaToolCall, ToolCall)
11 + FunctionCall, ToolCall)
12 from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
13 ToolParser)
14 from vllm.entrypoints.openai.tool_parsers.utils import (
15 extract_intermediate_diff)
16 from vllm.logger import init_logger
17 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18 +from vllm.utils import random_uuid
19
20 logger = init_logger(__name__)
21
@@ -34,7 +34,6 @@ class Hermes2ProToolParser(ToolParser):
34 self.prev_tool_call_arr: List[Dict] = []
35 self.current_tool_id: int = -1
36 self.current_tool_name_sent = False
37 - self.current_tool_initial_sent: bool = False
37 self.streamed_args_for_tool: List[str] = [
38 ] # map what has been streamed for each tool so far to a list
39
@@ -168,7 +167,6 @@ class Hermes2ProToolParser(ToolParser):
167 # set cursors and state appropriately
168 self.current_tool_id += 1
169 self.current_tool_name_sent = False
171 - self.current_tool_initial_sent = False
170 self.streamed_args_for_tool.append("")
171 logger.debug("Starting on a new tool %s", self.current_tool_id)
172
@@ -218,24 +216,16 @@ class Hermes2ProToolParser(ToolParser):
216 logger.debug('not enough tokens to parse into JSON yet')
217 return None
218
221 - # case - we haven't sent the initial delta with the tool call ID
222 - # (it will be sent)
223 - if not self.current_tool_initial_sent:
224 - self.current_tool_initial_sent = True
225 - return DeltaMessage(tool_calls=[
226 - InitialDeltaToolCall(
227 - index=self.current_tool_id).model_dump(
228 - exclude_none=True)
229 - ])
230 -
219 # case - we haven't sent the tool name yet. If it's available, send
220 # it. otherwise, wait until it's available.
233 - elif not self.current_tool_name_sent:
221 + if not self.current_tool_name_sent:
222 function_name: Union[str, None] = current_tool_call.get("name")
223 if function_name:
224 self.current_tool_name_sent = True
225 return DeltaMessage(tool_calls=[
226 DeltaToolCall(index=self.current_tool_id,
227 + type="function",
228 + id=f"chatcmpl-tool-{random_uuid()}",
229 function=DeltaFunctionCall(
230 name=function_name).model_dump(
231 exclude_none=True))
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+8 -19
@@ -8,14 +8,14 @@ from partial_json_parser.core.options import Allow
8 from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
9 DeltaToolCall,
10 ExtractedToolCallInformation,
11 - FunctionCall,
12 - InitialDeltaToolCall, ToolCall)
11 + FunctionCall, ToolCall)
12 from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
13 ToolParser)
14 from vllm.entrypoints.openai.tool_parsers.utils import (
15 extract_intermediate_diff)
16 from vllm.logger import init_logger
17 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18 +from vllm.utils import random_uuid
19
20 logger = init_logger(__name__)
21
@@ -25,7 +25,7 @@ class MistralToolParser(ToolParser):
25 Tool call parser for Mistral 7B Instruct v0.3, intended for use with the
26 examples/tool_chat_template_mistral.jinja template.
27
28 - Used when --enable-auto-tool-choice --tool-call-parser gmistral are all set
28 + Used when --enable-auto-tool-choice --tool-call-parser mistral are all set
29 """
30
31 def __init__(self, tokenizer: AnyTokenizer):
@@ -42,7 +42,6 @@ class MistralToolParser(ToolParser):
42 self.prev_tool_call_arr: List[Dict] = []
43 self.current_tool_id: int = -1
44 self.current_tool_name_sent: bool = False
45 - self.current_tool_initial_sent: bool = False
45 self.streamed_args_for_tool: List[str] = [
46 ] # map what has been streamed for each tool so far to a list
47 self.bot_token = "[TOOL_CALLS]"
@@ -91,7 +90,6 @@ class MistralToolParser(ToolParser):
90
91 except Exception as e:
92 logger.error("Error in extracting tool call from response: %s", e)
94 - print("ERROR", e)
93 # return information to just treat the tool call as regular JSON
94 return ExtractedToolCallInformation(tools_called=False,
95 tool_calls=[],
@@ -109,7 +107,7 @@ class MistralToolParser(ToolParser):
107
108 # if the tool call token is not in the tokens generated so far, append
109 # output to contents since it's not a tool
112 - if self.bot_token_id not in current_token_ids:
110 + if self.bot_token not in current_text:
111 return DeltaMessage(content=delta_text)
112
113 # if the tool call token ID IS in the tokens generated so far, that
@@ -134,7 +132,7 @@ class MistralToolParser(ToolParser):
132 # replace BOT token with empty string, and convert single quotes
133 # to double to allow parsing as JSON since mistral uses single
134 # quotes instead of double for tool calls
137 - parsable_arr = current_text.split(self.bot_token)[1]
135 + parsable_arr = current_text.split(self.bot_token)[-1]
136
137 # tool calls are generated in an array, so do partial JSON
138 # parsing on the entire array
@@ -186,31 +184,22 @@ class MistralToolParser(ToolParser):
184 # re-set stuff pertaining to progress in the current tool
185 self.current_tool_id = len(tool_call_arr) - 1
186 self.current_tool_name_sent = False
189 - self.current_tool_initial_sent = False
187 self.streamed_args_for_tool.append("")
188 logger.debug("starting on new tool %d", self.current_tool_id)
189 return delta
190
191 # case: update an existing tool - this is handled below
192
196 - # if the current tool initial data incl. the id, type=function
197 - # and idx not sent, send that
198 - if not self.current_tool_initial_sent:
199 - self.current_tool_initial_sent = True
200 - delta = DeltaMessage(tool_calls=[
201 - InitialDeltaToolCall(
202 - index=self.current_tool_id).model_dump(
203 - exclude_none=True)
204 - ])
205 -
193 # if the current tool name hasn't been sent, send if available
194 # - otherwise send nothing
208 - elif not self.current_tool_name_sent:
195 + if not self.current_tool_name_sent:
196 function_name = current_tool_call.get("name")
197 if function_name:
198
199 delta = DeltaMessage(tool_calls=[
200 DeltaToolCall(index=self.current_tool_id,
201 + type="function",
202 + id=f"chatcmpl-tool-{random_uuid()}",
203 function=DeltaFunctionCall(
204 name=function_name).model_dump(
205 exclude_none=True))