A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
Initialize the delta tool call fields explicitly (#17340)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: igmainc <igmainc@icloud.com>
Maximilien de Bayser committed
May 12, 2025 at 10:28 UTC
05a4324f8e3932c25554791ff248e3e0200eef92
12 files changed
+51
-34
tests/entrypoints/openai/tool_parsers/utils.py
+1
-1
@@ -32,7 +32,7 @@ class StreamingToolReconstructor:
32
assert len(delta.tool_calls) < 2, (
33
"Streaming should include only one tool call per update.")
34
for call_delta in delta.tool_calls:
35
- assert call_delta.type == "function", (
35
+ assert call_delta.type is None or call_delta.type == "function", (
36
"Streaming tool calls should only emit function calls. Got "
37
f"{call_delta.type}")
38
current_tool_call = self.tool_calls[
vllm/entrypoints/chat_utils.py
+4
@@ -44,6 +44,7 @@ from vllm.transformers_utils.chat_templates import (
44
# yapf: enable
45
from vllm.transformers_utils.processor import cached_get_processor
46
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
47
+from vllm.utils import random_uuid
48
49
logger = init_logger(__name__)
50
@@ -1272,3 +1273,6 @@ def apply_mistral_chat_template(
1273
"An error occurred in `mistral_common` while applying chat "
1274
"template")
1275
raise ValueError from e
1276
+
1277
+def random_tool_call_id() -> str:
1278
+ return f"chatcmpl-tool-{random_uuid()}"
\ No newline at end of file
vllm/entrypoints/openai/protocol.py
+5
-4
@@ -15,7 +15,8 @@ from pydantic import (BaseModel, ConfigDict, Field, TypeAdapter,
15
from typing_extensions import TypeAlias
16
17
from vllm import envs
18
-from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
18
+from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
19
+ random_tool_call_id)
20
from vllm.logger import init_logger
21
from vllm.pooling_params import PoolingParams
22
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
@@ -1339,7 +1340,7 @@ class FunctionCall(OpenAIBaseModel):
1340
1341
1342
class ToolCall(OpenAIBaseModel):
1342
- id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
1343
+ id: str = Field(default_factory=random_tool_call_id)
1344
type: Literal["function"] = "function"
1345
function: FunctionCall
1346
@@ -1351,8 +1352,8 @@ class DeltaFunctionCall(BaseModel):
1352
1353
# a tool call delta where everything is optional
1354
class DeltaToolCall(OpenAIBaseModel):
1354
- id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
1355
- type: Literal["function"] = "function"
1355
+ id: Optional[str] = None
1356
+ type: Optional[Literal["function"]] = None
1357
index: int
1358
function: Optional[DeltaFunctionCall] = None
1359
vllm/entrypoints/openai/serving_chat.py
+25
-14
@@ -16,7 +16,8 @@ from pydantic import TypeAdapter
16
from vllm.config import ModelConfig
17
from vllm.engine.protocol import EngineClient
18
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
19
- ConversationMessage)
19
+ ConversationMessage,
20
+ random_tool_call_id)
21
from vllm.entrypoints.logger import RequestLogger
22
from vllm.entrypoints.openai.protocol import (
23
ChatCompletionLogProb, ChatCompletionLogProbs,
@@ -363,9 +364,10 @@ class OpenAIServingChat(OpenAIServing):
364
365
function_name_returned = True
366
delta_message = DeltaMessage(tool_calls=[
366
- DeltaToolCall(function=DeltaFunctionCall(
367
- name=current_tool_call["name"],
368
- arguments=arguments),
367
+ DeltaToolCall(id=random_tool_call_id(),
368
+ function=DeltaFunctionCall(
369
+ name=current_tool_call["name"],
370
+ arguments=arguments),
371
index=len(obj) - 1,
372
type="function")
373
])
@@ -382,8 +384,7 @@ class OpenAIServingChat(OpenAIServing):
384
# instead of name every time
385
name=None,
386
arguments=delta_text),
385
- index=len(obj) - 1,
386
- type="function")
387
+ index=len(obj) - 1)
388
])
389
else:
390
delta_message = None
@@ -422,7 +423,7 @@ class OpenAIServingChat(OpenAIServing):
423
and self._should_stream_with_auto_tool_parsing(request))
424
425
all_previous_token_ids: Optional[list[list[int]]]
425
- function_name_returned: Optional[list[bool]] = None
426
+ function_name_returned = [False] * num_choices
427
428
# Only one of these will be used, thus previous_texts and
429
# all_previous_token_ids will not be used twice in the same iteration.
@@ -435,7 +436,6 @@ class OpenAIServingChat(OpenAIServing):
436
reasoning_end_arr = [False] * num_choices
437
elif request.tool_choice == "required":
438
previous_texts = [""] * num_choices
438
- function_name_returned = [False] * num_choices
439
all_previous_token_ids = None
440
else:
441
previous_texts, all_previous_token_ids = None, None
@@ -623,16 +623,27 @@ class OpenAIServingChat(OpenAIServing):
623
delta_text = previous_text + delta_text
624
current_text = ""
625
626
+ if function_name_returned[i]:
627
+ delta_tool_call = DeltaToolCall(
628
+ function=DeltaFunctionCall(
629
+ arguments=delta_text),
630
+ index=i)
631
+ else:
632
+ delta_tool_call = DeltaToolCall(
633
+ id=random_tool_call_id(),
634
+ type="function",
635
+ function=DeltaFunctionCall(
636
+ name=tool_choice_function_name,
637
+ arguments=delta_text),
638
+ index=i)
639
+ function_name_returned[i] = True
640
+
641
delta_message = DeltaMessage(tool_calls=[
627
- DeltaToolCall(function=DeltaFunctionCall(
628
- name=tool_choice_function_name,
629
- arguments=delta_text),
630
- index=i)
642
+ delta_tool_call,
643
])
644
645
elif request.tool_choice == "required":
646
assert previous_texts is not None
635
- assert function_name_returned is not None
647
previous_text = previous_texts[i]
648
current_text = previous_text + delta_text
649
fn_name_returned = function_name_returned[i]
@@ -835,7 +846,7 @@ class OpenAIServingChat(OpenAIServing):
846
total_tokens=num_prompt_tokens + completion_tokens,
847
)
848
838
- data = chunk.model_dump_json(exclude_unset=True)
849
+ data = chunk.model_dump_json(exclude_none=True)
850
yield f"data: {data}\n\n"
851
852
# once the final token is handled, if stream_options.include_usage
vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py
+2
-2
@@ -9,6 +9,7 @@ from typing import Union
9
import partial_json_parser
10
from partial_json_parser.core.options import Allow
11
12
+from vllm.entrypoints.chat_utils import random_tool_call_id
13
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14
DeltaFunctionCall, DeltaMessage,
15
DeltaToolCall,
@@ -22,7 +23,6 @@ from vllm.entrypoints.openai.tool_parsers.utils import (consume_space,
23
partial_json_loads)
24
from vllm.logger import init_logger
25
from vllm.transformers_utils.tokenizer import AnyTokenizer
25
-from vllm.utils import random_uuid
26
27
logger = init_logger(__name__)
28
@@ -200,7 +200,7 @@ class Granite20bFCToolParser(ToolParser):
200
delta = DeltaMessage(tool_calls=[
201
DeltaToolCall(index=self.current_tool_id,
202
type="function",
203
- id=f"chatcmpl-tool-{random_uuid()}",
203
+ id=random_tool_call_id(),
204
function=DeltaFunctionCall(
205
name=function_name).model_dump(
206
exclude_none=True))
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
+2
-2
@@ -7,6 +7,7 @@ from typing import Union
7
import partial_json_parser
8
from partial_json_parser.core.options import Allow
9
10
+from vllm.entrypoints.chat_utils import random_tool_call_id
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
DeltaFunctionCall, DeltaMessage,
13
DeltaToolCall,
@@ -20,7 +21,6 @@ from vllm.entrypoints.openai.tool_parsers.utils import (consume_space,
21
partial_json_loads)
22
from vllm.logger import init_logger
23
from vllm.transformers_utils.tokenizer import AnyTokenizer
23
-from vllm.utils import random_uuid
24
25
logger = init_logger(__name__)
26
@@ -182,7 +182,7 @@ class GraniteToolParser(ToolParser):
182
delta = DeltaMessage(tool_calls=[
183
DeltaToolCall(index=self.current_tool_id,
184
type="function",
185
- id=f"chatcmpl-tool-{random_uuid()}",
185
+ id=random_tool_call_id(),
186
function=DeltaFunctionCall(
187
name=function_name).model_dump(
188
exclude_none=True))
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+2
-2
@@ -8,6 +8,7 @@ from typing import Union
8
import partial_json_parser
9
from partial_json_parser.core.options import Allow
10
11
+from vllm.entrypoints.chat_utils import random_tool_call_id
12
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
13
DeltaFunctionCall, DeltaMessage,
14
DeltaToolCall,
@@ -17,7 +18,6 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
18
ToolParser, ToolParserManager)
19
from vllm.logger import init_logger
20
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
20
-from vllm.utils import random_uuid
21
22
logger = init_logger(__name__)
23
@@ -259,7 +259,7 @@ class Hermes2ProToolParser(ToolParser):
259
return DeltaMessage(tool_calls=[
260
DeltaToolCall(index=self.current_tool_id,
261
type="function",
262
- id=f"chatcmpl-tool-{random_uuid()}",
262
+ id=random_tool_call_id(),
263
function=DeltaFunctionCall(
264
name=function_name).model_dump(
265
exclude_none=True))
vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py
+2
-2
@@ -7,6 +7,7 @@ from typing import Union
7
import partial_json_parser
8
from partial_json_parser.core.options import Allow
9
10
+from vllm.entrypoints.chat_utils import random_tool_call_id
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
DeltaFunctionCall, DeltaMessage,
13
DeltaToolCall,
@@ -18,7 +19,6 @@ from vllm.entrypoints.openai.tool_parsers.utils import (
19
extract_intermediate_diff)
20
from vllm.logger import init_logger
21
from vllm.transformers_utils.tokenizer import AnyTokenizer
21
-from vllm.utils import random_uuid
22
23
logger = init_logger(__name__)
24
@@ -106,7 +106,7 @@ class Internlm2ToolParser(ToolParser):
106
delta = DeltaMessage(tool_calls=[
107
DeltaToolCall(index=self.current_tool_id,
108
type="function",
109
- id=f"chatcmpl-tool-{random_uuid()}",
109
+ id=random_tool_call_id(),
110
function=DeltaFunctionCall(
111
name=function_name).model_dump(
112
exclude_none=True))
vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py
+2
-2
@@ -8,6 +8,7 @@ from typing import Union
8
import partial_json_parser
9
from partial_json_parser.core.options import Allow
10
11
+from vllm.entrypoints.chat_utils import random_tool_call_id
12
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
13
DeltaFunctionCall, DeltaMessage,
14
DeltaToolCall,
@@ -19,7 +20,6 @@ from vllm.entrypoints.openai.tool_parsers.utils import (
20
from vllm.logger import init_logger
21
from vllm.transformers_utils.tokenizer import AnyTokenizer
22
from vllm.transformers_utils.tokenizers import MistralTokenizer
22
-from vllm.utils import random_uuid
23
24
logger = init_logger(__name__)
25
@@ -220,7 +220,7 @@ class JambaToolParser(ToolParser):
220
delta = DeltaMessage(tool_calls=[
221
DeltaToolCall(index=self.current_tool_id,
222
type="function",
223
- id=f"chatcmpl-tool-{random_uuid()}",
223
+ id=random_tool_call_id(),
224
function=DeltaFunctionCall(
225
name=function_name).model_dump(
226
exclude_none=True))
vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py
+2
-2
@@ -10,6 +10,7 @@ import partial_json_parser
10
from partial_json_parser.core.options import Allow
11
from transformers import PreTrainedTokenizerBase
12
13
+from vllm.entrypoints.chat_utils import random_tool_call_id
14
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
15
DeltaFunctionCall, DeltaMessage,
16
DeltaToolCall,
@@ -21,7 +22,6 @@ from vllm.entrypoints.openai.tool_parsers.utils import (find_common_prefix,
22
is_complete_json,
23
partial_json_loads)
24
from vllm.logger import init_logger
24
-from vllm.utils import random_uuid
25
26
logger = init_logger(__name__)
27
@@ -208,7 +208,7 @@ class Llama3JsonToolParser(ToolParser):
208
delta = DeltaMessage(tool_calls=[
209
DeltaToolCall(index=self.current_tool_id,
210
type="function",
211
- id=f"chatcmpl-tool-{random_uuid()}",
211
+ id=random_tool_call_id(),
212
function=DeltaFunctionCall(
213
name=function_name).model_dump(
214
exclude_none=True))
vllm/entrypoints/openai/tool_parsers/phi4mini_tool_parser.py
+2
-2
@@ -7,6 +7,7 @@ from typing import Any, Optional
7
8
from transformers import PreTrainedTokenizerBase
9
10
+from vllm.entrypoints.chat_utils import random_tool_call_id
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
DeltaMessage,
13
ExtractedToolCallInformation,
@@ -14,7 +15,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
15
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
16
ToolParser, ToolParserManager)
17
from vllm.logger import init_logger
17
-from vllm.utils import random_uuid
18
19
logger = init_logger(__name__)
20
@@ -73,7 +73,7 @@ class Phi4MiniJsonToolParser(ToolParser):
73
74
tool_calls: list[ToolCall] = [
75
ToolCall(
76
- id=f"chatcmpl-tool-{random_uuid()}",
76
+ id=random_tool_call_id(),
77
type="function",
78
function=FunctionCall(
79
name=raw_function_call["name"],
vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py
+2
-1
@@ -280,6 +280,7 @@ def _compute_tool_delta(previously_sent_args: str, new_call: ToolCall,
280
new_call_args = new_call_args[:-len(withheld_suffix)]
281
if not previously_sent_args:
282
return DeltaToolCall(id=new_call.id,
283
+ type="function",
284
index=index,
285
function=DeltaFunctionCall(
286
name=new_call.function.name,
@@ -288,5 +289,5 @@ def _compute_tool_delta(previously_sent_args: str, new_call: ToolCall,
289
290
arg_diff = new_call_args[len(previously_sent_args):]
291
return DeltaToolCall(
291
- id="", index=index, function=DeltaFunctionCall(
292
+ id=None, index=index, function=DeltaFunctionCall(
293
arguments=arg_diff)) if arg_diff else None