A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Feature] Support DeepSeekV3 Function Call (#17784)
Signed-off-by: 许文卿 <xwq391974@alibaba-inc.com> Signed-off-by: Xu Wenqing <xuwq1993@qq.com>
Xu Wenqing committed
May 12, 2025 at 15:45 UTC
3a5ea751292664265bdd0dd22da86d725e457816
4 files changed
+473
-1
docs/source/features/tool_calling.md
+7
@@ -236,6 +236,13 @@ For Qwen2.5, the chat template in tokenizer_config.json has already included sup
236
237
Flags: `--tool-call-parser hermes`
238
239
+### DeepSeek-V3 Models (`deepseek_v3`)
240
+
241
+Supported models:
242
+* `deepseek-ai/DeepSeek-V3-0324`
243
+
244
+Flags: `--tool-call-parser deepseek_v3 --chat-template examples/tool_chat_template_deepseekv3.jinja`
245
+
246
### Models with Pythonic Tool Calls (`pythonic`)
247
248
A growing number of models output a python list to represent tool calls instead of using JSON. This has the advantage of inherently supporting parallel tool calls and removing ambiguity around the JSON schema required for tool calls. The `pythonic` tool parser can support such models.
examples/tool_chat_template_deepseekv3.jinja
new
+96
@@ -0,0 +1,96 @@
1
+{% if not add_generation_prompt is defined %}
2
+ {% set add_generation_prompt = false %}
3
+{% endif %}
4
+
5
+{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %}
6
+
7
+{%- for message in messages %}
8
+ {%- if message['role'] == 'system' %}
9
+ {%- if ns.is_first_sp %}
10
+ {% set ns.system_prompt = ns.system_prompt + message['content'] %}
11
+ {% set ns.is_first_sp = false %}
12
+ {%- else %}
13
+ {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
14
+ {%- endif %}
15
+ {%- endif %}
16
+{%- endfor %}
17
+
18
+{{ bos_token }}
19
+{{ ns.system_prompt }}
20
+{%- if tools %}
21
+ {{"\n\n# Tools\n\nYou may call one or more functions to assist with the user query." }}
22
+ {%- for tool in tools %}
23
+ {{- "\n" }}
24
+ {{- tool | tojson }}
25
+ {%- endfor %}
26
+ {{"\n</tools>\n\n"}}
27
+
28
+ {{"For function call returns, you should first print <｜tool▁calls▁begin｜>"}}
29
+
30
+ {{"For each function call, you should return object like:\n" }}
31
+ {{"<｜tool▁call▁begin｜>function<｜tool▁sep｜><function_name>\n```json\n<function_arguments_in_json_format>\n```<｜tool▁call▁end｜>"}}
32
+
33
+ {{"At the end of function call returns, you should print <｜tool▁calls▁end｜><｜end▁of▁sentence｜>"}}
34
+{%- endif %}
35
+
36
+{%- for message in messages %}
37
+ {%- if message['role'] == 'user' %}
38
+ {%- set ns.is_tool = false -%}
39
+ {%- set ns.is_first = false -%}
40
+ {%- set ns.is_last_user = true -%}
41
+ {{'<｜User｜>' + message['content'] + '<｜Assistant｜>'}}
42
+ {%- endif %}
43
+
44
+ {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
45
+ {%- set ns.is_last_user = false -%}
46
+ {%- if ns.is_tool %}
47
+ {{'<｜tool▁outputs▁end｜>'}}
48
+ {%- endif %}
49
+ {%- set ns.is_first = false %}
50
+ {%- set ns.is_tool = false -%}
51
+ {%- set ns.is_output_first = true %}
52
+
53
+ {%- for tool in message['tool_calls'] %}
54
+ {%- if not ns.is_first %}
55
+ {%- if message['content'] is none %}
56
+ {{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<｜tool▁call▁end｜>'}}
57
+ {%- else %}
58
+ {{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<｜tool▁call▁end｜>'}}
59
+ {%- endif %}
60
+ {%- set ns.is_first = true -%}
61
+ {%- else %}
62
+ {{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<｜tool▁call▁end｜>'}}
63
+ {%- endif %}
64
+ {%- endfor %}
65
+ {{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}
66
+ {%- endif %}
67
+ {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%}
68
+ {%- set ns.is_last_user = false -%}
69
+ {%- if ns.is_tool %}
70
+ {{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}
71
+ {%- set ns.is_tool = false -%}
72
+ {%- else %}
73
+ {% set content = message['content'] %}
74
+ {{content + '<｜end▁of▁sentence｜>'}}
75
+ {%- endif %}
76
+ {%- endif %}
77
+
78
+ {%- if message['role'] == 'tool' %}
79
+ {%- set ns.is_last_user = false -%}
80
+ {%- set ns.is_tool = true -%}
81
+ {%- if ns.is_output_first %}
82
+ {{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
83
+ {%- set ns.is_output_first = false %}
84
+ {%- else %}
85
+ {{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
86
+ {%- endif %}
87
+ {%- endif %}
88
+{%- endfor -%}
89
+
90
+{% if ns.is_tool %}
91
+ {{'<｜tool▁outputs▁end｜>'}}
92
+{% endif %}
93
+
94
+{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %}
95
+ {{'<｜Assistant｜>'}}
96
+{% endif %}
vllm/entrypoints/openai/tool_parsers/__init__.py
+2
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
from .abstract_tool_parser import ToolParser, ToolParserManager
4
+from .deepseekv3_tool_parser import DeepSeekV3ToolParser
5
from .granite_20b_fc_tool_parser import Granite20bFCToolParser
6
from .granite_tool_parser import GraniteToolParser
7
from .hermes_tool_parser import Hermes2ProToolParser
@@ -15,5 +16,5 @@ __all__ = [
16
"ToolParser", "ToolParserManager", "Granite20bFCToolParser",
17
"GraniteToolParser", "Hermes2ProToolParser", "MistralToolParser",
18
"Internlm2ToolParser", "Llama3JsonToolParser", "JambaToolParser",
18
- "PythonicToolParser", "Phi4MiniJsonToolParser"
19
+ "PythonicToolParser", "Phi4MiniJsonToolParser", "DeepSeekV3ToolParser"
20
]
vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py
new
+368
@@ -0,0 +1,368 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import re
4
+from collections.abc import Sequence
5
+from typing import Union
6
+
7
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
8
+ DeltaFunctionCall, DeltaMessage,
9
+ DeltaToolCall,
10
+ ExtractedToolCallInformation,
11
+ FunctionCall, ToolCall)
12
+from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
13
+ ToolParser, ToolParserManager)
14
+from vllm.logger import init_logger
15
+from vllm.transformers_utils.tokenizer import AnyTokenizer
16
+from vllm.utils import random_uuid
17
+
18
+logger = init_logger(__name__)
19
+
20
+
21
+@ToolParserManager.register_module("deepseek_v3")
22
+class DeepSeekV3ToolParser(ToolParser):
23
+
24
+ def __init__(self, tokenizer: AnyTokenizer):
25
+ super().__init__(tokenizer)
26
+
27
+ self.current_tool_name_sent: bool = False
28
+ self.prev_tool_call_arr: list[dict] = []
29
+ self.current_tool_id: int = -1
30
+ self.streamed_args_for_tool: list[str] = (
31
+ []) # map what has been streamed for each tool so far to a list
32
+
33
+ self.tool_calls_start_token: str = "<｜tool▁calls▁begin｜>"
34
+ self.tool_calls_end_token: str = "<｜tool▁calls▁end｜>"
35
+
36
+ self.tool_call_start_token: str = "<｜tool▁call▁begin｜>"
37
+ self.tool_call_end_token: str = "<｜tool▁call▁end｜>"
38
+
39
+ self.tool_call_regex = re.compile(
40
+ r"<｜tool▁call▁begin｜>(?P<type>.*)<｜tool▁sep｜>(?P<function_name>.*)\n```json\n(?P<function_arguments>.*)\n```<｜tool▁call▁end｜>"
41
+ )
42
+
43
+ self.stream_tool_call_portion_regex = re.compile(
44
+ r"(?P<type>.*)<｜tool▁sep｜>(?P<function_name>.*)\n```json\n(?P<function_arguments>.*[^\n`])"
45
+ )
46
+
47
+ self.stream_tool_call_name_regex = re.compile(
48
+ r"(?P<type>.*)<｜tool▁sep｜>(?P<function_name>.*)\n")
49
+
50
+ if not self.model_tokenizer:
51
+ raise ValueError(
52
+ "The model tokenizer must be passed to the ToolParser "
53
+ "constructor during construction.")
54
+ self.tool_calls_start_token_id = self.vocab.get(
55
+ self.tool_calls_start_token)
56
+ self.tool_calls_end_token_id = self.vocab.get(
57
+ self.tool_calls_end_token)
58
+
59
+ self.tool_call_start_token_id = self.vocab.get(
60
+ self.tool_call_start_token)
61
+ self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
62
+
63
+ if (self.tool_calls_start_token_id is None
64
+ or self.tool_calls_end_token_id is None):
65
+ raise RuntimeError(
66
+ "DeepSeek-V3 Tool parser could not locate tool call start/end "
67
+ "tokens in the tokenizer!")
68
+
69
+ def extract_tool_calls(
70
+ self,
71
+ model_output: str,
72
+ request: ChatCompletionRequest,
73
+ ) -> ExtractedToolCallInformation:
74
+
75
+ # sanity check; avoid unnecessary processing
76
+ if self.tool_calls_start_token not in model_output:
77
+ return ExtractedToolCallInformation(tools_called=False,
78
+ tool_calls=[],
79
+ content=model_output)
80
+
81
+ else:
82
+ try:
83
+ # there are two possible captures - between tags, or between a
84
+ # tag and end-of-string so the result of
85
+ # findall is an array of tuples where one is a function call and
86
+ # the other is None
87
+ function_call_tuples = self.tool_call_regex.findall(
88
+ model_output)
89
+
90
+ tool_calls = []
91
+ for match in function_call_tuples:
92
+ tool_type, function_name, function_args = match
93
+ tool_calls.append(
94
+ ToolCall(
95
+ type=tool_type,
96
+ function=FunctionCall(name=function_name,
97
+ arguments=function_args),
98
+ ))
99
+
100
+ content = model_output[:model_output.
101
+ find(self.tool_calls_start_token)]
102
+ return ExtractedToolCallInformation(
103
+ tools_called=True,
104
+ tool_calls=tool_calls,
105
+ content=content if content else None,
106
+ )
107
+
108
+ except Exception:
109
+ logger.exception(
110
+ "Error in extracting tool call from response.")
111
+ return ExtractedToolCallInformation(tools_called=False,
112
+ tool_calls=[],
113
+ content=model_output)
114
+
115
+ def extract_tool_calls_streaming(
116
+ self,
117
+ previous_text: str,
118
+ current_text: str,
119
+ delta_text: str,
120
+ previous_token_ids: Sequence[int],
121
+ current_token_ids: Sequence[int],
122
+ delta_token_ids: Sequence[int],
123
+ request: ChatCompletionRequest,
124
+ ) -> Union[DeltaMessage, None]:
125
+
126
+ logger.debug("delta_text: %s", delta_text)
127
+ logger.debug("delta_token_ids: %s", delta_token_ids)
128
+ # check to see if we should be streaming a tool call - is there a
129
+ if self.tool_calls_start_token_id not in current_token_ids:
130
+ logger.debug("No tool call tokens found!")
131
+ return DeltaMessage(content=delta_text)
132
+ delta_text = delta_text.replace(self.tool_calls_start_token,
133
+ "").replace(self.tool_calls_end_token,
134
+ "")
135
+ try:
136
+
137
+ # figure out where we are in the parsing by counting tool call
138
+ # start & end tags
139
+ prev_tool_start_count = previous_token_ids.count(
140
+ self.tool_call_start_token_id)
141
+ prev_tool_end_count = previous_token_ids.count(
142
+ self.tool_call_end_token_id)
143
+ cur_tool_start_count = current_token_ids.count(
144
+ self.tool_call_start_token_id)
145
+ cur_tool_end_count = current_token_ids.count(
146
+ self.tool_call_end_token_id)
147
+ tool_call_portion = None
148
+ text_portion = None
149
+
150
+ # case: if we're generating text, OR rounding out a tool call
151
+ if (cur_tool_start_count == cur_tool_end_count
152
+ and prev_tool_end_count == cur_tool_end_count
153
+ and self.tool_call_end_token not in delta_text):
154
+ logger.debug("Generating text content! skipping tool parsing.")
155
+ return DeltaMessage(content=delta_text)
156
+
157
+ if self.tool_call_end_token in delta_text:
158
+ logger.debug("tool_call_end_token in delta_text")
159
+ full_text = current_text + delta_text
160
+ tool_call_portion = full_text.split(
161
+ self.tool_call_start_token)[-1].split(
162
+ self.tool_call_end_token)[0].rstrip()
163
+ delta_text = delta_text.split(
164
+ self.tool_call_end_token)[0].rstrip()
165
+ text_portion = delta_text.split(
166
+ self.tool_call_end_token)[-1].lstrip()
167
+
168
+ # case -- we're starting a new tool call
169
+ if (cur_tool_start_count > cur_tool_end_count
170
+ and cur_tool_start_count > prev_tool_start_count):
171
+ if len(delta_token_ids) > 1:
172
+ tool_call_portion = current_text.split(
173
+ self.tool_call_start_token)[-1]
174
+ else:
175
+ tool_call_portion = None
176
+ delta = None
177
+
178
+ text_portion = None
179
+
180
+ # set cursors and state appropriately
181
+ self.current_tool_id += 1
182
+ self.current_tool_name_sent = False
183
+ self.streamed_args_for_tool.append("")
184
+ logger.debug("Starting on a new tool %s", self.current_tool_id)
185
+
186
+ # case -- we're updating an existing tool call
187
+ elif (cur_tool_start_count > cur_tool_end_count
188
+ and cur_tool_start_count == prev_tool_start_count):
189
+
190
+ # get the portion of the text that's the tool call
191
+ tool_call_portion = current_text.split(
192
+ self.tool_call_start_token)[-1]
193
+ text_portion = None
194
+
195
+ # case -- the current tool call is being closed.
196
+ elif (cur_tool_start_count == cur_tool_end_count
197
+ and cur_tool_end_count >= prev_tool_end_count):
198
+ if self.prev_tool_call_arr is None or len(
199
+ self.prev_tool_call_arr) == 0:
200
+ logger.debug(
201
+ "attempting to close tool call, but no tool call")
202
+ return None
203
+ diff = self.prev_tool_call_arr[self.current_tool_id].get(
204
+ "arguments")
205
+ if diff:
206
+ diff = (diff.encode("utf-8").decode("unicode_escape")
207
+ if diff is str else diff)
208
+ if '"}' not in delta_text:
209
+ return None
210
+ end_loc = delta_text.rindex('"}')
211
+ diff = delta_text[:end_loc] + '"}'
212
+ logger.debug(
213
+ "Finishing tool and found diff that had not "
214
+ "been streamed yet: %s",
215
+ diff,
216
+ )
217
+ self.streamed_args_for_tool[self.current_tool_id] += diff
218
+ return DeltaMessage(tool_calls=[
219
+ DeltaToolCall(
220
+ index=self.current_tool_id,
221
+ function=DeltaFunctionCall(
222
+ arguments=diff).model_dump(exclude_none=True),
223
+ )
224
+ ])
225
+
226
+ # case -- otherwise we're just generating text
227
+ else:
228
+ text = delta_text.replace(self.tool_call_start_token, "")
229
+ text = text.replace(self.tool_call_end_token, "")
230
+ delta = DeltaMessage(tool_calls=[], content=text)
231
+ return delta
232
+
233
+ current_tool_call = dict()
234
+ if tool_call_portion:
235
+ current_tool_call_matches = (
236
+ self.stream_tool_call_portion_regex.match(
237
+ tool_call_portion))
238
+ if current_tool_call_matches:
239
+ tool_type, tool_name, tool_args = (
240
+ current_tool_call_matches.groups())
241
+ current_tool_call["name"] = tool_name
242
+ current_tool_call["arguments"] = tool_args
243
+ else:
244
+ current_tool_call_name_matches = (
245
+ self.stream_tool_call_name_regex.match(
246
+ tool_call_portion))
247
+ if current_tool_call_name_matches:
248
+ tool_type, tool_name = (
249
+ current_tool_call_name_matches.groups())
250
+ current_tool_call["name"] = tool_name
251
+ current_tool_call["arguments"] = ""
252
+ else:
253
+ logger.debug("Not enough token")
254
+ return None
255
+
256
+ # case - we haven't sent the tool name yet. If it's available, send
257
+ # it. otherwise, wait until it's available.
258
+ if not self.current_tool_name_sent:
259
+ if current_tool_call is None:
260
+ return None
261
+ function_name: Union[str, None] = current_tool_call.get("name")
262
+ if function_name:
263
+ self.current_tool_name_sent = True
264
+ return DeltaMessage(tool_calls=[
265
+ DeltaToolCall(
266
+ index=self.current_tool_id,
267
+ type="function",
268
+ id=f"chatcmpl-tool-{random_uuid()}",
269
+ function=DeltaFunctionCall(
270
+ name=function_name).model_dump(
271
+ exclude_none=True),
272
+ )
273
+ ])
274
+ else:
275
+ return None
276
+
277
+ # case -- otherwise, send the tool call delta
278
+
279
+ # if the tool call portion is None, send the delta as text
280
+ if tool_call_portion is None:
281
+ # if there's text but not tool calls, send that -
282
+ # otherwise None to skip chunk
283
+ delta = (DeltaMessage(
284
+ content=delta_text) if text_portion is not None else None)
285
+ return delta
286
+
287
+ # now, the nitty-gritty of tool calls
288
+ # now we have the portion to parse as tool call.
289
+
290
+ logger.debug("Trying to parse current tool call with ID %s",
291
+ self.current_tool_id)
292
+
293
+ # if we're starting a new tool call, push an empty object in as
294
+ # a placeholder for the arguments
295
+ if len(self.prev_tool_call_arr) <= self.current_tool_id:
296
+ self.prev_tool_call_arr.append({})
297
+
298
+ # main logic for tool parsing here - compare prev. partially-parsed
299
+ # JSON to the current partially-parsed JSON
300
+ prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get(
301
+ "arguments")
302
+ cur_arguments = current_tool_call.get("arguments")
303
+
304
+ logger.debug("diffing old arguments: %s", prev_arguments)
305
+ logger.debug("against new ones: %s", cur_arguments)
306
+
307
+ # case -- no arguments have been created yet. skip sending a delta.
308
+ if not cur_arguments and not prev_arguments:
309
+ logger.debug("Skipping text %s - no arguments", delta_text)
310
+ delta = None
311
+
312
+ # case -- prev arguments are defined, but non are now.
313
+ # probably impossible, but not a fatal error - just keep going
314
+ elif not cur_arguments and prev_arguments:
315
+ logger.error("should be impossible to have arguments reset "
316
+ "mid-call. skipping streaming anything.")
317
+ delta = None
318
+
319
+ # case -- we now have the first info about arguments available from
320
+ # autocompleting the JSON
321
+ elif cur_arguments and not prev_arguments:
322
+
323
+ delta = DeltaMessage(tool_calls=[
324
+ DeltaToolCall(
325
+ index=self.current_tool_id,
326
+ function=DeltaFunctionCall(
327
+ arguments=cur_arguments).model_dump(
328
+ exclude_none=True),
329
+ )
330
+ ])
331
+ self.streamed_args_for_tool[
332
+ self.current_tool_id] = cur_arguments
333
+
334
+ # last case -- we have an update to existing arguments.
335
+ elif cur_arguments and prev_arguments:
336
+ if (isinstance(delta_text, str)
337
+ and cur_arguments != prev_arguments
338
+ and len(cur_arguments) > len(prev_arguments)
339
+ and cur_arguments.startswith(prev_arguments)):
340
+ delta_arguments = cur_arguments[len(prev_arguments):]
341
+ logger.debug("got diff %s", delta_text)
342
+
343
+ delta = DeltaMessage(tool_calls=[
344
+ DeltaToolCall(
345
+ index=self.current_tool_id,
346
+ function=DeltaFunctionCall(
347
+ arguments=delta_arguments).model_dump(
348
+ exclude_none=True),
349
+ )
350
+ ])
351
+ self.streamed_args_for_tool[
352
+ self.current_tool_id] = cur_arguments
353
+ else:
354
+ delta = None
355
+
356
+ # handle saving the state for the current tool into
357
+ # the "prev" list for use in diffing for the next iteration
358
+ if self.current_tool_id == len(self.prev_tool_call_arr) - 1:
359
+ self.prev_tool_call_arr[
360
+ self.current_tool_id] = current_tool_call
361
+ else:
362
+ self.prev_tool_call_arr.append(current_tool_call)
363
+
364
+ return delta
365
+
366
+ except Exception:
367
+ logger.exception("Error trying to handle streaming tool call.")
368
+ return None # do not stream a delta. skip this token ID.