A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Support DeepSeek-V3.1 tool call (#23454)
Signed-off-by: Xu Wenqing <xuwq1993@qq.com>
Xu Wenqing committed
Aug 23, 2025 at 13:50 UTC
b8f17f5d980eb95838753a183d87d744b87ad489
4 files changed
+468
docs/features/tool_calling.md
+8
@@ -284,6 +284,14 @@ Supported models:
284
285
Flags: `--tool-call-parser deepseek_v3 --chat-template {see_above}`
286
287
+### DeepSeek-V3.1 Models (`deepseek_v31`)
288
+
289
+Supported models:
290
+
291
+* `deepseek-ai/DeepSeek-V3.1` (use with <gh-file:examples/tool_chat_template_deepseekv31.jinja>)
292
+
293
+Flags: `--tool-call-parser deepseek_v31 --chat-template {see_above}`
294
+
295
### Kimi-K2 Models (`kimi_k2`)
296
297
Supported models:
examples/tool_chat_template_deepseekv31.jinja
new
+91
@@ -0,0 +1,91 @@
1
+{% if not add_generation_prompt is defined %}
2
+ {% set add_generation_prompt = false %}
3
+{% endif %}
4
+{% if not thinking is defined %}
5
+ {% set thinking = false %}
6
+{% endif %}
7
+{% set ns = namespace(is_first=false, is_tool=false, system_prompt='', is_first_sp=true, is_last_user=false) %}
8
+{%- for message in messages %}
9
+ {%- if message['role'] == 'system' %}
10
+ {%- if ns.is_first_sp %}
11
+ {% set ns.system_prompt = ns.system_prompt + message['content'] %}
12
+ {% set ns.is_first_sp = false %}
13
+ {%- else %}
14
+ {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
15
+ {%- endif %}
16
+ {%- endif %}
17
+{%- endfor %}
18
+
19
+{% if tools is defined and tools is not none %}
20
+ {% set tool_ns = namespace(text='## Tools\nYou have access to the following tools:\n') %}
21
+ {% for tool in tools %}
22
+ {% set tool_ns.text = tool_ns.text + '\n### ' + tool.function.name + '\nDescription: ' + tool.function.description + '\n\nParameters: ' + (tool.function.parameters | tojson) + '\n' %}
23
+ {% endfor %}
24
+ {% set tool_ns.text = tool_ns.text + "\nIMPORTANT: ALWAYS adhere to this exact format for tool use:\n<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>tool_call_name<｜tool▁sep｜>tool_call_arguments<｜tool▁call▁end｜>{{additional_tool_calls}}<｜tool▁calls▁end｜>\n\nWhere:\n\n- `tool_call_name` must be an exact match to one of the available tools\n- `tool_call_arguments` must be valid JSON that strictly follows the tool's Parameters Schema\n- For multiple tool calls, chain them directly without separators or spaces\n" %}
25
+ {% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %}
26
+{% endif %}
27
+
28
+{{ bos_token }}{{ ns.system_prompt }}
29
+{%- for message in messages %}
30
+ {%- if message['role'] == 'user' %}
31
+ {%- set ns.is_tool = false -%}
32
+ {%- set ns.is_first = false -%}
33
+ {%- set ns.is_last_user = true -%}
34
+ {{'<｜User｜>' + message['content']}}
35
+ {%- endif %}
36
+ {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
37
+ {%- if ns.is_last_user %}
38
+ {{'<｜Assistant｜></think>'}}
39
+ {%- endif %}
40
+ {%- set ns.is_last_user = false -%}
41
+ {%- set ns.is_first = false %}
42
+ {%- set ns.is_tool = false -%}
43
+ {%- for tool in message['tool_calls'] %}
44
+ {%- if not ns.is_first %}
45
+ {%- if message['content'] is none %}
46
+ {{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments']|tojson + '<｜tool▁call▁end｜>'}}
47
+ {%- else %}
48
+ {{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments']|tojson + '<｜tool▁call▁end｜>'}}
49
+ {%- endif %}
50
+ {%- set ns.is_first = true -%}
51
+ {%- else %}
52
+ {{'<｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments']|tojson + '<｜tool▁call▁end｜>'}}
53
+ {%- endif %}
54
+ {%- endfor %}
55
+ {{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}
56
+ {%- endif %}
57
+ {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none) %}
58
+ {%- if ns.is_last_user %}
59
+ {{'<｜Assistant｜>'}}
60
+ {%- if message['prefix'] is defined and message['prefix'] and thinking %}
61
+ {{'<think>'}}
62
+ {%- else %}
63
+ {{'</think>'}}
64
+ {%- endif %}
65
+ {%- endif %}
66
+ {%- set ns.is_last_user = false -%}
67
+ {%- if ns.is_tool %}
68
+ {{message['content'] + '<｜end▁of▁sentence｜>'}}
69
+ {%- set ns.is_tool = false -%}
70
+ {%- else %}
71
+ {%- set content = message['content'] -%}
72
+ {%- if '</think>' in content %}
73
+ {%- set content = content.split('</think>', 1)[1] -%}
74
+ {%- endif %}
75
+ {{content + '<｜end▁of▁sentence｜>'}}
76
+ {%- endif %}
77
+ {%- endif %}
78
+ {%- if message['role'] == 'tool' %}
79
+ {%- set ns.is_last_user = false -%}
80
+ {%- set ns.is_tool = true -%}
81
+ {{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
82
+ {%- endif %}
83
+{%- endfor -%}
84
+{%- if add_generation_prompt and ns.is_last_user and not ns.is_tool %}
85
+ {{'<｜Assistant｜>'}}
86
+ {%- if not thinking %}
87
+ {{'</think>'}}
88
+ {%- else %}
89
+ {{'<think>'}}
90
+ {%- endif %}
91
+{% endif %}
vllm/entrypoints/openai/tool_parsers/__init__.py
+2
@@ -3,6 +3,7 @@
3
4
from .abstract_tool_parser import ToolParser, ToolParserManager
5
from .deepseekv3_tool_parser import DeepSeekV3ToolParser
6
+from .deepseekv31_tool_parser import DeepSeekV31ToolParser
7
from .glm4_moe_tool_parser import Glm4MoeModelToolParser
8
from .granite_20b_fc_tool_parser import Granite20bFCToolParser
9
from .granite_tool_parser import GraniteToolParser
@@ -36,6 +37,7 @@ __all__ = [
37
"PythonicToolParser",
38
"Phi4MiniJsonToolParser",
39
"DeepSeekV3ToolParser",
40
+ "DeepSeekV31ToolParser",
41
"xLAMToolParser",
42
"MinimaxToolParser",
43
"KimiK2ToolParser",
vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py
new
+367
@@ -0,0 +1,367 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from collections.abc import Sequence
5
+from typing import Union
6
+
7
+import regex as re
8
+
9
+from vllm.entrypoints.chat_utils import make_tool_call_id
10
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
11
+ DeltaFunctionCall, DeltaMessage,
12
+ DeltaToolCall,
13
+ ExtractedToolCallInformation,
14
+ FunctionCall, ToolCall)
15
+from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
16
+ ToolParser, ToolParserManager)
17
+from vllm.logger import init_logger
18
+from vllm.transformers_utils.tokenizer import AnyTokenizer
19
+
20
+logger = init_logger(__name__)
21
+
22
+
23
+@ToolParserManager.register_module("deepseek_v31")
24
+class DeepSeekV31ToolParser(ToolParser):
25
+
26
+ def __init__(self, tokenizer: AnyTokenizer):
27
+ super().__init__(tokenizer)
28
+
29
+ self.current_tool_name_sent: bool = False
30
+ self.prev_tool_call_arr: list[dict] = []
31
+ self.current_tool_id: int = -1
32
+ self.streamed_args_for_tool: list[str] = (
33
+ []) # map what has been streamed for each tool so far to a list
34
+
35
+ self.tool_calls_start_token: str = "<｜tool▁calls▁begin｜>"
36
+ self.tool_calls_end_token: str = "<｜tool▁calls▁end｜>"
37
+
38
+ self.tool_call_start_token: str = "<｜tool▁call▁begin｜>"
39
+ self.tool_call_end_token: str = "<｜tool▁call▁end｜>"
40
+
41
+ self.tool_call_regex = re.compile(
42
+ r"<｜tool▁call▁begin｜>(?P<function_name>.*)<｜tool▁sep｜>(?P<function_arguments>.*)<｜tool▁call▁end｜>"
43
+ )
44
+
45
+ self.stream_tool_call_portion_regex = re.compile(
46
+ r"(?P<function_name>.*)<｜tool▁sep｜>(?P<function_arguments>.*)")
47
+
48
+ self.stream_tool_call_name_regex = re.compile(
49
+ r"(?P<function_name>.*)<｜tool▁sep｜>")
50
+
51
+ if not self.model_tokenizer:
52
+ raise ValueError(
53
+ "The model tokenizer must be passed to the ToolParser "
54
+ "constructor during construction.")
55
+ self.tool_calls_start_token_id = self.vocab.get(
56
+ self.tool_calls_start_token)
57
+ self.tool_calls_end_token_id = self.vocab.get(
58
+ self.tool_calls_end_token)
59
+
60
+ self.tool_call_start_token_id = self.vocab.get(
61
+ self.tool_call_start_token)
62
+ self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
63
+
64
+ if (self.tool_calls_start_token_id is None
65
+ or self.tool_calls_end_token_id is None):
66
+ raise RuntimeError(
67
+ "DeepSeek-V3 Tool parser could not locate tool call start/end "
68
+ "tokens in the tokenizer!")
69
+
70
+ def extract_tool_calls(
71
+ self,
72
+ model_output: str,
73
+ request: ChatCompletionRequest,
74
+ ) -> ExtractedToolCallInformation:
75
+
76
+ # sanity check; avoid unnecessary processing
77
+ if self.tool_calls_start_token not in model_output:
78
+ return ExtractedToolCallInformation(tools_called=False,
79
+ tool_calls=[],
80
+ content=model_output)
81
+
82
+ else:
83
+ try:
84
+ # there are two possible captures - between tags, or between a
85
+ # tag and end-of-string so the result of
86
+ # findall is an array of tuples where one is a function call and
87
+ # the other is None
88
+ function_call_tuples = self.tool_call_regex.findall(
89
+ model_output)
90
+
91
+ tool_calls = []
92
+ for match in function_call_tuples:
93
+ function_name, function_args = match
94
+ tool_calls.append(
95
+ ToolCall(
96
+ type="function",
97
+ function=FunctionCall(name=function_name,
98
+ arguments=function_args),
99
+ ))
100
+
101
+ content = model_output[:model_output.
102
+ find(self.tool_calls_start_token)]
103
+ return ExtractedToolCallInformation(
104
+ tools_called=True,
105
+ tool_calls=tool_calls,
106
+ content=content if content else None,
107
+ )
108
+
109
+ except Exception:
110
+ logger.exception(
111
+ "Error in extracting tool call from response.")
112
+ return ExtractedToolCallInformation(tools_called=False,
113
+ tool_calls=[],
114
+ content=model_output)
115
+
116
+ def extract_tool_calls_streaming(
117
+ self,
118
+ previous_text: str,
119
+ current_text: str,
120
+ delta_text: str,
121
+ previous_token_ids: Sequence[int],
122
+ current_token_ids: Sequence[int],
123
+ delta_token_ids: Sequence[int],
124
+ request: ChatCompletionRequest,
125
+ ) -> Union[DeltaMessage, None]:
126
+
127
+ logger.debug("delta_text: %s", delta_text)
128
+ logger.debug("delta_token_ids: %s", delta_token_ids)
129
+ # check to see if we should be streaming a tool call - is there a
130
+ if self.tool_calls_start_token_id not in current_token_ids:
131
+ logger.debug("No tool call tokens found!")
132
+ return DeltaMessage(content=delta_text)
133
+ delta_text = delta_text.replace(self.tool_calls_start_token,
134
+ "").replace(self.tool_calls_end_token,
135
+ "")
136
+ try:
137
+
138
+ # figure out where we are in the parsing by counting tool call
139
+ # start & end tags
140
+ prev_tool_start_count = previous_token_ids.count(
141
+ self.tool_call_start_token_id)
142
+ prev_tool_end_count = previous_token_ids.count(
143
+ self.tool_call_end_token_id)
144
+ cur_tool_start_count = current_token_ids.count(
145
+ self.tool_call_start_token_id)
146
+ cur_tool_end_count = current_token_ids.count(
147
+ self.tool_call_end_token_id)
148
+ tool_call_portion = None
149
+ text_portion = None
150
+
151
+ # case: if we're generating text, OR rounding out a tool call
152
+ if (cur_tool_start_count == cur_tool_end_count
153
+ and prev_tool_end_count == cur_tool_end_count
154
+ and self.tool_call_end_token not in delta_text):
155
+ logger.debug("Generating text content! skipping tool parsing.")
156
+ return DeltaMessage(content=delta_text)
157
+
158
+ if self.tool_call_end_token in delta_text:
159
+ logger.debug("tool_call_end_token in delta_text")
160
+ full_text = current_text + delta_text
161
+ tool_call_portion = full_text.split(
162
+ self.tool_call_start_token)[-1].split(
163
+ self.tool_call_end_token)[0].rstrip()
164
+ delta_text = delta_text.split(
165
+ self.tool_call_end_token)[0].rstrip()
166
+ text_portion = delta_text.split(
167
+ self.tool_call_end_token)[-1].lstrip()
168
+
169
+ # case -- we're starting a new tool call
170
+ if (cur_tool_start_count > cur_tool_end_count
171
+ and cur_tool_start_count > prev_tool_start_count):
172
+ if len(delta_token_ids) > 1:
173
+ tool_call_portion = current_text.split(
174
+ self.tool_call_start_token)[-1]
175
+ else:
176
+ tool_call_portion = None
177
+ delta = None
178
+
179
+ text_portion = None
180
+
181
+ # set cursors and state appropriately
182
+ self.current_tool_id += 1
183
+ self.current_tool_name_sent = False
184
+ self.streamed_args_for_tool.append("")
185
+ logger.debug("Starting on a new tool %s", self.current_tool_id)
186
+
187
+ # case -- we're updating an existing tool call
188
+ elif (cur_tool_start_count > cur_tool_end_count
189
+ and cur_tool_start_count == prev_tool_start_count):
190
+
191
+ # get the portion of the text that's the tool call
192
+ tool_call_portion = current_text.split(
193
+ self.tool_call_start_token)[-1]
194
+ text_portion = None
195
+
196
+ # case -- the current tool call is being closed.
197
+ elif (cur_tool_start_count == cur_tool_end_count
198
+ and cur_tool_end_count >= prev_tool_end_count):
199
+ if self.prev_tool_call_arr is None or len(
200
+ self.prev_tool_call_arr) == 0:
201
+ logger.debug(
202
+ "attempting to close tool call, but no tool call")
203
+ return None
204
+ diff = self.prev_tool_call_arr[self.current_tool_id].get(
205
+ "arguments")
206
+ if diff:
207
+ diff = (diff.encode("utf-8").decode("unicode_escape")
208
+ if diff is str else diff)
209
+ if '"}' not in delta_text:
210
+ return None
211
+ end_loc = delta_text.rindex('"}')
212
+ diff = delta_text[:end_loc] + '"}'
213
+ logger.debug(
214
+ "Finishing tool and found diff that had not "
215
+ "been streamed yet: %s",
216
+ diff,
217
+ )
218
+ self.streamed_args_for_tool[self.current_tool_id] += diff
219
+ return DeltaMessage(tool_calls=[
220
+ DeltaToolCall(
221
+ index=self.current_tool_id,
222
+ function=DeltaFunctionCall(
223
+ arguments=diff).model_dump(exclude_none=True),
224
+ )
225
+ ])
226
+
227
+ # case -- otherwise we're just generating text
228
+ else:
229
+ text = delta_text.replace(self.tool_call_start_token, "")
230
+ text = text.replace(self.tool_call_end_token, "")
231
+ delta = DeltaMessage(tool_calls=[], content=text)
232
+ return delta
233
+
234
+ current_tool_call = dict()
235
+ if tool_call_portion:
236
+ current_tool_call_matches = (
237
+ self.stream_tool_call_portion_regex.match(
238
+ tool_call_portion))
239
+ if current_tool_call_matches:
240
+ tool_name, tool_args = current_tool_call_matches.groups()
241
+ current_tool_call["name"] = tool_name
242
+ current_tool_call["arguments"] = tool_args
243
+ else:
244
+ current_tool_call_name_matches = (
245
+ self.stream_tool_call_name_regex.match(
246
+ tool_call_portion))
247
+ if current_tool_call_name_matches:
248
+ tool_name = current_tool_call_name_matches.groups()
249
+ current_tool_call["name"] = tool_name
250
+ current_tool_call["arguments"] = ""
251
+ else:
252
+ logger.debug("Not enough token")
253
+ return None
254
+
255
+ # case - we haven't sent the tool name yet. If it's available, send
256
+ # it. otherwise, wait until it's available.
257
+ if not self.current_tool_name_sent:
258
+ if current_tool_call is None:
259
+ return None
260
+ function_name: Union[str, None] = current_tool_call.get("name")
261
+ if function_name:
262
+ self.current_tool_name_sent = True
263
+ return DeltaMessage(tool_calls=[
264
+ DeltaToolCall(
265
+ index=self.current_tool_id,
266
+ type="function",
267
+ id=make_tool_call_id(),
268
+ function=DeltaFunctionCall(
269
+ name=function_name).model_dump(
270
+ exclude_none=True),
271
+ )
272
+ ])
273
+ else:
274
+ return None
275
+
276
+ # case -- otherwise, send the tool call delta
277
+
278
+ # if the tool call portion is None, send the delta as text
279
+ if tool_call_portion is None:
280
+ # if there's text but not tool calls, send that -
281
+ # otherwise None to skip chunk
282
+ delta = (DeltaMessage(
283
+ content=delta_text) if text_portion is not None else None)
284
+ return delta
285
+
286
+ # now, the nitty-gritty of tool calls
287
+ # now we have the portion to parse as tool call.
288
+
289
+ logger.debug("Trying to parse current tool call with ID %s",
290
+ self.current_tool_id)
291
+
292
+ # if we're starting a new tool call, push an empty object in as
293
+ # a placeholder for the arguments
294
+ if len(self.prev_tool_call_arr) <= self.current_tool_id:
295
+ self.prev_tool_call_arr.append({})
296
+
297
+ # main logic for tool parsing here - compare prev. partially-parsed
298
+ # JSON to the current partially-parsed JSON
299
+ prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get(
300
+ "arguments")
301
+ cur_arguments = current_tool_call.get("arguments")
302
+
303
+ logger.debug("diffing old arguments: %s", prev_arguments)
304
+ logger.debug("against new ones: %s", cur_arguments)
305
+
306
+ # case -- no arguments have been created yet. skip sending a delta.
307
+ if not cur_arguments and not prev_arguments:
308
+ logger.debug("Skipping text %s - no arguments", delta_text)
309
+ delta = None
310
+
311
+ # case -- prev arguments are defined, but non are now.
312
+ # probably impossible, but not a fatal error - just keep going
313
+ elif not cur_arguments and prev_arguments:
314
+ logger.error("should be impossible to have arguments reset "
315
+ "mid-call. skipping streaming anything.")
316
+ delta = None
317
+
318
+ # case -- we now have the first info about arguments available from
319
+ # autocompleting the JSON
320
+ elif cur_arguments and not prev_arguments:
321
+
322
+ delta = DeltaMessage(tool_calls=[
323
+ DeltaToolCall(
324
+ index=self.current_tool_id,
325
+ function=DeltaFunctionCall(
326
+ arguments=cur_arguments).model_dump(
327
+ exclude_none=True),
328
+ )
329
+ ])
330
+ self.streamed_args_for_tool[
331
+ self.current_tool_id] = cur_arguments
332
+
333
+ # last case -- we have an update to existing arguments.
334
+ elif cur_arguments and prev_arguments:
335
+ if (isinstance(delta_text, str)
336
+ and cur_arguments != prev_arguments
337
+ and len(cur_arguments) > len(prev_arguments)
338
+ and cur_arguments.startswith(prev_arguments)):
339
+ delta_arguments = cur_arguments[len(prev_arguments):]
340
+ logger.debug("got diff %s", delta_text)
341
+
342
+ delta = DeltaMessage(tool_calls=[
343
+ DeltaToolCall(
344
+ index=self.current_tool_id,
345
+ function=DeltaFunctionCall(
346
+ arguments=delta_arguments).model_dump(
347
+ exclude_none=True),
348
+ )
349
+ ])
350
+ self.streamed_args_for_tool[
351
+ self.current_tool_id] = cur_arguments
352
+ else:
353
+ delta = None
354
+
355
+ # handle saving the state for the current tool into
356
+ # the "prev" list for use in diffing for the next iteration
357
+ if self.current_tool_id == len(self.prev_tool_call_arr) - 1:
358
+ self.prev_tool_call_arr[
359
+ self.current_tool_id] = current_tool_call
360
+ else:
361
+ self.prev_tool_call_arr.append(current_tool_call)
362
+
363
+ return delta
364
+
365
+ except Exception:
366
+ logger.exception("Error trying to handle streaming tool call.")
367
+ return None # do not stream a delta. skip this token ID.