| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Sequence |
| 5 | |
| 6 | import regex as re |
| 7 | |
| 8 | from vllm.entrypoints.chat_utils import make_tool_call_id |
| 9 | from vllm.entrypoints.generate.base.protocol import ( |
| 10 | DeltaFunctionCall, |
| 11 | DeltaMessage, |
| 12 | DeltaToolCall, |
| 13 | ExtractedToolCallInformation, |
| 14 | FunctionCall, |
| 15 | ToolCall, |
| 16 | ) |
| 17 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 18 | ChatCompletionRequest, |
| 19 | ) |
| 20 | from vllm.logger import init_logger |
| 21 | from vllm.tokenizers import TokenizerLike |
| 22 | from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser |
| 23 | |
| 24 | logger = init_logger(__name__) |
| 25 | |
| 26 | |
| 27 | class DeepSeekV31ToolParser(ToolParser): |
| 28 | structural_tag_model = "deepseek_v3_1" |
| 29 | |
| 30 | def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): |
| 31 | super().__init__(tokenizer, tools) |
| 32 | |
| 33 | self.current_tool_name_sent: bool = False |
| 34 | self.prev_tool_call_arr: list[dict] = [] |
| 35 | self.current_tool_id: int = -1 |
| 36 | self.streamed_args_for_tool: list[ |
| 37 | str |
| 38 | ] = [] # map what has been streamed for each tool so far to a list |
| 39 | |
| 40 | self.tool_calls_start_token: str = "<｜tool▁calls▁begin｜>" |
| 41 | self.tool_calls_end_token: str = "<｜tool▁calls▁end｜>" |
| 42 | |
| 43 | self.tool_call_start_token: str = "<｜tool▁call▁begin｜>" |
| 44 | self.tool_call_end_token: str = "<｜tool▁call▁end｜>" |
| 45 | |
| 46 | self.tool_call_regex = re.compile( |
| 47 | r"<｜tool▁call▁begin｜>(?P<function_name>.*?)<｜tool▁sep｜>(?P<function_arguments>.*?)<｜tool▁call▁end｜>" |
| 48 | ) |
| 49 | |
| 50 | self.stream_tool_call_portion_regex = re.compile( |
| 51 | r"(?P<function_name>.*)<｜tool▁sep｜>(?P<function_arguments>.*)" |
| 52 | ) |
| 53 | |
| 54 | self.stream_tool_call_name_regex = re.compile( |
| 55 | r"(?P<function_name>.*)<｜tool▁sep｜>" |
| 56 | ) |
| 57 | |
| 58 | if not self.model_tokenizer: |
| 59 | raise ValueError( |
| 60 | "The model tokenizer must be passed to the ToolParser " |
| 61 | "constructor during construction." |
| 62 | ) |
| 63 | self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token) |
| 64 | self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token) |
| 65 | |
| 66 | self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) |
| 67 | self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) |
| 68 | |
| 69 | if ( |
| 70 | self.tool_calls_start_token_id is None |
| 71 | or self.tool_calls_end_token_id is None |
| 72 | ): |
| 73 | raise RuntimeError( |
| 74 | "DeepSeek-V3.1 Tool parser could not locate tool call " |
| 75 | "start/end tokens in the tokenizer!" |
| 76 | ) |
| 77 | |
| 78 | def extract_tool_calls( |
| 79 | self, |
| 80 | model_output: str, |
| 81 | request: ChatCompletionRequest, |
| 82 | ) -> ExtractedToolCallInformation: |
| 83 | # sanity check; avoid unnecessary processing |
| 84 | if self.tool_calls_start_token not in model_output: |
| 85 | return ExtractedToolCallInformation( |
| 86 | tools_called=False, tool_calls=[], content=model_output |
| 87 | ) |
| 88 | |
| 89 | else: |
| 90 | try: |
| 91 | # there are two possible captures - between tags, or between a |
| 92 | # tag and end-of-string so the result of |
| 93 | # findall is an array of tuples where one is a function call and |
| 94 | # the other is None |
| 95 | function_call_tuples = self.tool_call_regex.findall(model_output) |
| 96 | |
| 97 | tool_calls = [] |
| 98 | for match in function_call_tuples: |
| 99 | function_name, function_args = match |
| 100 | tool_calls.append( |
| 101 | ToolCall( |
| 102 | type="function", |
| 103 | function=FunctionCall( |
| 104 | name=function_name, arguments=function_args |
| 105 | ), |
| 106 | ) |
| 107 | ) |
| 108 | |
| 109 | content = model_output[: model_output.find(self.tool_calls_start_token)] |
| 110 | return ExtractedToolCallInformation( |
| 111 | tools_called=True, |
| 112 | tool_calls=tool_calls, |
| 113 | content=content if content else None, |
| 114 | ) |
| 115 | |
| 116 | except Exception: |
| 117 | logger.exception("Error in extracting tool call from response.") |
| 118 | return ExtractedToolCallInformation( |
| 119 | tools_called=False, tool_calls=[], content=model_output |
| 120 | ) |
| 121 | |
| 122 | def extract_tool_calls_streaming( |
| 123 | self, |
| 124 | previous_text: str, |
| 125 | current_text: str, |
| 126 | delta_text: str, |
| 127 | previous_token_ids: Sequence[int], |
| 128 | current_token_ids: Sequence[int], |
| 129 | delta_token_ids: Sequence[int], |
| 130 | request: ChatCompletionRequest, |
| 131 | ) -> DeltaMessage | None: |
| 132 | logger.debug("delta_text: %s", delta_text) |
| 133 | logger.debug("delta_token_ids: %s", delta_token_ids) |
| 134 | # check to see if we should be streaming a tool call - is there a |
| 135 | if self.tool_calls_start_token_id not in current_token_ids: |
| 136 | logger.debug("No tool call tokens found!") |
| 137 | return DeltaMessage(content=delta_text) |
| 138 | delta_text = delta_text.replace(self.tool_calls_start_token, "").replace( |
| 139 | self.tool_calls_end_token, "" |
| 140 | ) |
| 141 | try: |
| 142 | # figure out where we are in the parsing by counting tool call |
| 143 | # start & end tags |
| 144 | prev_tool_start_count = previous_token_ids.count( |
| 145 | self.tool_call_start_token_id |
| 146 | ) |
| 147 | prev_tool_end_count = previous_token_ids.count(self.tool_call_end_token_id) |
| 148 | cur_tool_start_count = current_token_ids.count( |
| 149 | self.tool_call_start_token_id |
| 150 | ) |
| 151 | cur_tool_end_count = current_token_ids.count(self.tool_call_end_token_id) |
| 152 | tool_call_portion = None |
| 153 | text_portion = None |
| 154 | |
| 155 | # case: if we're generating text, OR rounding out a tool call |
| 156 | if ( |
| 157 | cur_tool_start_count == cur_tool_end_count |
| 158 | and prev_tool_end_count == cur_tool_end_count |
| 159 | and self.tool_call_end_token not in delta_text |
| 160 | ): |
| 161 | logger.debug("Generating text content! skipping tool parsing.") |
| 162 | return DeltaMessage(content=delta_text) |
| 163 | |
| 164 | if self.tool_call_end_token in delta_text: |
| 165 | logger.debug("tool_call_end_token in delta_text") |
| 166 | full_text = current_text + delta_text |
| 167 | tool_call_portion = ( |
| 168 | full_text.split(self.tool_call_start_token)[-1] |
| 169 | .split(self.tool_call_end_token)[0] |
| 170 | .rstrip() |
| 171 | ) |
| 172 | delta_text = delta_text.split(self.tool_call_end_token)[0].rstrip() |
| 173 | text_portion = delta_text.split(self.tool_call_end_token)[-1].lstrip() |
| 174 | |
| 175 | # case -- we're starting a new tool call |
| 176 | if ( |
| 177 | cur_tool_start_count > cur_tool_end_count |
| 178 | and cur_tool_start_count > prev_tool_start_count |
| 179 | ): |
| 180 | if len(delta_token_ids) > 1: |
| 181 | tool_call_portion = current_text.split(self.tool_call_start_token)[ |
| 182 | -1 |
| 183 | ] |
| 184 | else: |
| 185 | tool_call_portion = None |
| 186 | delta = None |
| 187 | |
| 188 | text_portion = None |
| 189 | |
| 190 | # set cursors and state appropriately |
| 191 | self.current_tool_id += 1 |
| 192 | self.current_tool_name_sent = False |
| 193 | self.streamed_args_for_tool.append("") |
| 194 | logger.debug("Starting on a new tool %s", self.current_tool_id) |
| 195 | |
| 196 | # case -- we're updating an existing tool call |
| 197 | elif ( |
| 198 | cur_tool_start_count > cur_tool_end_count |
| 199 | and cur_tool_start_count == prev_tool_start_count |
| 200 | ): |
| 201 | # get the portion of the text that's the tool call |
| 202 | tool_call_portion = current_text.split(self.tool_call_start_token)[-1] |
| 203 | text_portion = None |
| 204 | |
| 205 | # case -- the current tool call is being closed. |
| 206 | elif ( |
| 207 | cur_tool_start_count == cur_tool_end_count |
| 208 | and cur_tool_end_count >= prev_tool_end_count |
| 209 | ): |
| 210 | if self.prev_tool_call_arr is None or len(self.prev_tool_call_arr) == 0: |
| 211 | logger.debug("attempting to close tool call, but no tool call") |
| 212 | return None |
| 213 | diff = self.prev_tool_call_arr[self.current_tool_id].get("arguments") |
| 214 | if diff: |
| 215 | diff = ( |
| 216 | diff.encode("utf-8").decode("unicode_escape") |
| 217 | if diff is str |
| 218 | else diff |
| 219 | ) |
| 220 | if '"}' not in delta_text: |
| 221 | return None |
| 222 | end_loc = delta_text.rindex('"}') |
| 223 | diff = delta_text[:end_loc] + '"}' |
| 224 | logger.debug( |
| 225 | "Finishing tool and found diff that had not " |
| 226 | "been streamed yet: %s", |
| 227 | diff, |
| 228 | ) |
| 229 | self.streamed_args_for_tool[self.current_tool_id] += diff |
| 230 | return DeltaMessage( |
| 231 | tool_calls=[ |
| 232 | DeltaToolCall( |
| 233 | index=self.current_tool_id, |
| 234 | function=DeltaFunctionCall(arguments=diff).model_dump( |
| 235 | exclude_none=True |
| 236 | ), |
| 237 | ) |
| 238 | ] |
| 239 | ) |
| 240 | |
| 241 | # case -- otherwise we're just generating text |
| 242 | else: |
| 243 | text = delta_text.replace(self.tool_call_start_token, "") |
| 244 | text = text.replace(self.tool_call_end_token, "") |
| 245 | delta = DeltaMessage(tool_calls=[], content=text) |
| 246 | return delta |
| 247 | |
| 248 | current_tool_call = dict() |
| 249 | if tool_call_portion: |
| 250 | current_tool_call_matches = self.stream_tool_call_portion_regex.match( |
| 251 | tool_call_portion |
| 252 | ) |
| 253 | if current_tool_call_matches: |
| 254 | tool_name, tool_args = current_tool_call_matches.groups() |
| 255 | current_tool_call["name"] = tool_name |
| 256 | current_tool_call["arguments"] = tool_args |
| 257 | else: |
| 258 | current_tool_call_name_matches = ( |
| 259 | self.stream_tool_call_name_regex.match(tool_call_portion) |
| 260 | ) |
| 261 | if current_tool_call_name_matches: |
| 262 | tool_name = current_tool_call_name_matches.groups() |
| 263 | current_tool_call["name"] = tool_name |
| 264 | current_tool_call["arguments"] = "" |
| 265 | else: |
| 266 | logger.debug("Not enough token") |
| 267 | return None |
| 268 | |
| 269 | # case - we haven't sent the tool name yet. If it's available, send |
| 270 | # it. otherwise, wait until it's available. |
| 271 | if not self.current_tool_name_sent: |
| 272 | if current_tool_call is None: |
| 273 | return None |
| 274 | function_name: str | None = current_tool_call.get("name") |
| 275 | if function_name: |
| 276 | self.current_tool_name_sent = True |
| 277 | return DeltaMessage( |
| 278 | tool_calls=[ |
| 279 | DeltaToolCall( |
| 280 | index=self.current_tool_id, |
| 281 | type="function", |
| 282 | id=make_tool_call_id(), |
| 283 | function=DeltaFunctionCall( |
| 284 | name=function_name |
| 285 | ).model_dump(exclude_none=True), |
| 286 | ) |
| 287 | ] |
| 288 | ) |
| 289 | else: |
| 290 | return None |
| 291 | |
| 292 | # case -- otherwise, send the tool call delta |
| 293 | |
| 294 | # if the tool call portion is None, send the delta as text |
| 295 | if tool_call_portion is None: |
| 296 | # if there's text but not tool calls, send that - |
| 297 | # otherwise None to skip chunk |
| 298 | delta = ( |
| 299 | DeltaMessage(content=delta_text) |
| 300 | if text_portion is not None |
| 301 | else None |
| 302 | ) |
| 303 | return delta |
| 304 | |
| 305 | # now, the nitty-gritty of tool calls |
| 306 | # now we have the portion to parse as tool call. |
| 307 | |
| 308 | logger.debug( |
| 309 | "Trying to parse current tool call with ID %s", self.current_tool_id |
| 310 | ) |
| 311 | |
| 312 | # if we're starting a new tool call, push an empty object in as |
| 313 | # a placeholder for the arguments |
| 314 | if len(self.prev_tool_call_arr) <= self.current_tool_id: |
| 315 | self.prev_tool_call_arr.append({}) |
| 316 | |
| 317 | # main logic for tool parsing here - compare prev. partially-parsed |
| 318 | # JSON to the current partially-parsed JSON |
| 319 | prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( |
| 320 | "arguments" |
| 321 | ) |
| 322 | cur_arguments = current_tool_call.get("arguments") |
| 323 | |
| 324 | logger.debug("diffing old arguments: %s", prev_arguments) |
| 325 | logger.debug("against new ones: %s", cur_arguments) |
| 326 | |
| 327 | # case -- no arguments have been created yet. skip sending a delta. |
| 328 | if not cur_arguments and not prev_arguments: |
| 329 | logger.debug("Skipping text %s - no arguments", delta_text) |
| 330 | delta = None |
| 331 | |
| 332 | # case -- prev arguments are defined, but non are now. |
| 333 | # probably impossible, but not a fatal error - just keep going |
| 334 | elif not cur_arguments and prev_arguments: |
| 335 | logger.error( |
| 336 | "should be impossible to have arguments reset " |
| 337 | "mid-call. skipping streaming anything." |
| 338 | ) |
| 339 | delta = None |
| 340 | |
| 341 | # case -- we now have the first info about arguments available from |
| 342 | # autocompleting the JSON |
| 343 | elif cur_arguments and not prev_arguments: |
| 344 | delta = DeltaMessage( |
| 345 | tool_calls=[ |
| 346 | DeltaToolCall( |
| 347 | index=self.current_tool_id, |
| 348 | function=DeltaFunctionCall( |
| 349 | arguments=cur_arguments |
| 350 | ).model_dump(exclude_none=True), |
| 351 | ) |
| 352 | ] |
| 353 | ) |
| 354 | self.streamed_args_for_tool[self.current_tool_id] = cur_arguments |
| 355 | |
| 356 | # last case -- we have an update to existing arguments. |
| 357 | elif cur_arguments and prev_arguments: |
| 358 | if ( |
| 359 | isinstance(delta_text, str) |
| 360 | and cur_arguments != prev_arguments |
| 361 | and len(cur_arguments) > len(prev_arguments) |
| 362 | and cur_arguments.startswith(prev_arguments) |
| 363 | ): |
| 364 | delta_arguments = cur_arguments[len(prev_arguments) :] |
| 365 | logger.debug("got diff %s", delta_text) |
| 366 | |
| 367 | delta = DeltaMessage( |
| 368 | tool_calls=[ |
| 369 | DeltaToolCall( |
| 370 | index=self.current_tool_id, |
| 371 | function=DeltaFunctionCall( |
| 372 | arguments=delta_arguments |
| 373 | ).model_dump(exclude_none=True), |
| 374 | ) |
| 375 | ] |
| 376 | ) |
| 377 | self.streamed_args_for_tool[self.current_tool_id] = cur_arguments |
| 378 | else: |
| 379 | delta = None |
| 380 | |
| 381 | # handle saving the state for the current tool into |
| 382 | # the "prev" list for use in diffing for the next iteration |
| 383 | if self.current_tool_id == len(self.prev_tool_call_arr) - 1: |
| 384 | self.prev_tool_call_arr[self.current_tool_id] = current_tool_call |
| 385 | else: |
| 386 | self.prev_tool_call_arr.append(current_tool_call) |
| 387 | |
| 388 | return delta |
| 389 | |
| 390 | except Exception: |
| 391 | logger.exception("Error trying to handle streaming tool call.") |
| 392 | return None # do not stream a delta. skip this token ID. |