| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Sequence |
| 5 | |
| 6 | import regex as re |
| 7 | |
| 8 | from vllm.entrypoints.chat_utils import make_tool_call_id |
| 9 | from vllm.entrypoints.generate.base.protocol import ( |
| 10 | DeltaFunctionCall, |
| 11 | DeltaMessage, |
| 12 | DeltaToolCall, |
| 13 | ExtractedToolCallInformation, |
| 14 | FunctionCall, |
| 15 | ToolCall, |
| 16 | ) |
| 17 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 18 | ChatCompletionRequest, |
| 19 | ) |
| 20 | from vllm.logger import init_logger |
| 21 | from vllm.tokenizers import TokenizerLike |
| 22 | from vllm.tool_parsers.abstract_tool_parser import ( |
| 23 | Tool, |
| 24 | ToolParser, |
| 25 | ) |
| 26 | |
| 27 | logger = init_logger(__name__) |
| 28 | |
| 29 | |
| 30 | class DeepSeekV3ToolParser(ToolParser): |
| 31 | structural_tag_model = "deepseek_r1" |
| 32 | |
| 33 | def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): |
| 34 | super().__init__(tokenizer, tools) |
| 35 | |
| 36 | self.current_tool_name_sent: bool = False |
| 37 | self.prev_tool_call_arr: list[dict] = [] |
| 38 | self.current_tool_id: int = -1 |
| 39 | self.streamed_args_for_tool: list[ |
| 40 | str |
| 41 | ] = [] # map what has been streamed for each tool so far to a list |
| 42 | |
| 43 | self.tool_calls_start_token: str = "<｜tool▁calls▁begin｜>" |
| 44 | self.tool_calls_end_token: str = "<｜tool▁calls▁end｜>" |
| 45 | |
| 46 | self.tool_call_start_token: str = "<｜tool▁call▁begin｜>" |
| 47 | self.tool_call_end_token: str = "<｜tool▁call▁end｜>" |
| 48 | |
| 49 | self.tool_call_regex = re.compile( |
| 50 | r"<｜tool▁call▁begin｜>(?P<type>.*)<｜tool▁sep｜>(?P<function_name>.*)\n```json\n(?P<function_arguments>.*)\n```<｜tool▁call▁end｜>" |
| 51 | ) |
| 52 | |
| 53 | self.stream_tool_call_portion_regex = re.compile( |
| 54 | r"(?P<type>.*)<｜tool▁sep｜>(?P<function_name>.*)\n```json\n(?P<function_arguments>.*[^\n`])" |
| 55 | ) |
| 56 | |
| 57 | self.stream_tool_call_name_regex = re.compile( |
| 58 | r"(?P<type>.*)<｜tool▁sep｜>(?P<function_name>.*)\n" |
| 59 | ) |
| 60 | |
| 61 | if not self.model_tokenizer: |
| 62 | raise ValueError( |
| 63 | "The model tokenizer must be passed to the ToolParser " |
| 64 | "constructor during construction." |
| 65 | ) |
| 66 | self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token) |
| 67 | self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token) |
| 68 | |
| 69 | self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) |
| 70 | self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) |
| 71 | |
| 72 | if ( |
| 73 | self.tool_calls_start_token_id is None |
| 74 | or self.tool_calls_end_token_id is None |
| 75 | ): |
| 76 | raise RuntimeError( |
| 77 | "DeepSeek-V3 Tool parser could not locate tool call start/end " |
| 78 | "tokens in the tokenizer!" |
| 79 | ) |
| 80 | |
| 81 | def extract_tool_calls( |
| 82 | self, |
| 83 | model_output: str, |
| 84 | request: ChatCompletionRequest, |
| 85 | ) -> ExtractedToolCallInformation: |
| 86 | # sanity check; avoid unnecessary processing |
| 87 | if self.tool_calls_start_token not in model_output: |
| 88 | return ExtractedToolCallInformation( |
| 89 | tools_called=False, tool_calls=[], content=model_output |
| 90 | ) |
| 91 | |
| 92 | else: |
| 93 | try: |
| 94 | # there are two possible captures - between tags, or between a |
| 95 | # tag and end-of-string so the result of |
| 96 | # findall is an array of tuples where one is a function call and |
| 97 | # the other is None |
| 98 | function_call_tuples = self.tool_call_regex.findall(model_output) |
| 99 | |
| 100 | tool_calls = [] |
| 101 | for match in function_call_tuples: |
| 102 | tool_type, function_name, function_args = match |
| 103 | tool_calls.append( |
| 104 | ToolCall( |
| 105 | type=tool_type, |
| 106 | function=FunctionCall( |
| 107 | name=function_name, arguments=function_args |
| 108 | ), |
| 109 | ) |
| 110 | ) |
| 111 | |
| 112 | content = model_output[: model_output.find(self.tool_calls_start_token)] |
| 113 | return ExtractedToolCallInformation( |
| 114 | tools_called=True, |
| 115 | tool_calls=tool_calls, |
| 116 | content=content if content else None, |
| 117 | ) |
| 118 | |
| 119 | except Exception: |
| 120 | logger.exception("Error in extracting tool call from response.") |
| 121 | return ExtractedToolCallInformation( |
| 122 | tools_called=False, tool_calls=[], content=model_output |
| 123 | ) |
| 124 | |
| 125 | def extract_tool_calls_streaming( |
| 126 | self, |
| 127 | previous_text: str, |
| 128 | current_text: str, |
| 129 | delta_text: str, |
| 130 | previous_token_ids: Sequence[int], |
| 131 | current_token_ids: Sequence[int], |
| 132 | delta_token_ids: Sequence[int], |
| 133 | request: ChatCompletionRequest, |
| 134 | ) -> DeltaMessage | None: |
| 135 | logger.debug("delta_text: %s", delta_text) |
| 136 | logger.debug("delta_token_ids: %s", delta_token_ids) |
| 137 | # check to see if we should be streaming a tool call - is there a |
| 138 | if self.tool_calls_start_token_id not in current_token_ids: |
| 139 | logger.debug("No tool call tokens found!") |
| 140 | return DeltaMessage(content=delta_text) |
| 141 | delta_text = delta_text.replace(self.tool_calls_start_token, "").replace( |
| 142 | self.tool_calls_end_token, "" |
| 143 | ) |
| 144 | try: |
| 145 | # figure out where we are in the parsing by counting tool call |
| 146 | # start & end tags |
| 147 | prev_tool_start_count = previous_token_ids.count( |
| 148 | self.tool_call_start_token_id |
| 149 | ) |
| 150 | prev_tool_end_count = previous_token_ids.count(self.tool_call_end_token_id) |
| 151 | cur_tool_start_count = current_token_ids.count( |
| 152 | self.tool_call_start_token_id |
| 153 | ) |
| 154 | cur_tool_end_count = current_token_ids.count(self.tool_call_end_token_id) |
| 155 | tool_call_portion = None |
| 156 | text_portion = None |
| 157 | |
| 158 | # case: if we're generating text, OR rounding out a tool call |
| 159 | if ( |
| 160 | cur_tool_start_count == cur_tool_end_count |
| 161 | and prev_tool_end_count == cur_tool_end_count |
| 162 | and self.tool_call_end_token not in delta_text |
| 163 | ): |
| 164 | logger.debug("Generating text content! skipping tool parsing.") |
| 165 | return DeltaMessage(content=delta_text) |
| 166 | |
| 167 | if self.tool_call_end_token in delta_text: |
| 168 | logger.debug("tool_call_end_token in delta_text") |
| 169 | full_text = current_text + delta_text |
| 170 | tool_call_portion = ( |
| 171 | full_text.split(self.tool_call_start_token)[-1] |
| 172 | .split(self.tool_call_end_token)[0] |
| 173 | .rstrip() |
| 174 | ) |
| 175 | delta_text = delta_text.split(self.tool_call_end_token)[0].rstrip() |
| 176 | text_portion = delta_text.split(self.tool_call_end_token)[-1].lstrip() |
| 177 | |
| 178 | # case -- we're starting a new tool call |
| 179 | if ( |
| 180 | cur_tool_start_count > cur_tool_end_count |
| 181 | and cur_tool_start_count > prev_tool_start_count |
| 182 | ): |
| 183 | if len(delta_token_ids) > 1: |
| 184 | tool_call_portion = current_text.split(self.tool_call_start_token)[ |
| 185 | -1 |
| 186 | ] |
| 187 | else: |
| 188 | tool_call_portion = None |
| 189 | delta = None |
| 190 | |
| 191 | text_portion = None |
| 192 | |
| 193 | # set cursors and state appropriately |
| 194 | self.current_tool_id += 1 |
| 195 | self.current_tool_name_sent = False |
| 196 | self.streamed_args_for_tool.append("") |
| 197 | logger.debug("Starting on a new tool %s", self.current_tool_id) |
| 198 | |
| 199 | # case -- we're updating an existing tool call |
| 200 | elif ( |
| 201 | cur_tool_start_count > cur_tool_end_count |
| 202 | and cur_tool_start_count == prev_tool_start_count |
| 203 | ): |
| 204 | # get the portion of the text that's the tool call |
| 205 | tool_call_portion = current_text.split(self.tool_call_start_token)[-1] |
| 206 | text_portion = None |
| 207 | |
| 208 | # case -- the current tool call is being closed. |
| 209 | elif ( |
| 210 | cur_tool_start_count == cur_tool_end_count |
| 211 | and cur_tool_end_count >= prev_tool_end_count |
| 212 | ): |
| 213 | if self.prev_tool_call_arr is None or len(self.prev_tool_call_arr) == 0: |
| 214 | logger.debug("attempting to close tool call, but no tool call") |
| 215 | return None |
| 216 | diff = self.prev_tool_call_arr[self.current_tool_id].get("arguments") |
| 217 | if diff: |
| 218 | diff = ( |
| 219 | diff.encode("utf-8").decode("unicode_escape") |
| 220 | if diff is str |
| 221 | else diff |
| 222 | ) |
| 223 | if '"}' not in delta_text: |
| 224 | return None |
| 225 | end_loc = delta_text.rindex('"}') |
| 226 | diff = delta_text[:end_loc] + '"}' |
| 227 | logger.debug( |
| 228 | "Finishing tool and found diff that had not " |
| 229 | "been streamed yet: %s", |
| 230 | diff, |
| 231 | ) |
| 232 | self.streamed_args_for_tool[self.current_tool_id] += diff |
| 233 | return DeltaMessage( |
| 234 | tool_calls=[ |
| 235 | DeltaToolCall( |
| 236 | index=self.current_tool_id, |
| 237 | function=DeltaFunctionCall(arguments=diff).model_dump( |
| 238 | exclude_none=True |
| 239 | ), |
| 240 | ) |
| 241 | ] |
| 242 | ) |
| 243 | |
| 244 | # case -- otherwise we're just generating text |
| 245 | else: |
| 246 | text = delta_text.replace(self.tool_call_start_token, "") |
| 247 | text = text.replace(self.tool_call_end_token, "") |
| 248 | delta = DeltaMessage(tool_calls=[], content=text) |
| 249 | return delta |
| 250 | |
| 251 | current_tool_call = dict() |
| 252 | if tool_call_portion: |
| 253 | current_tool_call_matches = self.stream_tool_call_portion_regex.match( |
| 254 | tool_call_portion |
| 255 | ) |
| 256 | if current_tool_call_matches: |
| 257 | tool_type, tool_name, tool_args = current_tool_call_matches.groups() |
| 258 | current_tool_call["name"] = tool_name |
| 259 | current_tool_call["arguments"] = tool_args |
| 260 | else: |
| 261 | current_tool_call_name_matches = ( |
| 262 | self.stream_tool_call_name_regex.match(tool_call_portion) |
| 263 | ) |
| 264 | if current_tool_call_name_matches: |
| 265 | tool_type, tool_name = current_tool_call_name_matches.groups() |
| 266 | current_tool_call["name"] = tool_name |
| 267 | current_tool_call["arguments"] = "" |
| 268 | else: |
| 269 | logger.debug("Not enough token") |
| 270 | return None |
| 271 | |
| 272 | # case - we haven't sent the tool name yet. If it's available, send |
| 273 | # it. otherwise, wait until it's available. |
| 274 | if not self.current_tool_name_sent: |
| 275 | if current_tool_call is None: |
| 276 | return None |
| 277 | function_name: str | None = current_tool_call.get("name") |
| 278 | if function_name: |
| 279 | self.current_tool_name_sent = True |
| 280 | return DeltaMessage( |
| 281 | tool_calls=[ |
| 282 | DeltaToolCall( |
| 283 | index=self.current_tool_id, |
| 284 | type="function", |
| 285 | id=make_tool_call_id(), |
| 286 | function=DeltaFunctionCall( |
| 287 | name=function_name |
| 288 | ).model_dump(exclude_none=True), |
| 289 | ) |
| 290 | ] |
| 291 | ) |
| 292 | else: |
| 293 | return None |
| 294 | |
| 295 | # case -- otherwise, send the tool call delta |
| 296 | |
| 297 | # if the tool call portion is None, send the delta as text |
| 298 | if tool_call_portion is None: |
| 299 | # if there's text but not tool calls, send that - |
| 300 | # otherwise None to skip chunk |
| 301 | delta = ( |
| 302 | DeltaMessage(content=delta_text) |
| 303 | if text_portion is not None |
| 304 | else None |
| 305 | ) |
| 306 | return delta |
| 307 | |
| 308 | # now, the nitty-gritty of tool calls |
| 309 | # now we have the portion to parse as tool call. |
| 310 | |
| 311 | logger.debug( |
| 312 | "Trying to parse current tool call with ID %s", self.current_tool_id |
| 313 | ) |
| 314 | |
| 315 | # if we're starting a new tool call, push an empty object in as |
| 316 | # a placeholder for the arguments |
| 317 | if len(self.prev_tool_call_arr) <= self.current_tool_id: |
| 318 | self.prev_tool_call_arr.append({}) |
| 319 | |
| 320 | # main logic for tool parsing here - compare prev. partially-parsed |
| 321 | # JSON to the current partially-parsed JSON |
| 322 | prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( |
| 323 | "arguments" |
| 324 | ) |
| 325 | cur_arguments = current_tool_call.get("arguments") |
| 326 | |
| 327 | logger.debug("diffing old arguments: %s", prev_arguments) |
| 328 | logger.debug("against new ones: %s", cur_arguments) |
| 329 | |
| 330 | # case -- no arguments have been created yet. skip sending a delta. |
| 331 | if not cur_arguments and not prev_arguments: |
| 332 | logger.debug("Skipping text %s - no arguments", delta_text) |
| 333 | delta = None |
| 334 | |
| 335 | # case -- prev arguments are defined, but non are now. |
| 336 | # probably impossible, but not a fatal error - just keep going |
| 337 | elif not cur_arguments and prev_arguments: |
| 338 | logger.error( |
| 339 | "should be impossible to have arguments reset " |
| 340 | "mid-call. skipping streaming anything." |
| 341 | ) |
| 342 | delta = None |
| 343 | |
| 344 | # case -- we now have the first info about arguments available from |
| 345 | # autocompleting the JSON |
| 346 | elif cur_arguments and not prev_arguments: |
| 347 | delta = DeltaMessage( |
| 348 | tool_calls=[ |
| 349 | DeltaToolCall( |
| 350 | index=self.current_tool_id, |
| 351 | function=DeltaFunctionCall( |
| 352 | arguments=cur_arguments |
| 353 | ).model_dump(exclude_none=True), |
| 354 | ) |
| 355 | ] |
| 356 | ) |
| 357 | self.streamed_args_for_tool[self.current_tool_id] = cur_arguments |
| 358 | |
| 359 | # last case -- we have an update to existing arguments. |
| 360 | elif cur_arguments and prev_arguments: |
| 361 | if ( |
| 362 | isinstance(delta_text, str) |
| 363 | and cur_arguments != prev_arguments |
| 364 | and len(cur_arguments) > len(prev_arguments) |
| 365 | and cur_arguments.startswith(prev_arguments) |
| 366 | ): |
| 367 | delta_arguments = cur_arguments[len(prev_arguments) :] |
| 368 | logger.debug("got diff %s", delta_text) |
| 369 | |
| 370 | delta = DeltaMessage( |
| 371 | tool_calls=[ |
| 372 | DeltaToolCall( |
| 373 | index=self.current_tool_id, |
| 374 | function=DeltaFunctionCall( |
| 375 | arguments=delta_arguments |
| 376 | ).model_dump(exclude_none=True), |
| 377 | ) |
| 378 | ] |
| 379 | ) |
| 380 | self.streamed_args_for_tool[self.current_tool_id] = cur_arguments |
| 381 | else: |
| 382 | delta = None |
| 383 | |
| 384 | # handle saving the state for the current tool into |
| 385 | # the "prev" list for use in diffing for the next iteration |
| 386 | if self.current_tool_id == len(self.prev_tool_call_arr) - 1: |
| 387 | self.prev_tool_call_arr[self.current_tool_id] = current_tool_call |
| 388 | else: |
| 389 | self.prev_tool_call_arr.append(current_tool_call) |
| 390 | |
| 391 | return delta |
| 392 | |
| 393 | except Exception: |
| 394 | logger.exception("Error trying to handle streaming tool call.") |
| 395 | return None # do not stream a delta. skip this token ID. |