| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import ast |
| 5 | from collections.abc import Sequence |
| 6 | |
| 7 | import regex as re |
| 8 | from transformers import PreTrainedTokenizerBase |
| 9 | |
| 10 | import vllm.envs as envs |
| 11 | from vllm.entrypoints.generate.base.protocol import ( |
| 12 | DeltaMessage, |
| 13 | ExtractedToolCallInformation, |
| 14 | ) |
| 15 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 16 | ChatCompletionRequest, |
| 17 | ) |
| 18 | from vllm.logger import init_logger |
| 19 | from vllm.tool_parsers.abstract_tool_parser import ( |
| 20 | Tool, |
| 21 | ToolParser, |
| 22 | ) |
| 23 | from vllm.tool_parsers.utils import ( |
| 24 | UnexpectedAstError, |
| 25 | compute_tool_delta, |
| 26 | handle_single_tool, |
| 27 | make_valid_python, |
| 28 | ) |
| 29 | |
| 30 | logger = init_logger(__name__) |
| 31 | |
| 32 | |
| 33 | class PythonicToolParser(ToolParser): |
| 34 | """Tool call parser for models that produce tool calls in a pythonic style, |
| 35 | such as Llama 3.2 and Llama 4 models. |
| 36 | |
| 37 | Used when --enable-auto-tool-choice --tool-call-parser pythonic are all set |
| 38 | """ |
| 39 | |
| 40 | # TODO(mdepinet): Possible future improvements: |
| 41 | # 1. Support text + tools separated by either <|python_tag|> or \n\n |
| 42 | # 2. Support tools outside of a list (or separated by a semicolon). |
| 43 | # This depends on item 1 for consistent streaming. |
| 44 | # Neither of these are necessary for e.g. ToolACE, but both would help make |
| 45 | # Llama3.2 models more reliable. |
| 46 | |
| 47 | TOOL_CALL_REGEX = re.compile( |
| 48 | r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]", |
| 49 | re.DOTALL, |
| 50 | ) |
| 51 | |
| 52 | def __init__( |
| 53 | self, |
| 54 | tokenizer: PreTrainedTokenizerBase, |
| 55 | tools: list[Tool] | None = None, |
| 56 | ): |
| 57 | super().__init__(tokenizer, tools) |
| 58 | |
| 59 | # Rename for readability. This is NOT a tool id. |
| 60 | @property |
| 61 | def current_tool_index(self) -> int: |
| 62 | return self.current_tool_id |
| 63 | |
| 64 | @current_tool_index.setter |
| 65 | def current_tool_index(self, value: int) -> None: |
| 66 | self.current_tool_id = value |
| 67 | |
| 68 | def extract_tool_calls( |
| 69 | self, model_output: str, request: ChatCompletionRequest |
| 70 | ) -> ExtractedToolCallInformation: |
| 71 | """Extract the tool calls from a complete model response.""" |
| 72 | is_tool_call_pattern = False |
| 73 | try: |
| 74 | is_tool_call_pattern = ( |
| 75 | self.TOOL_CALL_REGEX.match( |
| 76 | model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS |
| 77 | ) |
| 78 | is not None |
| 79 | ) |
| 80 | except TimeoutError: |
| 81 | logger.warning("Regex timeout occurred when matching tool call pattern.") |
| 82 | logger.debug( |
| 83 | "Regex timeout occurred when matching user input: %s", model_output |
| 84 | ) |
| 85 | |
| 86 | if not is_tool_call_pattern: |
| 87 | return ExtractedToolCallInformation( |
| 88 | tools_called=False, tool_calls=[], content=model_output |
| 89 | ) |
| 90 | |
| 91 | try: |
| 92 | module = ast.parse(model_output) |
| 93 | parsed = getattr(module.body[0], "value", None) |
| 94 | if isinstance(parsed, ast.List) and all( |
| 95 | isinstance(e, ast.Call) for e in parsed.elts |
| 96 | ): |
| 97 | return ExtractedToolCallInformation( |
| 98 | tools_called=True, |
| 99 | tool_calls=[ |
| 100 | handle_single_tool(e) # type: ignore |
| 101 | for e in parsed.elts |
| 102 | ], |
| 103 | content=None, |
| 104 | ) |
| 105 | else: |
| 106 | raise UnexpectedAstError("Tool output must be a list of function calls") |
| 107 | except Exception: |
| 108 | logger.exception("Error in extracting tool call from response.") |
| 109 | # Treat as regular text |
| 110 | return ExtractedToolCallInformation( |
| 111 | tools_called=False, tool_calls=[], content=model_output |
| 112 | ) |
| 113 | |
| 114 | def extract_tool_calls_streaming( |
| 115 | self, |
| 116 | previous_text: str, |
| 117 | current_text: str, |
| 118 | delta_text: str, |
| 119 | previous_token_ids: Sequence[int], |
| 120 | current_token_ids: Sequence[int], |
| 121 | delta_token_ids: Sequence[int], |
| 122 | request: ChatCompletionRequest, |
| 123 | ) -> DeltaMessage | None: |
| 124 | if not current_text.startswith("["): |
| 125 | return DeltaMessage(content=delta_text) |
| 126 | |
| 127 | try: |
| 128 | valid_and_added_text = make_valid_python(current_text) |
| 129 | if valid_and_added_text is None: |
| 130 | return None |
| 131 | valid_text, added_text = valid_and_added_text |
| 132 | |
| 133 | module = ast.parse(valid_text) |
| 134 | parsed = getattr(module.body[0], "value", None) |
| 135 | if not isinstance(parsed, ast.List) or not all( |
| 136 | isinstance(e, ast.Call) for e in parsed.elts |
| 137 | ): |
| 138 | raise UnexpectedAstError("Tool output must be a list of function calls") |
| 139 | tool_calls = [ |
| 140 | handle_single_tool(e) # type: ignore |
| 141 | for e in parsed.elts |
| 142 | ] |
| 143 | |
| 144 | tool_deltas = [] |
| 145 | for index, new_call in enumerate(tool_calls): |
| 146 | if index < self.current_tool_index: |
| 147 | continue |
| 148 | |
| 149 | self.current_tool_index = index |
| 150 | if len(self.streamed_args_for_tool) == index: |
| 151 | self.streamed_args_for_tool.append("") |
| 152 | |
| 153 | new_call_complete = ( |
| 154 | index < len(tool_calls) - 1 or ")]" not in added_text |
| 155 | ) |
| 156 | if new_call_complete: |
| 157 | self.current_tool_index += 1 |
| 158 | |
| 159 | withheld_suffix = added_text[:-2] if not new_call_complete else "" |
| 160 | if not new_call_complete and added_text[-2] == ")": |
| 161 | # Function call is incomplete. Withhold the closing bracket. |
| 162 | withheld_suffix = withheld_suffix + "}" |
| 163 | # Strings get single quotes in the model-produced string. |
| 164 | # JSON requires double quotes. |
| 165 | withheld_suffix = withheld_suffix.replace("'", '"') |
| 166 | delta = compute_tool_delta( |
| 167 | self.streamed_args_for_tool[index], new_call, index, withheld_suffix |
| 168 | ) |
| 169 | |
| 170 | if delta is not None: |
| 171 | tool_deltas.append(delta) |
| 172 | if ( |
| 173 | delta.function is not None |
| 174 | and delta.function.arguments is not None |
| 175 | ): |
| 176 | self.streamed_args_for_tool[index] += delta.function.arguments |
| 177 | |
| 178 | # HACK: serving_chat.py inspects the internal state of tool parsers |
| 179 | # when determining its final streaming delta, automatically |
| 180 | # adding autocompleted JSON. |
| 181 | # These two lines avoid that nonsense while ensuring finish_reason |
| 182 | # is set to tool_calls when at least one tool is called. |
| 183 | if tool_deltas and not self.prev_tool_call_arr: |
| 184 | self.prev_tool_call_arr = [{"arguments": {}}] |
| 185 | |
| 186 | if tool_deltas: |
| 187 | return DeltaMessage(tool_calls=tool_deltas) |
| 188 | elif not added_text and self.current_tool_id > 0: |
| 189 | # Return an empty DeltaMessage once the tool calls are all done |
| 190 | # so that finish_reason gets set. |
| 191 | return DeltaMessage(content="") |
| 192 | else: |
| 193 | return None |
| 194 | except Exception: |
| 195 | logger.exception("Error trying to handle streaming tool call.") |
| 196 | logger.debug( |
| 197 | "Skipping chunk as a result of tool streaming extraction error" |
| 198 | ) |
| 199 | return None |