| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | from collections.abc import Sequence |
| 6 | |
| 7 | import regex as re |
| 8 | |
| 9 | from vllm.entrypoints.chat_utils import make_tool_call_id |
| 10 | from vllm.entrypoints.generate.base.protocol import ( |
| 11 | DeltaFunctionCall, |
| 12 | DeltaMessage, |
| 13 | DeltaToolCall, |
| 14 | ExtractedToolCallInformation, |
| 15 | FunctionCall, |
| 16 | ToolCall, |
| 17 | ) |
| 18 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 19 | ChatCompletionRequest, |
| 20 | ) |
| 21 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 22 | from vllm.logger import init_logger |
| 23 | from vllm.tokenizers import TokenizerLike |
| 24 | from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser |
| 25 | |
| 26 | logger = init_logger(__name__) |
| 27 | |
| 28 | REGEX_FUNCTION_CALL = re.compile( |
| 29 | r"(?:function call<\|role_sep\|>\n|<\|function_call\|>)(.*)", |
| 30 | re.DOTALL, |
| 31 | ) |
| 32 | |
| 33 | REGEX_CONTENT_PATTERN = re.compile( |
| 34 | r"^(.*?)(?:<\|message_sep\|>|<\|function_call\|>)", |
| 35 | re.DOTALL, |
| 36 | ) |
| 37 | |
| 38 | NAME_REGEX = re.compile( |
| 39 | r'"name"\s*:\s*"([^"]*)"', |
| 40 | re.DOTALL, |
| 41 | ) |
| 42 | |
| 43 | ARGS_REGEX = re.compile( |
| 44 | r'"arguments"\s*:\s*(.*)', |
| 45 | re.DOTALL, |
| 46 | ) |
| 47 | |
| 48 | |
| 49 | class GigaChat3ToolParser(ToolParser): |
| 50 | def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): |
| 51 | super().__init__(tokenizer, tools) |
| 52 | self.tool_started: bool = False |
| 53 | self.tool_name_sent: bool = False |
| 54 | self.tool_id: str | None = None |
| 55 | self.prev_tool_call_arr: list[dict] = [] |
| 56 | self.end_content: bool = False |
| 57 | self.streamed_args_for_tool: list[str] = [] |
| 58 | |
| 59 | def adjust_request( |
| 60 | self, request: ChatCompletionRequest | ResponsesRequest |
| 61 | ) -> ChatCompletionRequest | ResponsesRequest: |
| 62 | request = super().adjust_request(request) |
| 63 | if request.tools and request.tool_choice != "none": |
| 64 | request.skip_special_tokens = False |
| 65 | return request |
| 66 | |
| 67 | def extract_tool_calls( |
| 68 | self, |
| 69 | model_output: str, |
| 70 | request: ChatCompletionRequest, |
| 71 | ) -> ExtractedToolCallInformation: |
| 72 | function_call = None |
| 73 | content = None |
| 74 | if model_output.rstrip().endswith("</s>"): |
| 75 | model_output = model_output[: model_output.rfind("</s>")] |
| 76 | m_func = REGEX_FUNCTION_CALL.search(model_output) |
| 77 | if m_func: |
| 78 | try: |
| 79 | function_call = json.loads(m_func.group(1), strict=False) |
| 80 | if ( |
| 81 | isinstance(function_call, dict) |
| 82 | and "name" in function_call |
| 83 | and "arguments" in function_call |
| 84 | ): |
| 85 | if not isinstance(function_call["arguments"], dict): |
| 86 | function_call = None |
| 87 | else: |
| 88 | function_call = None |
| 89 | except json.JSONDecodeError: |
| 90 | return ExtractedToolCallInformation( |
| 91 | tools_called=False, |
| 92 | tool_calls=[], |
| 93 | content=model_output, |
| 94 | ) |
| 95 | m_content = REGEX_CONTENT_PATTERN.search(model_output) |
| 96 | content = m_content.group(1) if m_content else model_output |
| 97 | if not function_call: |
| 98 | return ExtractedToolCallInformation( |
| 99 | tools_called=False, |
| 100 | tool_calls=[], |
| 101 | content=content if content else None, |
| 102 | ) |
| 103 | name = function_call["name"] |
| 104 | args = function_call["arguments"] |
| 105 | if not isinstance(args, str): |
| 106 | args = json.dumps(function_call["arguments"], ensure_ascii=False) |
| 107 | return ExtractedToolCallInformation( |
| 108 | tools_called=True, |
| 109 | tool_calls=[ |
| 110 | ToolCall( |
| 111 | type="function", |
| 112 | function=FunctionCall( |
| 113 | name=name, |
| 114 | arguments=args, |
| 115 | ), |
| 116 | ) |
| 117 | ], |
| 118 | content=content if content else None, |
| 119 | ) |
| 120 | |
| 121 | def extract_tool_calls_streaming( |
| 122 | self, |
| 123 | previous_text: str, |
| 124 | current_text: str, |
| 125 | delta_text: str, |
| 126 | previous_token_ids: Sequence[int], |
| 127 | current_token_ids: Sequence[int], |
| 128 | delta_token_ids: Sequence[int], |
| 129 | request: ChatCompletionRequest, |
| 130 | ) -> DeltaMessage | None: |
| 131 | content = None |
| 132 | func_name = None |
| 133 | cur_args = None |
| 134 | m_func = REGEX_FUNCTION_CALL.search(current_text) |
| 135 | if not self.tool_started: |
| 136 | m_content = REGEX_CONTENT_PATTERN.search(delta_text) |
| 137 | if m_content: |
| 138 | content = m_content.group(1) |
| 139 | self.end_content = True |
| 140 | else: |
| 141 | if not self.end_content: |
| 142 | content = delta_text |
| 143 | if m_func: |
| 144 | self.tool_started = True |
| 145 | if content: |
| 146 | return DeltaMessage(content=content) |
| 147 | if not m_func: |
| 148 | return None |
| 149 | json_tail = m_func.group(1).strip() |
| 150 | name_match = NAME_REGEX.search(json_tail) |
| 151 | if name_match: |
| 152 | func_name = name_match.group(1) |
| 153 | args_match = ARGS_REGEX.search(json_tail) |
| 154 | if args_match: |
| 155 | cur_args = args_match.group(1).strip() |
| 156 | if cur_args.endswith("</s>"): |
| 157 | cur_args = cur_args[: -len("</s>")] |
| 158 | if cur_args.endswith("}"): # last '}' end of json |
| 159 | try: |
| 160 | candidate = cur_args[:-1].strip() |
| 161 | json.loads(candidate, strict=False) |
| 162 | cur_args = candidate |
| 163 | except json.JSONDecodeError: |
| 164 | pass |
| 165 | if not self.prev_tool_call_arr: |
| 166 | self.prev_tool_call_arr.append({}) |
| 167 | if not self.tool_name_sent: |
| 168 | if not func_name: |
| 169 | return None |
| 170 | self.tool_name_sent = True |
| 171 | self.tool_id = make_tool_call_id() |
| 172 | self.prev_tool_call_arr[0]["name"] = func_name |
| 173 | return DeltaMessage( |
| 174 | tool_calls=[ |
| 175 | DeltaToolCall( |
| 176 | index=0, |
| 177 | id=self.tool_id, |
| 178 | type="function", |
| 179 | function=DeltaFunctionCall( |
| 180 | name=func_name, |
| 181 | ).model_dump(exclude_none=True), |
| 182 | ) |
| 183 | ], |
| 184 | ) |
| 185 | if cur_args is None: |
| 186 | return None |
| 187 | prev_args = self.prev_tool_call_arr[0].get("arguments_str", "") |
| 188 | if not prev_args: |
| 189 | delta_args = cur_args |
| 190 | elif cur_args.startswith(prev_args): |
| 191 | delta_args = cur_args[len(prev_args) :] |
| 192 | else: |
| 193 | return None |
| 194 | if not delta_args: |
| 195 | return None |
| 196 | self.prev_tool_call_arr[0]["arguments_str"] = cur_args |
| 197 | try: |
| 198 | args_dict = json.loads(cur_args, strict=False) |
| 199 | self.prev_tool_call_arr[0]["arguments"] = args_dict |
| 200 | except json.JSONDecodeError: |
| 201 | self.prev_tool_call_arr[0]["arguments"] = {} |
| 202 | if len(self.streamed_args_for_tool) <= 0: |
| 203 | self.streamed_args_for_tool.append("") |
| 204 | self.streamed_args_for_tool[0] = cur_args |
| 205 | return DeltaMessage( |
| 206 | tool_calls=[ |
| 207 | DeltaToolCall( |
| 208 | index=0, |
| 209 | function=DeltaFunctionCall( |
| 210 | arguments=delta_args, |
| 211 | ).model_dump(exclude_none=True), |
| 212 | ) |
| 213 | ], |
| 214 | ) |