| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import importlib |
| 5 | from collections.abc import Sequence |
| 6 | from typing import Any |
| 7 | |
| 8 | from openai.types.responses.function_tool import FunctionTool |
| 9 | |
| 10 | from vllm.entrypoints.chat_utils import make_tool_call_id |
| 11 | from vllm.entrypoints.generate.base.protocol import ( |
| 12 | DeltaFunctionCall, |
| 13 | DeltaMessage, |
| 14 | DeltaToolCall, |
| 15 | ExtractedToolCallInformation, |
| 16 | FunctionCall, |
| 17 | ToolCall, |
| 18 | ) |
| 19 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 20 | ChatCompletionRequest, |
| 21 | ChatCompletionToolsParam, |
| 22 | ) |
| 23 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 24 | from vllm.logger import init_logger |
| 25 | from vllm.tokenizers import TokenizerLike |
| 26 | from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser |
| 27 | |
| 28 | logger = init_logger(__name__) |
| 29 | |
| 30 | |
| 31 | def _rust_tool_parser_module() -> Any: |
| 32 | try: |
| 33 | return importlib.import_module("vllm._rust_tool_parser") |
| 34 | except ImportError as exc: |
| 35 | raise RuntimeError( |
| 36 | "Rust tool parsing requires the vllm._rust_tool_parser PyO3 " |
| 37 | "extension. Rebuild vLLM with Rust frontend/extensions enabled." |
| 38 | ) from exc |
| 39 | |
| 40 | |
| 41 | class RustToolParser(ToolParser): |
| 42 | """Adapter from an opaque Rust parser to the vLLM ToolParser API. |
| 43 | |
| 44 | Subclasses provide only model-specific configuration: the exact Rust parser |
| 45 | name and an optional tool-call start marker for fast complete-output |
| 46 | rejection. |
| 47 | |
| 48 | This class keeps the vLLM-specific bridge work: |
| 49 | - convert vLLM tool definitions into the Rust ``Tool`` shape; |
| 50 | - translate typed Rust parser outputs into vLLM protocol objects; and |
| 51 | - maintain vLLM streaming bookkeeping used by finish-reason handling. |
| 52 | |
| 53 | The parser grammar and incremental parser state stay in Rust. |
| 54 | """ |
| 55 | |
| 56 | # Rust-backed parsers are opaque to Python by default. Do not use vLLM's |
| 57 | # standard JSON required/named handling; let the Rust parser consume the |
| 58 | # model's native tool-call syntax. |
| 59 | supports_required_and_named = False |
| 60 | |
| 61 | rust_parser_name: str |
| 62 | tool_call_start_token: str | None = None |
| 63 | |
| 64 | def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): |
| 65 | super().__init__(tokenizer, tools) |
| 66 | self._parser: Any | None = None |
| 67 | self._error: Exception | None = None |
| 68 | |
| 69 | if not self.model_tokenizer: |
| 70 | raise ValueError( |
| 71 | "The model tokenizer must be passed to the ToolParser " |
| 72 | "constructor during construction." |
| 73 | ) |
| 74 | |
| 75 | logger.debug( |
| 76 | "vLLM successfully imported tool parser %s", self.__class__.__name__ |
| 77 | ) |
| 78 | |
| 79 | def adjust_request( |
| 80 | self, request: ChatCompletionRequest | ResponsesRequest |
| 81 | ) -> ChatCompletionRequest | ResponsesRequest: |
| 82 | """Adjust request options without installing Python-side constraints. |
| 83 | |
| 84 | Rust-backed parsers are treated as source-of-truth opaque parsers. The |
| 85 | bridge intentionally avoids ``super().adjust_request()`` so Python does |
| 86 | not install JSON schema guidance or structural-tag constraints that may |
| 87 | conflict with the Rust parser's native grammar. |
| 88 | """ |
| 89 | if self._get_parser().preserve_special_tokens(): |
| 90 | request.skip_special_tokens = False |
| 91 | return request |
| 92 | |
| 93 | def _rust_tools(self) -> list[Any]: |
| 94 | """Build Rust ``Tool`` objects from vLLM tool definitions.""" |
| 95 | if not self.tools: |
| 96 | return [] |
| 97 | |
| 98 | tools: list[Any] = [] |
| 99 | for tool in self.tools: |
| 100 | if isinstance(tool, FunctionTool): |
| 101 | name = tool.name |
| 102 | description = tool.description |
| 103 | parameters = tool.parameters or {} |
| 104 | strict = getattr(tool, "strict", None) |
| 105 | elif isinstance(tool, ChatCompletionToolsParam): |
| 106 | name = tool.function.name |
| 107 | description = tool.function.description |
| 108 | parameters = tool.function.parameters or {} |
| 109 | strict = getattr(tool.function, "strict", None) |
| 110 | else: |
| 111 | continue |
| 112 | tools.append( |
| 113 | _rust_tool_parser_module().Tool(name, description, parameters, strict) |
| 114 | ) |
| 115 | return tools |
| 116 | |
| 117 | def _new_parser(self) -> Any: |
| 118 | """Create a fresh Rust parser with the current tool schemas.""" |
| 119 | return _rust_tool_parser_module().ToolParser( |
| 120 | self.rust_parser_name, self._rust_tools() |
| 121 | ) |
| 122 | |
| 123 | def _get_parser(self) -> Any: |
| 124 | if self._parser is None: |
| 125 | self._parser = self._new_parser() |
| 126 | return self._parser |
| 127 | |
| 128 | def _reset_streaming_state(self) -> None: |
| 129 | """Reset parser state for a new request on a reused parser instance.""" |
| 130 | self._parser = self._new_parser() |
| 131 | self._error = None |
| 132 | self.prev_tool_call_arr.clear() |
| 133 | self.streamed_args_for_tool.clear() |
| 134 | self.current_tool_id = -1 |
| 135 | self.current_tool_name_sent = False |
| 136 | |
| 137 | def _ensure_tool_state(self, index: int) -> None: |
| 138 | """Grow vLLM streaming state arrays to contain ``index``.""" |
| 139 | while len(self.prev_tool_call_arr) <= index: |
| 140 | self.prev_tool_call_arr.append({}) |
| 141 | while len(self.streamed_args_for_tool) <= index: |
| 142 | self.streamed_args_for_tool.append("") |
| 143 | |
| 144 | def _record_delta( |
| 145 | self, index: int, name: str | None, arguments: str | None |
| 146 | ) -> str | None: |
| 147 | """Mirror a Rust parser delta into vLLM streaming bookkeeping. |
| 148 | |
| 149 | ``prev_tool_call_arr`` and ``streamed_args_for_tool`` are read later by |
| 150 | the chat serving layer to decide the final ``tool_calls`` finish reason |
| 151 | and to flush any remaining argument bytes. |
| 152 | """ |
| 153 | tool_call_id = None |
| 154 | self._ensure_tool_state(index) |
| 155 | |
| 156 | if name is not None: |
| 157 | # Prefer the model-emitted ID surfaced by the Rust parser (e.g. |
| 158 | # Kimi K2) over a randomly generated one. |
| 159 | tool_call_id = self._get_parser().tool_call_id(index) or make_tool_call_id() |
| 160 | self.prev_tool_call_arr[index] = {"name": name, "arguments": {}} |
| 161 | self.current_tool_name_sent = True |
| 162 | |
| 163 | if arguments is not None: |
| 164 | self.streamed_args_for_tool[index] += arguments |
| 165 | self.prev_tool_call_arr[index]["arguments"] = self.streamed_args_for_tool[ |
| 166 | index |
| 167 | ] |
| 168 | self.current_tool_id = index |
| 169 | |
| 170 | return tool_call_id |
| 171 | |
| 172 | def _delta_message_from_parser_output( |
| 173 | self, parser_output: Any | None |
| 174 | ) -> DeltaMessage | None: |
| 175 | """Translate one Rust parser output into a vLLM ``DeltaMessage``.""" |
| 176 | if parser_output is None: |
| 177 | return None |
| 178 | |
| 179 | normal_text = parser_output.normal_text or None |
| 180 | tool_calls: list[DeltaToolCall] = [] |
| 181 | for tool_call in parser_output.calls: |
| 182 | index = tool_call.tool_index |
| 183 | name = tool_call.name |
| 184 | arguments: str | None = tool_call.arguments |
| 185 | if name is None and arguments is None: |
| 186 | continue |
| 187 | |
| 188 | tool_call_id = self._record_delta(index, name, arguments) |
| 189 | tool_calls.append( |
| 190 | DeltaToolCall( |
| 191 | index=index, |
| 192 | id=tool_call_id, |
| 193 | type="function" if name is not None else None, |
| 194 | function=DeltaFunctionCall( |
| 195 | name=name, |
| 196 | arguments=arguments, |
| 197 | ), |
| 198 | ) |
| 199 | ) |
| 200 | |
| 201 | if normal_text is None and not tool_calls: |
| 202 | return None |
| 203 | return DeltaMessage(content=normal_text, tool_calls=tool_calls) |
| 204 | |
| 205 | def _parse_complete(self, model_output: str) -> tuple[Any, dict[int, str]] | None: |
| 206 | """Parse complete model output with a throwaway Rust parser instance. |
| 207 | |
| 208 | Returns the coalesced parser output along with any model-emitted tool |
| 209 | call IDs keyed by tool index. |
| 210 | """ |
| 211 | parser = self._new_parser() |
| 212 | output = _rust_tool_parser_module().ToolParserOutput() |
| 213 | try: |
| 214 | parser.parse_into(model_output, output) |
| 215 | # finish() clears parser state, so snapshot model-emitted IDs first. |
| 216 | tool_call_ids = { |
| 217 | call.tool_index: tool_call_id |
| 218 | for call in output.calls |
| 219 | if (tool_call_id := parser.tool_call_id(call.tool_index)) is not None |
| 220 | } |
| 221 | output.append(parser.finish()) |
| 222 | except Exception: |
| 223 | logger.exception( |
| 224 | "Error parsing %s tool call output.", self.rust_parser_name |
| 225 | ) |
| 226 | return None |
| 227 | return output.coalesce(), tool_call_ids |
| 228 | |
| 229 | def extract_tool_calls( |
| 230 | self, |
| 231 | model_output: str, |
| 232 | request: ChatCompletionRequest, |
| 233 | ) -> ExtractedToolCallInformation: |
| 234 | """Extract tool calls from complete model output (non-streaming).""" |
| 235 | if ( |
| 236 | self.tool_call_start_token is not None |
| 237 | and self.tool_call_start_token not in model_output |
| 238 | ): |
| 239 | return ExtractedToolCallInformation( |
| 240 | tools_called=False, |
| 241 | tool_calls=[], |
| 242 | content=model_output, |
| 243 | ) |
| 244 | |
| 245 | parse_result = self._parse_complete(model_output) |
| 246 | if parse_result is None: |
| 247 | return ExtractedToolCallInformation( |
| 248 | tools_called=False, |
| 249 | tool_calls=[], |
| 250 | content=model_output, |
| 251 | ) |
| 252 | parsed, tool_call_ids = parse_result |
| 253 | |
| 254 | tool_calls: list[ToolCall] = [] |
| 255 | self.prev_tool_call_arr.clear() |
| 256 | for parsed_tool_call in parsed.calls: |
| 257 | name = parsed_tool_call.name |
| 258 | arguments = parsed_tool_call.arguments or "{}" |
| 259 | if name is None: |
| 260 | continue |
| 261 | tool_calls.append( |
| 262 | ToolCall( |
| 263 | id=tool_call_ids.get(parsed_tool_call.tool_index) |
| 264 | or make_tool_call_id(), |
| 265 | type="function", |
| 266 | function=FunctionCall(name=name, arguments=arguments), |
| 267 | ) |
| 268 | ) |
| 269 | self.prev_tool_call_arr.append({"name": name, "arguments": arguments}) |
| 270 | |
| 271 | if not tool_calls: |
| 272 | return ExtractedToolCallInformation( |
| 273 | tools_called=False, |
| 274 | tool_calls=[], |
| 275 | content=model_output, |
| 276 | ) |
| 277 | |
| 278 | content = parsed.normal_text or None |
| 279 | return ExtractedToolCallInformation( |
| 280 | tools_called=True, |
| 281 | tool_calls=tool_calls, |
| 282 | content=content, |
| 283 | ) |
| 284 | |
| 285 | def extract_tool_calls_streaming( |
| 286 | self, |
| 287 | previous_text: str, |
| 288 | current_text: str, |
| 289 | delta_text: str, |
| 290 | previous_token_ids: Sequence[int], # pylint: disable=unused-argument |
| 291 | current_token_ids: Sequence[int], # pylint: disable=unused-argument |
| 292 | delta_token_ids: Sequence[int], # pylint: disable=unused-argument |
| 293 | request: ChatCompletionRequest, # pylint: disable=unused-argument |
| 294 | ) -> DeltaMessage | None: |
| 295 | """Extract tool calls from streaming model output. |
| 296 | |
| 297 | The Rust parser owns the incremental buffer, so this adapter feeds only |
| 298 | the newest text delta and lets the serving layer handle final empty |
| 299 | chunks. |
| 300 | """ |
| 301 | # TODO: Add a final-chunk hook if streaming needs to call Rust finish(). |
| 302 | if not previous_text: |
| 303 | self._reset_streaming_state() |
| 304 | |
| 305 | if self._error is not None: |
| 306 | return None |
| 307 | |
| 308 | parser_output = _rust_tool_parser_module().ToolParserOutput() |
| 309 | try: |
| 310 | self._get_parser().parse_into(delta_text, parser_output) |
| 311 | except Exception as error: |
| 312 | self._error = error |
| 313 | logger.exception( |
| 314 | "Error parsing %s streaming tool call output.", |
| 315 | self.rust_parser_name, |
| 316 | ) |
| 317 | |
| 318 | delta_message = self._delta_message_from_parser_output(parser_output) |
| 319 | if delta_message is not None: |
| 320 | return delta_message |
| 321 | |
| 322 | return None |