| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import importlib |
| 5 | import json |
| 6 | from collections.abc import Callable, Sequence |
| 7 | from functools import cached_property |
| 8 | from typing import Any |
| 9 | |
| 10 | from openai.types.responses import ( |
| 11 | ResponseFormatTextJSONSchemaConfig, |
| 12 | ResponseTextConfig, |
| 13 | ) |
| 14 | from openai.types.responses.function_tool import FunctionTool |
| 15 | |
| 16 | import vllm.envs as envs |
| 17 | from vllm.entrypoints.generate.base.protocol import ( |
| 18 | DeltaMessage, |
| 19 | ExtractedToolCallInformation, |
| 20 | ) |
| 21 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 22 | ChatCompletionRequest, |
| 23 | ChatCompletionToolsParam, |
| 24 | ) |
| 25 | from vllm.entrypoints.openai.responses.protocol import ( |
| 26 | ResponsesRequest, |
| 27 | ) |
| 28 | from vllm.logger import init_logger |
| 29 | from vllm.sampling_params import ( |
| 30 | StructuredOutputsParams, |
| 31 | ) |
| 32 | from vllm.tokenizers import TokenizerLike |
| 33 | from vllm.tool_parsers.tool_strict_level import ToolStrictLevel |
| 34 | from vllm.tool_parsers.utils import Tool, get_json_schema_from_tools |
| 35 | from vllm.utils.collection_utils import is_list_of |
| 36 | from vllm.utils.import_utils import import_plugin |
| 37 | |
| 38 | __all__ = ["Tool"] |
| 39 | |
| 40 | logger = init_logger(__name__) |
| 41 | |
| 42 | |
| 43 | class ToolParser: |
| 44 | """Abstract ToolParser class that should not be used directly. Provided |
| 45 | properties and methods should be used in |
| 46 | derived classes. |
| 47 | """ |
| 48 | |
| 49 | # When True (default), the serving layer uses the standard JSON-based |
| 50 | # parsing for tool_choice="required" and named function tool_choice, |
| 51 | # which works for models where guided decoding produces well-formed |
| 52 | # JSON output (e.g. Hermes). |
| 53 | # Subclasses set False when the standard parsing does not work for |
| 54 | # their model's output format (e.g. GLM models that use XML). When |
| 55 | # False, the serving layer falls back to the tool_parser's |
| 56 | # extract_tool_calls / extract_tool_calls_streaming methods for |
| 57 | # required/named tool_choice, treating them the same as "auto". |
| 58 | supports_required_and_named: bool = True |
| 59 | # xgrammar builtin structural tag model key. Subclasses set this when |
| 60 | # their parsed tool-call syntax matches a builtin xgrammar format. |
| 61 | structural_tag_model: str | None = None |
| 62 | engine_based_streaming: bool = False |
| 63 | |
| 64 | def __init_subclass__(cls, **kwargs: Any) -> None: |
| 65 | super().__init_subclass__(**kwargs) |
| 66 | if ( |
| 67 | cls.structural_tag_model is not None |
| 68 | and envs.VLLM_ENFORCE_STRICT_TOOL_CALLING |
| 69 | ): |
| 70 | cls.supports_required_and_named = False |
| 71 | |
| 72 | def __init__( |
| 73 | self, |
| 74 | tokenizer: TokenizerLike, |
| 75 | tools: list[Tool] | None = None, |
| 76 | ): |
| 77 | self.prev_tool_call_arr: list[dict] = [] |
| 78 | # the index of the tool call that is currently being parsed |
| 79 | self.current_tool_id: int = -1 |
| 80 | self.current_tool_name_sent: bool = False |
| 81 | self.streamed_args_for_tool: list[str] = [] |
| 82 | |
| 83 | self.model_tokenizer = tokenizer |
| 84 | if tools: |
| 85 | self.tools: list[ChatCompletionToolsParam | FunctionTool] = [ |
| 86 | tool |
| 87 | for tool in tools |
| 88 | if isinstance(tool, (ChatCompletionToolsParam, FunctionTool)) |
| 89 | ] |
| 90 | else: |
| 91 | self.tools = [] |
| 92 | |
| 93 | def get_remaining_unstreamed_args(self) -> str: |
| 94 | """Return tool call arguments parsed but not yet streamed.""" |
| 95 | if not self.prev_tool_call_arr: |
| 96 | return "" |
| 97 | index = len(self.prev_tool_call_arr) - 1 |
| 98 | args = self.prev_tool_call_arr[index].get("arguments", {}) |
| 99 | if isinstance(args, str): |
| 100 | expected = args |
| 101 | else: |
| 102 | expected = json.dumps(args, ensure_ascii=False) |
| 103 | actual = ( |
| 104 | self.streamed_args_for_tool[index] |
| 105 | if index < len(self.streamed_args_for_tool) |
| 106 | else "" |
| 107 | ) |
| 108 | if expected.startswith(actual): |
| 109 | return expected[len(actual) :] |
| 110 | return "" |
| 111 | |
| 112 | @cached_property |
| 113 | def vocab(self) -> dict[str, int]: |
| 114 | # NOTE: Only TokenizersBackend is guaranteed to have .vocab |
| 115 | # whereas all tokenizers have .get_vocab() |
| 116 | return self.model_tokenizer.get_vocab() |
| 117 | |
| 118 | def adjust_request( |
| 119 | self, |
| 120 | request: ChatCompletionRequest | ResponsesRequest, |
| 121 | ) -> ChatCompletionRequest | ResponsesRequest: |
| 122 | # If there are no tools, return the request as is. |
| 123 | if not request.tools: |
| 124 | return request |
| 125 | |
| 126 | # Set structured output params when tool constraints are derived from |
| 127 | # the tool schema. Unified parsers handle model-specific structural |
| 128 | # tags before calling into the tool parser. |
| 129 | structured_outputs = getattr(request, "structured_outputs", None) |
| 130 | if ( |
| 131 | structured_outputs is not None |
| 132 | and structured_outputs.structural_tag is not None |
| 133 | ): |
| 134 | return request |
| 135 | |
| 136 | # Parsers that extract required/named tool calls from their native |
| 137 | # format (supports_required_and_named=False) must not be forced into |
| 138 | # the JSON tool-call format below: they would never find a call in |
| 139 | # the JSON output and would return it as content instead. |
| 140 | if not self.supports_required_and_named: |
| 141 | return request |
| 142 | |
| 143 | json_schema_from_tool = get_json_schema_from_tools( |
| 144 | tool_choice=request.tool_choice, |
| 145 | tools=request.tools, |
| 146 | parallel_tool_calls=request.parallel_tool_calls, |
| 147 | ) |
| 148 | # Set structured output params for tool calling |
| 149 | if json_schema_from_tool is not None: |
| 150 | if isinstance(request, ChatCompletionRequest): |
| 151 | # tool_choice: "Forced Function" or "required" will override |
| 152 | # structured output json settings to make tool calling work correctly |
| 153 | request.structured_outputs = StructuredOutputsParams( |
| 154 | json=json_schema_from_tool # type: ignore[call-arg] |
| 155 | ) |
| 156 | request.response_format = None |
| 157 | if isinstance(request, ResponsesRequest): |
| 158 | # Single-shot construction so Pydantic v2 tracks `format` |
| 159 | # in __fields_set__ — assigning to `.format` after the bare |
| 160 | # `ResponseTextConfig()` constructor does not, which can |
| 161 | # drop the nested config from `model_dump`. Also drop the |
| 162 | # `description` kwarg: it is not a field on |
| 163 | # ResponseFormatTextJSONSchemaConfig and was being silently |
| 164 | # passed through as extra. |
| 165 | request.text = ResponseTextConfig( |
| 166 | format=ResponseFormatTextJSONSchemaConfig( |
| 167 | type="json_schema", |
| 168 | name="tool_calling_response", |
| 169 | schema=json_schema_from_tool, |
| 170 | strict=True, |
| 171 | ) |
| 172 | ) |
| 173 | |
| 174 | return request |
| 175 | |
| 176 | def get_structural_tag( |
| 177 | self, |
| 178 | request: ChatCompletionRequest | ResponsesRequest, |
| 179 | *, |
| 180 | reasoning: bool = False, |
| 181 | strict_level: ToolStrictLevel = ToolStrictLevel.AUTO, |
| 182 | ): |
| 183 | if self.structural_tag_model is None: |
| 184 | return None |
| 185 | if not envs.VLLM_ENFORCE_STRICT_TOOL_CALLING: |
| 186 | return None |
| 187 | from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag |
| 188 | |
| 189 | return get_model_structural_tag( |
| 190 | model=self.structural_tag_model, |
| 191 | tools=request.tools, |
| 192 | tool_choice=request.tool_choice, |
| 193 | reasoning=reasoning, |
| 194 | strict_level=strict_level, |
| 195 | ) |
| 196 | |
| 197 | def extract_tool_calls( |
| 198 | self, model_output: str, request: ChatCompletionRequest |
| 199 | ) -> ExtractedToolCallInformation: |
| 200 | """Static method that should be implemented for extracting tool calls from |
| 201 | a complete model-generated string. |
| 202 | Used for non-streaming responses where we have the entire model response |
| 203 | available before sending to the client. |
| 204 | Static because it's stateless. |
| 205 | """ |
| 206 | raise NotImplementedError( |
| 207 | "AbstractToolParser.extract_tool_calls has not been implemented!" |
| 208 | ) |
| 209 | |
| 210 | def extract_tool_calls_streaming( |
| 211 | self, |
| 212 | previous_text: str, |
| 213 | current_text: str, |
| 214 | delta_text: str, |
| 215 | previous_token_ids: Sequence[int], |
| 216 | current_token_ids: Sequence[int], |
| 217 | delta_token_ids: Sequence[int], |
| 218 | request: ChatCompletionRequest, |
| 219 | ) -> DeltaMessage | None: |
| 220 | """Instance method that should be implemented for extracting tool calls |
| 221 | from an incomplete response; for use when handling tool calls and |
| 222 | streaming. Has to be an instance method because it requires state - |
| 223 | the current tokens/diffs, but also the information about what has |
| 224 | previously been parsed and extracted (see constructor) |
| 225 | """ |
| 226 | raise NotImplementedError( |
| 227 | "AbstractToolParser.extract_tool_calls_streaming has not been implemented!" |
| 228 | ) |
| 229 | |
| 230 | |
| 231 | class ToolParserManager: |
| 232 | """Central registry for ToolParser implementations. |
| 233 | |
| 234 | Supports two modes: |
| 235 | - Eager (immediate) registration via `register_module` |
| 236 | - Lazy registration via `register_lazy_module` |
| 237 | """ |
| 238 | |
| 239 | tool_parsers: dict[str, type[ToolParser]] = {} |
| 240 | lazy_parsers: dict[str, tuple[str, str]] = {} # name -> (module_path, class_name) |
| 241 | |
| 242 | @classmethod |
| 243 | def get_tool_parser(cls, name: str) -> type[ToolParser]: |
| 244 | """Retrieve a registered or lazily registered ToolParser class. |
| 245 | |
| 246 | If the parser is lazily registered, |
| 247 | it will be imported and cached on first access. |
| 248 | Raises KeyError if not found. |
| 249 | """ |
| 250 | if name in cls.tool_parsers: |
| 251 | return cls.tool_parsers[name] |
| 252 | |
| 253 | if name in cls.lazy_parsers: |
| 254 | return cls._load_lazy_parser(name) |
| 255 | |
| 256 | raise KeyError(f"Tool parser '{name}' not found.") |
| 257 | |
| 258 | @classmethod |
| 259 | def _load_lazy_parser(cls, name: str) -> type[ToolParser]: |
| 260 | """Import and register a lazily loaded parser.""" |
| 261 | module_path, class_name = cls.lazy_parsers[name] |
| 262 | try: |
| 263 | mod = importlib.import_module(module_path) |
| 264 | parser_cls = getattr(mod, class_name) |
| 265 | if not issubclass(parser_cls, ToolParser): |
| 266 | raise TypeError( |
| 267 | f"{class_name} in {module_path} is not a ToolParser subclass." |
| 268 | ) |
| 269 | cls.tool_parsers[name] = parser_cls # cache |
| 270 | return parser_cls |
| 271 | except Exception as e: |
| 272 | logger.exception( |
| 273 | "Failed to import lazy tool parser '%s' from %s: %s", |
| 274 | name, |
| 275 | module_path, |
| 276 | e, |
| 277 | ) |
| 278 | raise |
| 279 | |
| 280 | @classmethod |
| 281 | def _register_module( |
| 282 | cls, |
| 283 | module: type[ToolParser], |
| 284 | module_name: str | list[str] | None = None, |
| 285 | force: bool = True, |
| 286 | ) -> None: |
| 287 | """Register a ToolParser class immediately.""" |
| 288 | if not issubclass(module, ToolParser): |
| 289 | raise TypeError( |
| 290 | f"module must be subclass of ToolParser, but got {type(module)}" |
| 291 | ) |
| 292 | |
| 293 | if module_name is None: |
| 294 | module_name = module.__name__ |
| 295 | |
| 296 | if isinstance(module_name, str): |
| 297 | module_names = [module_name] |
| 298 | elif is_list_of(module_name, str): |
| 299 | module_names = module_name |
| 300 | else: |
| 301 | raise TypeError("module_name must be str, list[str], or None.") |
| 302 | |
| 303 | for name in module_names: |
| 304 | if not force and name in cls.tool_parsers: |
| 305 | existed = cls.tool_parsers[name] |
| 306 | raise KeyError(f"{name} is already registered at {existed.__module__}") |
| 307 | cls.tool_parsers[name] = module |
| 308 | |
| 309 | @classmethod |
| 310 | def register_lazy_module(cls, name: str, module_path: str, class_name: str) -> None: |
| 311 | """Register a lazy module mapping. |
| 312 | |
| 313 | Example: |
| 314 | ToolParserManager.register_lazy_module( |
| 315 | name="kimi_k2", |
| 316 | module_path="vllm.tool_parsers.kimi_k2_parser", |
| 317 | class_name="KimiK2ToolParser", |
| 318 | ) |
| 319 | |
| 320 | """ |
| 321 | cls.lazy_parsers[name] = (module_path, class_name) |
| 322 | |
| 323 | @classmethod |
| 324 | def register_module( |
| 325 | cls, |
| 326 | name: str | list[str] | None = None, |
| 327 | force: bool = True, |
| 328 | module: type[ToolParser] | None = None, |
| 329 | ) -> type[ToolParser] | Callable[[type[ToolParser]], type[ToolParser]]: |
| 330 | """Register module immediately or lazily (as a decorator). |
| 331 | |
| 332 | Usage: |
| 333 | @ToolParserManager.register_module("kimi_k2") |
| 334 | class KimiK2ToolParser(ToolParser): |
| 335 | ... |
| 336 | |
| 337 | Or: |
| 338 | ToolParserManager.register_module(module=SomeToolParser) |
| 339 | """ |
| 340 | if not isinstance(force, bool): |
| 341 | raise TypeError(f"force must be a boolean, but got {type(force)}") |
| 342 | |
| 343 | # Immediate registration |
| 344 | if module is not None: |
| 345 | cls._register_module(module=module, module_name=name, force=force) |
| 346 | return module |
| 347 | |
| 348 | # Decorator usage |
| 349 | def _decorator(obj: type[ToolParser]) -> type[ToolParser]: |
| 350 | module_path = obj.__module__ |
| 351 | class_name = obj.__name__ |
| 352 | |
| 353 | if isinstance(name, str): |
| 354 | names = [name] |
| 355 | elif name is not None and is_list_of(name, str): |
| 356 | names = name |
| 357 | else: |
| 358 | names = [class_name] |
| 359 | |
| 360 | for n in names: |
| 361 | # Lazy mapping only: do not import now |
| 362 | cls.lazy_parsers[n] = (module_path, class_name) |
| 363 | |
| 364 | return obj |
| 365 | |
| 366 | return _decorator |
| 367 | |
| 368 | @classmethod |
| 369 | def list_registered(cls) -> list[str]: |
| 370 | """Return names of all eagerly and lazily registered tool parsers.""" |
| 371 | return sorted(set(cls.tool_parsers.keys()) | set(cls.lazy_parsers.keys())) |
| 372 | |
| 373 | @classmethod |
| 374 | def import_tool_parser(cls, plugin_path: str) -> None: |
| 375 | """Import a user-defined tool parser.""" |
| 376 | import_plugin(plugin_path) |