| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import functools |
| 7 | import json |
| 8 | from collections.abc import Sequence |
| 9 | from enum import Enum, auto |
| 10 | from random import choices |
| 11 | from string import ascii_letters, digits |
| 12 | from typing import TYPE_CHECKING, Any, Literal |
| 13 | |
| 14 | import ijson |
| 15 | import regex as re |
| 16 | from mistral_common.protocol.instruct.tool_calls import ( |
| 17 | NamedToolChoice as MistralNamedToolChoice, |
| 18 | ) |
| 19 | from mistral_common.protocol.instruct.tool_calls import ( |
| 20 | Tool as MistralTool, |
| 21 | ) |
| 22 | from mistral_common.protocol.instruct.tool_calls import ( |
| 23 | ToolChoice as MistralToolChoice, |
| 24 | ) |
| 25 | from mistral_common.protocol.instruct.tool_calls import ( |
| 26 | ToolChoiceEnum as MistralToolChoiceEnum, |
| 27 | ) |
| 28 | from mistral_common.tokens.tokenizers.base import SpecialTokens |
| 29 | from pydantic import Field |
| 30 | |
| 31 | from vllm.entrypoints.generate.base.protocol import ( |
| 32 | DeltaFunctionCall, |
| 33 | DeltaMessage, |
| 34 | DeltaToolCall, |
| 35 | ExtractedToolCallInformation, |
| 36 | FunctionCall, |
| 37 | ToolCall, |
| 38 | ) |
| 39 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 40 | ChatCompletionNamedToolChoiceParam, |
| 41 | ) |
| 42 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 43 | from vllm.logger import init_logger |
| 44 | from vllm.parser.engine.events import EventType |
| 45 | from vllm.parser.engine.parser_engine import ParserEngine |
| 46 | from vllm.parser.engine.parser_engine_config import ( |
| 47 | ParserEngineConfig, |
| 48 | ParserState, |
| 49 | Transition, |
| 50 | ) |
| 51 | from vllm.sampling_params import StructuredOutputsParams |
| 52 | from vllm.utils.mistral import is_mistral_tokenizer |
| 53 | |
| 54 | if TYPE_CHECKING: |
| 55 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 56 | ChatCompletionRequest, |
| 57 | ) |
| 58 | from vllm.parser.engine.events import SemanticEvent |
| 59 | from vllm.parser.engine.parser_engine import ToolCallSlot |
| 60 | from vllm.tokenizers import TokenizerLike |
| 61 | from vllm.tool_parsers.abstract_tool_parser import Tool |
| 62 | |
| 63 | logger = init_logger(__name__) |
| 64 | |
| 65 | _ALPHANUMERIC = ascii_letters + digits |
| 66 | _DEFAULT_JSON_SCHEMA: dict[str, Any] = { |
| 67 | "anyOf": [{"type": "object"}, {"type": "array"}] |
| 68 | } |
| 69 | |
| 70 | # Special token strings as emitted/decoded by MistralTokenizer. |
| 71 | _TOOL_CALLS = SpecialTokens.tool_calls.value |
| 72 | _ARGS = SpecialTokens.args.value |
| 73 | _THINK_START_SPECIAL = SpecialTokens.begin_think.value |
| 74 | _THINK_END_SPECIAL = SpecialTokens.end_think.value |
| 75 | |
| 76 | # Plain-text reasoning markers used by v11 tokenizers. |
| 77 | _THINK_START_TEXT = "<think>" |
| 78 | _THINK_END_TEXT = "</think>" |
| 79 | |
| 80 | _OPEN_BRACE = "{" |
| 81 | |
| 82 | |
| 83 | class StreamingState(Enum): |
| 84 | """Streaming parsing state for pre-v11 tool call extraction.""" |
| 85 | |
| 86 | WAITING_FOR_TOOL_START = auto() |
| 87 | WAITING_FOR_TOOL_KEY = auto() |
| 88 | PARSING_NAME = auto() |
| 89 | PARSING_NAME_COMPLETED = auto() |
| 90 | WAITING_FOR_ARGUMENTS_START = auto() |
| 91 | PARSING_ARGUMENTS = auto() |
| 92 | PARSING_ARGUMENTS_COMPLETED = auto() |
| 93 | TOOL_COMPLETE = auto() |
| 94 | ALL_TOOLS_COMPLETE = auto() |
| 95 | |
| 96 | |
| 97 | class MistralToolCall(ToolCall): |
| 98 | """ToolCall with a Mistral-compatible random alphanumeric id.""" |
| 99 | |
| 100 | id: str = Field(default_factory=lambda: MistralToolCall.generate_random_id()) |
| 101 | |
| 102 | @staticmethod |
| 103 | def generate_random_id() -> str: |
| 104 | # Mistral Tool Call Ids must be alphanumeric with a length of 9. |
| 105 | # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299 |
| 106 | return "".join(choices(_ALPHANUMERIC, k=9)) |
| 107 | |
| 108 | @staticmethod |
| 109 | def is_valid_id(id: str) -> bool: |
| 110 | return id.isalnum() and len(id) == 9 |
| 111 | |
| 112 | |
| 113 | def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool: |
| 114 | if is_mistral_tokenizer(model_tokenizer): |
| 115 | return model_tokenizer.version < 11 |
| 116 | vocab: dict[str, int] = getattr(model_tokenizer, "get_vocab", lambda: {})() |
| 117 | return _ARGS not in vocab |
| 118 | |
| 119 | |
| 120 | @functools.cache |
| 121 | def mistral_config( |
| 122 | *, |
| 123 | reasoning_encoding: Literal["special_token", "text", "none"], |
| 124 | name: str = "mistral", |
| 125 | ) -> ParserEngineConfig: |
| 126 | """Return a :class:`ParserEngineConfig` for the Mistral output format. |
| 127 | |
| 128 | Args: |
| 129 | reasoning_encoding: Reasoning token format. |
| 130 | ``"special_token"`` – v13+ ``[THINK]``/``[/THINK]`` special |
| 131 | tokens placed in both ``terminals`` and ``token_id_terminals``. |
| 132 | ``"text"`` – v11 ``<think>``/``</think>`` plain text placed in |
| 133 | ``terminals`` only; no ``token_id_terminals`` for think tokens. |
| 134 | ``"none"`` – no reasoning support. |
| 135 | name: Name embedded in the returned config (used for debugging). |
| 136 | |
| 137 | Returns: |
| 138 | A frozen :class:`ParserEngineConfig` with ``initial_state=CONTENT``. |
| 139 | |
| 140 | """ |
| 141 | if reasoning_encoding == "special_token": |
| 142 | think_start = _THINK_START_SPECIAL |
| 143 | think_end = _THINK_END_SPECIAL |
| 144 | reasoning_terminals: dict[str, str] = { |
| 145 | "THINK_START": think_start, |
| 146 | "THINK_END": think_end, |
| 147 | } |
| 148 | reasoning_token_id_terminals: dict[str, str] = { |
| 149 | "THINK_START": think_start, |
| 150 | "THINK_END": think_end, |
| 151 | } |
| 152 | elif reasoning_encoding == "text": |
| 153 | think_start = _THINK_START_TEXT |
| 154 | think_end = _THINK_END_TEXT |
| 155 | reasoning_terminals = { |
| 156 | "THINK_START": think_start, |
| 157 | "THINK_END": think_end, |
| 158 | } |
| 159 | # Text think markers have no token-id terminals. |
| 160 | reasoning_token_id_terminals = {} |
| 161 | else: |
| 162 | reasoning_terminals = {} |
| 163 | reasoning_token_id_terminals = {} |
| 164 | |
| 165 | if reasoning_encoding != "none": |
| 166 | reasoning_transitions: dict[tuple[ParserState, str], Transition] = { |
| 167 | (ParserState.CONTENT, "THINK_START"): Transition( |
| 168 | ParserState.REASONING, |
| 169 | (), |
| 170 | ), |
| 171 | # Absorb a duplicate/re-emitted THINK_START inside reasoning. |
| 172 | (ParserState.REASONING, "THINK_START"): Transition( |
| 173 | ParserState.REASONING, |
| 174 | (), |
| 175 | ), |
| 176 | (ParserState.REASONING, "THINK_END"): Transition( |
| 177 | ParserState.CONTENT, |
| 178 | (EventType.REASONING_END,), |
| 179 | ), |
| 180 | # Absorb stray THINK_END that arrives after reasoning ended. |
| 181 | (ParserState.CONTENT, "THINK_END"): Transition( |
| 182 | ParserState.CONTENT, |
| 183 | (), |
| 184 | ), |
| 185 | # [TOOL_CALLS] directly from reasoning implicitly ends it. |
| 186 | (ParserState.REASONING, "TOOL_CALLS"): Transition( |
| 187 | ParserState.TOOL_NAME, |
| 188 | (EventType.REASONING_END, EventType.TOOL_CALL_START), |
| 189 | ), |
| 190 | } |
| 191 | else: |
| 192 | reasoning_transitions = {} |
| 193 | |
| 194 | return ParserEngineConfig( |
| 195 | name=name, |
| 196 | initial_state=ParserState.CONTENT, |
| 197 | wait_for_reasoning=reasoning_encoding != "none", |
| 198 | terminals={ |
| 199 | **reasoning_terminals, |
| 200 | "TOOL_CALLS": _TOOL_CALLS, |
| 201 | "ARGS": _ARGS, |
| 202 | "OPEN_BRACE": _OPEN_BRACE, |
| 203 | }, |
| 204 | token_id_terminals={ |
| 205 | **reasoning_token_id_terminals, |
| 206 | "TOOL_CALLS": _TOOL_CALLS, |
| 207 | "ARGS": _ARGS, |
| 208 | }, |
| 209 | transitions={ |
| 210 | **reasoning_transitions, |
| 211 | # A tool call from content implicitly ends reasoning when enabled. |
| 212 | (ParserState.CONTENT, "TOOL_CALLS"): Transition( |
| 213 | ParserState.TOOL_NAME, |
| 214 | (EventType.REASONING_END, EventType.TOOL_CALL_START) |
| 215 | if reasoning_encoding != "none" |
| 216 | else (EventType.TOOL_CALL_START,), |
| 217 | ), |
| 218 | # NAME→ARGS via explicit [ARGS] separator (v11+): consumed, no events. |
| 219 | (ParserState.TOOL_NAME, "ARGS"): Transition( |
| 220 | ParserState.TOOL_ARGS, |
| 221 | (), |
| 222 | ), |
| 223 | # NAME→ARGS via "{": carried as ARG_VALUE_CHUNK so the opening |
| 224 | # brace lands in the JSON argument buffer (fallback for name{args}). |
| 225 | (ParserState.TOOL_NAME, "OPEN_BRACE"): Transition( |
| 226 | ParserState.TOOL_ARGS, |
| 227 | (EventType.ARG_VALUE_CHUNK,), |
| 228 | ), |
| 229 | # Parallel tool calls: next [TOOL_CALLS] ends current call. |
| 230 | (ParserState.TOOL_ARGS, "TOOL_CALLS"): Transition( |
| 231 | ParserState.TOOL_NAME, |
| 232 | (EventType.TOOL_CALL_END, EventType.TOOL_CALL_START), |
| 233 | ), |
| 234 | }, |
| 235 | stream_arg_deltas=True, |
| 236 | tool_args_json=True, |
| 237 | strip_trailing_reasoning_whitespace=True, |
| 238 | drop_whitespace_only_content_before_tools=True, |
| 239 | ) |
| 240 | |
| 241 | |
| 242 | class MistralParser(ParserEngine): |
| 243 | """Mistral parser: engine-based reasoning + ``[TOOL_CALLS]`` tool calls. |
| 244 | |
| 245 | Reasoning encoding is auto-detected from the tokenizer: |
| 246 | |
| 247 | - ``"special_token"`` – ``[THINK]`` present in vocab (v13+). |
| 248 | - ``"text"`` – tokenizer supports grammar but has no ``[THINK]`` (v11). |
| 249 | - ``"none"`` – no grammar support; reasoning disabled. |
| 250 | |
| 251 | Tool calls use the ``[TOOL_CALLS]func_name{...}`` format. The opening |
| 252 | ``{`` doubles as the NAME→ARGS separator and is included in the argument |
| 253 | buffer via an ``ARG_VALUE_CHUNK`` event on the transition (mirrors |
| 254 | kimi_k2, JSON args, `tool_args_json=True`). |
| 255 | |
| 256 | When the tokenizer does not support grammar (``_reasoning_encoding == |
| 257 | "none"``), the legacy ``[TOOL_CALLS]``-based extraction path is used |
| 258 | instead of the declarative engine, handling both pre-v11 JSON-array and |
| 259 | v11+ ``funcname{args}`` formats. |
| 260 | """ |
| 261 | |
| 262 | def __init__( |
| 263 | self, |
| 264 | tokenizer: TokenizerLike, |
| 265 | tools: list[Tool] | None = None, |
| 266 | **kwargs, |
| 267 | ) -> None: |
| 268 | vocab = tokenizer.get_vocab() |
| 269 | self._reasoning_encoding: Literal["special_token", "text", "none"] |
| 270 | if _THINK_START_SPECIAL in vocab: |
| 271 | self._reasoning_encoding = "special_token" |
| 272 | elif getattr(tokenizer, "supports_grammar", False): |
| 273 | self._reasoning_encoding = "text" |
| 274 | else: |
| 275 | self._reasoning_encoding = "none" |
| 276 | |
| 277 | kwargs.setdefault( |
| 278 | "parser_engine_config", |
| 279 | mistral_config(reasoning_encoding=self._reasoning_encoding), |
| 280 | ) |
| 281 | super().__init__(tokenizer, tools, **kwargs) |
| 282 | |
| 283 | self._tool_calls_token_id: int | None = self.vocab.get(_TOOL_CALLS) |
| 284 | |
| 285 | # Tool calls use the legacy parser for all tokenizer versions; |
| 286 | # reasoning is handled by the engine. |
| 287 | self.bot_token: str = _TOOL_CALLS |
| 288 | self.bot_token_id: int | None = self._tool_calls_token_id |
| 289 | if self.bot_token_id is None: |
| 290 | raise RuntimeError( |
| 291 | "Mistral parser could not locate the tool call token in the tokenizer!" |
| 292 | ) |
| 293 | |
| 294 | # Legacy tool-call streaming state. |
| 295 | self.prev_tool_call_arr: list[dict[str, Any]] = [] |
| 296 | self.current_tool_id: int = -1 |
| 297 | self.streaming_state: StreamingState = StreamingState.WAITING_FOR_TOOL_START |
| 298 | self.tool_call_started: bool = False |
| 299 | self.current_tool_name: str | None = None |
| 300 | self.current_tool_mistral_id: str | None = None |
| 301 | self.starting_new_tool: bool = False |
| 302 | self.streamed_args_for_tool: list[str] = [] |
| 303 | self._is_pre_v11: bool = _is_pre_v11_tokeniser(tokenizer) |
| 304 | self.parse_coro = None |
| 305 | if self._is_pre_v11: |
| 306 | self.parse_coro = ijson.parse_coro( |
| 307 | self.update_stream_state_pre_v11_tokenizer() |
| 308 | ) |
| 309 | self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL) |
| 310 | |
| 311 | def adjust_request( |
| 312 | self, request: ChatCompletionRequest | ResponsesRequest |
| 313 | ) -> ChatCompletionRequest | ResponsesRequest: |
| 314 | if not isinstance(request, ResponsesRequest) and request._grammar_from_parser: |
| 315 | return request |
| 316 | so_non_supported_attributes = [ |
| 317 | "regex", |
| 318 | "choice", |
| 319 | "grammar", |
| 320 | # whitespace_pattern is not a constraint type but an option; |
| 321 | # Mistral grammar factory does not support it. |
| 322 | "whitespace_pattern", |
| 323 | "structural_tag", |
| 324 | ] |
| 325 | any_so_non_supported_active = request.structured_outputs is not None and any( |
| 326 | getattr(request.structured_outputs, attribute) is not None |
| 327 | for attribute in so_non_supported_attributes |
| 328 | ) |
| 329 | response_format_non_supported_active = ( |
| 330 | isinstance(request, ResponsesRequest) |
| 331 | or request.response_format is not None |
| 332 | and request.response_format.type == "structural_tag" |
| 333 | ) |
| 334 | |
| 335 | if ( |
| 336 | not is_mistral_tokenizer(self.model_tokenizer) |
| 337 | or isinstance(request, ResponsesRequest) |
| 338 | or not self.model_tokenizer.supports_grammar |
| 339 | or any_so_non_supported_active |
| 340 | or response_format_non_supported_active |
| 341 | ): |
| 342 | request = super().adjust_request(request) |
| 343 | if request.tools and request.tool_choice != "none": |
| 344 | # Keep special tokens so the [TOOL_CALLS] marker |
| 345 | # survives for tool detection. |
| 346 | request.skip_special_tokens = False |
| 347 | # Inject a guided JSON schema for pre-v11 required/named tool choice |
| 348 | # so the model emits a well-formed bare JSON array instead of |
| 349 | # rambling. tool_choice forces a tool call, so a json_object / |
| 350 | # json_schema response_format is cleared here (the tool schema is |
| 351 | # the sole structured-output constraint), mirroring the base |
| 352 | # ToolParser; a structural_tag response_format is left untouched. |
| 353 | req_tool_choice = request.tool_choice |
| 354 | is_required_or_named = req_tool_choice == "required" or isinstance( |
| 355 | req_tool_choice, ChatCompletionNamedToolChoiceParam |
| 356 | ) |
| 357 | response_format = getattr(request, "response_format", None) |
| 358 | response_format_overridable = response_format is None or ( |
| 359 | response_format.type in ("text", "json_object", "json_schema") |
| 360 | ) |
| 361 | # This runs only in the non-grammar (legacy) branch, so it covers |
| 362 | # both pre-v11 Mistral tokenizers and non-Mistral (e.g. HF-mode) |
| 363 | # tokenizers driving the Mistral tool parser. The remaining guards |
| 364 | # keep it off when the user supplied their own structured output. |
| 365 | if ( |
| 366 | not isinstance(request, ResponsesRequest) |
| 367 | and request.tools |
| 368 | and request.structured_outputs is None |
| 369 | and is_required_or_named |
| 370 | and response_format_overridable |
| 371 | ): |
| 372 | schema = self._build_guided_schema_pre_v11(request) |
| 373 | if schema is not None: |
| 374 | request.structured_outputs = StructuredOutputsParams(json=schema) |
| 375 | request.response_format = None |
| 376 | return request |
| 377 | |
| 378 | json_schema: dict[str, Any] | None = None |
| 379 | if request.structured_outputs is not None: |
| 380 | if request.structured_outputs.json_object is not None: |
| 381 | json_schema = _DEFAULT_JSON_SCHEMA |
| 382 | elif request.structured_outputs.json is not None: |
| 383 | if isinstance(request.structured_outputs.json, str): |
| 384 | json_schema = json.loads(request.structured_outputs.json) |
| 385 | else: |
| 386 | json_schema = request.structured_outputs.json |
| 387 | else: |
| 388 | raise ValueError( |
| 389 | "Unsupported request.structured_outputs for MistralParser. " |
| 390 | "Only `json` and `json_object` are supported." |
| 391 | ) |
| 392 | elif ( |
| 393 | request.response_format is not None |
| 394 | and request.response_format.type != "text" |
| 395 | ): |
| 396 | if request.response_format.type == "json_object": |
| 397 | json_schema = _DEFAULT_JSON_SCHEMA |
| 398 | elif request.response_format.type == "json_schema": |
| 399 | if request.response_format.json_schema is not None: |
| 400 | json_schema = request.response_format.json_schema.json_schema |
| 401 | else: |
| 402 | json_schema = _DEFAULT_JSON_SCHEMA |
| 403 | else: |
| 404 | raise ValueError( |
| 405 | "MistralParser only accepts `text`, `json_object` or " |
| 406 | f"`json_schema`, got {request.response_format=}" |
| 407 | ) |
| 408 | request.response_format = None |
| 409 | |
| 410 | grammar_factory = self.model_tokenizer.grammar_factory |
| 411 | |
| 412 | # Rendering grammar is cached in mistral-common given tools, template and mode. |
| 413 | template = grammar_factory.select_jinja_template() |
| 414 | |
| 415 | mistral_tools = ( |
| 416 | [MistralTool.from_openai(tool.model_dump()) for tool in request.tools] |
| 417 | if request.tools is not None |
| 418 | else None |
| 419 | ) |
| 420 | |
| 421 | tool_choice: MistralToolChoice |
| 422 | match request.tool_choice: |
| 423 | case "none" | "auto" | "required": |
| 424 | tool_choice = MistralToolChoiceEnum(request.tool_choice) |
| 425 | case None: |
| 426 | tool_choice = MistralToolChoiceEnum.auto |
| 427 | # _ == Named tool choice |
| 428 | case _: |
| 429 | tool_choice = MistralNamedToolChoice.model_validate( |
| 430 | { |
| 431 | "type": "function", |
| 432 | "function": {"name": request.tool_choice.function.name}, |
| 433 | } |
| 434 | ) |
| 435 | |
| 436 | match tool_choice, json_schema is not None: |
| 437 | case MistralToolChoiceEnum.none, True: |
| 438 | lark_grammar = grammar_factory.get_lark_for_json_schema( |
| 439 | template=template, json_schema=json_schema |
| 440 | ) |
| 441 | case _, _: |
| 442 | lark_grammar = grammar_factory.get_lark_from_jinja( |
| 443 | template=template, |
| 444 | mode=tool_choice, |
| 445 | tools=mistral_tools, |
| 446 | json_schema=json_schema, |
| 447 | parallel_tool_calls=request.parallel_tool_calls, |
| 448 | json_only=False, |
| 449 | ) |
| 450 | |
| 451 | request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar) |
| 452 | request._grammar_from_parser = True |
| 453 | return request |
| 454 | |
| 455 | def _build_guided_schema_pre_v11( |
| 456 | self, |
| 457 | request: ChatCompletionRequest, |
| 458 | ) -> dict[str, Any] | None: |
| 459 | """Build a guided JSON schema for pre-v11 required/named tool choice. |
| 460 | |
| 461 | The schema enforces the Mistral-native array format |
| 462 | ``[{"name": ..., "arguments": {...}}]`` so the model emits a parseable |
| 463 | bare JSON array instead of free-form text. |
| 464 | |
| 465 | Args: |
| 466 | request: The chat completion request carrying `tools` and |
| 467 | `tool_choice`. |
| 468 | |
| 469 | Returns: |
| 470 | A JSON Schema dict, or ``None`` if the named tool is not found. |
| 471 | |
| 472 | """ |
| 473 | tool_choice = request.tool_choice |
| 474 | tools = request.tools or [] |
| 475 | extra: dict[str, Any] = {} |
| 476 | |
| 477 | if tool_choice == "required": |
| 478 | applicable_tools = tools |
| 479 | else: |
| 480 | # Named tool choice — restrict to the single requested tool. |
| 481 | assert isinstance(tool_choice, ChatCompletionNamedToolChoiceParam) |
| 482 | chosen_name = tool_choice.function.name |
| 483 | applicable_tools = [t for t in tools if t.function.name == chosen_name] |
| 484 | if not applicable_tools: |
| 485 | logger.warning( |
| 486 | "Named tool %r not found in tools list; " |
| 487 | "skipping guided schema injection.", |
| 488 | chosen_name, |
| 489 | ) |
| 490 | return None |
| 491 | extra["maxItems"] = 1 |
| 492 | |
| 493 | any_of = [ |
| 494 | { |
| 495 | "type": "object", |
| 496 | "properties": { |
| 497 | "name": {"type": "string", "enum": [tool.function.name]}, |
| 498 | "arguments": tool.function.parameters or {"type": "object"}, |
| 499 | }, |
| 500 | "required": ["name", "arguments"], |
| 501 | } |
| 502 | for tool in applicable_tools |
| 503 | ] |
| 504 | |
| 505 | return { |
| 506 | "type": "array", |
| 507 | "minItems": 1, |
| 508 | **extra, |
| 509 | "items": { |
| 510 | "type": "object", |
| 511 | "anyOf": any_of, |
| 512 | }, |
| 513 | } |
| 514 | |
| 515 | def _ensure_tool_id(self, slot: ToolCallSlot, name: str) -> None: |
| 516 | """Assign a Mistral-compatible 9-char alphanumeric id to `slot`.""" |
| 517 | if not slot.id: |
| 518 | slot.id = MistralToolCall.generate_random_id() |
| 519 | |
| 520 | def extract_tool_calls_from_content( |
| 521 | self, |
| 522 | content: str, |
| 523 | request: ChatCompletionRequest, |
| 524 | ) -> ExtractedToolCallInformation: |
| 525 | if self._is_pre_v11: |
| 526 | return self._legacy_extract_tool_calls(content, request) |
| 527 | return super().extract_tool_calls_from_content(content, request) |
| 528 | |
| 529 | def _legacy_extract_tool_calls( |
| 530 | self, |
| 531 | model_output: str, |
| 532 | request: ChatCompletionRequest | None, |
| 533 | ) -> ExtractedToolCallInformation: |
| 534 | """Pre-v11 non-streaming extraction. |
| 535 | |
| 536 | Handles ``[TOOL_CALLS][{...}]`` and guided bare-array formats. |
| 537 | """ |
| 538 | if request is None: |
| 539 | tool_choice = None |
| 540 | tools = None |
| 541 | else: |
| 542 | tool_choice = request.tool_choice |
| 543 | tools = request.tools |
| 544 | |
| 545 | # tool_choice="none" with tools: never produce tool calls. |
| 546 | if tool_choice == "none" and tools: |
| 547 | return ExtractedToolCallInformation( |
| 548 | tools_called=False, tool_calls=[], content=model_output |
| 549 | ) |
| 550 | |
| 551 | content: str | None = None |
| 552 | if self.bot_token in model_output: |
| 553 | content_and_raw_tool_calls = model_output.split(self.bot_token) |
| 554 | content = content_and_raw_tool_calls[0] |
| 555 | raw_tool_calls = content_and_raw_tool_calls[1:] |
| 556 | # pre-v11: content[BOT] [{tool_call1},{tool_call2}] |
| 557 | if len(raw_tool_calls) != 1: |
| 558 | raise ValueError( |
| 559 | "Only one BOT token should have been outputted, " |
| 560 | f"but got {model_output}." |
| 561 | ) |
| 562 | stringified_tool_calls = raw_tool_calls[0].strip() |
| 563 | elif tool_choice == "required" or isinstance( |
| 564 | tool_choice, ChatCompletionNamedToolChoiceParam |
| 565 | ): |
| 566 | # Guided bare-array output (no [TOOL_CALLS] marker). |
| 567 | stringified_tool_calls = model_output.strip() |
| 568 | else: |
| 569 | return ExtractedToolCallInformation( |
| 570 | tools_called=False, tool_calls=[], content=model_output |
| 571 | ) |
| 572 | |
| 573 | try: |
| 574 | # Use raw_decode to parse the first valid JSON value, |
| 575 | # ignoring trailing tokens the model may emit after |
| 576 | # the tool call array. |
| 577 | tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls) |
| 578 | except json.JSONDecodeError: |
| 579 | try: |
| 580 | raw_tool_call = self.tool_call_regex.findall(stringified_tool_calls)[0] |
| 581 | tool_calls = json.loads(raw_tool_call) |
| 582 | tool_calls = [ |
| 583 | { |
| 584 | "name": tool_call["name"], |
| 585 | "arguments": json.dumps( |
| 586 | tool_call.get("arguments", {}), |
| 587 | ensure_ascii=False, |
| 588 | ), |
| 589 | } |
| 590 | for tool_call in tool_calls |
| 591 | ] |
| 592 | except (IndexError, json.JSONDecodeError): |
| 593 | logger.exception("Error in extracting tool call from response.") |
| 594 | return ExtractedToolCallInformation( |
| 595 | tools_called=False, |
| 596 | tool_calls=[], |
| 597 | content=stringified_tool_calls, |
| 598 | ) |
| 599 | else: |
| 600 | tool_calls = [ |
| 601 | { |
| 602 | "name": tool_call["name"], |
| 603 | "arguments": json.dumps( |
| 604 | tool_call.get("arguments", {}), |
| 605 | ensure_ascii=False, |
| 606 | ), |
| 607 | } |
| 608 | for tool_call in tool_calls |
| 609 | ] |
| 610 | |
| 611 | mistral_tool_calls: list[MistralToolCall] = [ |
| 612 | MistralToolCall( |
| 613 | type="function", |
| 614 | function=FunctionCall( |
| 615 | name=tool_call["name"], |
| 616 | arguments=tool_call.get("arguments", "{}"), |
| 617 | ), |
| 618 | ) |
| 619 | for tool_call in tool_calls |
| 620 | ] |
| 621 | |
| 622 | return ExtractedToolCallInformation( |
| 623 | tools_called=True, |
| 624 | tool_calls=mistral_tool_calls, |
| 625 | content=content if content and content.strip() else None, |
| 626 | ) |
| 627 | |
| 628 | def extract_tool_calls_streaming( |
| 629 | self, |
| 630 | previous_text: str, |
| 631 | current_text: str, |
| 632 | delta_text: str, |
| 633 | previous_token_ids: Sequence[int], |
| 634 | current_token_ids: Sequence[int], |
| 635 | delta_token_ids: Sequence[int], |
| 636 | request: ChatCompletionRequest | ResponsesRequest, |
| 637 | ) -> DeltaMessage | None: |
| 638 | if not self._is_pre_v11: |
| 639 | return super().extract_tool_calls_streaming( |
| 640 | previous_text, |
| 641 | current_text, |
| 642 | delta_text, |
| 643 | previous_token_ids, |
| 644 | current_token_ids, |
| 645 | delta_token_ids, |
| 646 | request, |
| 647 | ) |
| 648 | # Pre-v11: latch on [TOOL_CALLS] or on the first content of a guided |
| 649 | # required/named request (bare JSON array, no special token). |
| 650 | if self.bot_token_id in delta_token_ids or self.bot_token in delta_text: |
| 651 | self.tool_call_started = True |
| 652 | elif not self.tool_call_started and delta_text: |
| 653 | is_guided = request.tool_choice == "required" or isinstance( |
| 654 | request.tool_choice, ChatCompletionNamedToolChoiceParam |
| 655 | ) |
| 656 | if is_guided: |
| 657 | self.tool_call_started = True |
| 658 | if not self.tool_call_started: |
| 659 | return DeltaMessage(content=delta_text) |
| 660 | try: |
| 661 | return self._extract_tool_calls_streaming_pre_v11_tokenizer( |
| 662 | delta_text=delta_text, |
| 663 | delta_token_ids=delta_token_ids, |
| 664 | ) |
| 665 | except Exception: |
| 666 | logger.exception("Error trying to handle streaming tool call.") |
| 667 | return None |
| 668 | |
| 669 | @ijson.coroutine |
| 670 | def update_stream_state_pre_v11_tokenizer(self): |
| 671 | while True: |
| 672 | (prefix, event, value) = yield |
| 673 | |
| 674 | if prefix == "item" and event == "start_map": |
| 675 | self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY |
| 676 | self.starting_new_tool = True |
| 677 | if prefix == "item" and event == "map_key" and value == "name": |
| 678 | self.streaming_state = StreamingState.PARSING_NAME |
| 679 | if prefix == "item.name" and event == "string": |
| 680 | self.current_tool_name = value |
| 681 | self.streaming_state = StreamingState.PARSING_NAME_COMPLETED |
| 682 | if prefix == "item" and event == "map_key" and value == "arguments": |
| 683 | self.streaming_state = StreamingState.WAITING_FOR_ARGUMENTS_START |
| 684 | if prefix == "item.arguments" and event == "start_map": |
| 685 | self.streaming_state = StreamingState.PARSING_ARGUMENTS |
| 686 | if prefix == "item.arguments" and event == "end_map": |
| 687 | self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED |
| 688 | if prefix == "item" and event == "end_map": |
| 689 | self.streaming_state = StreamingState.TOOL_COMPLETE |
| 690 | if prefix == "" and event == "end_array": |
| 691 | self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE |
| 692 | |
| 693 | def _extract_tool_calls_streaming_pre_v11_tokenizer( |
| 694 | self, |
| 695 | delta_text: str, |
| 696 | delta_token_ids: Sequence[int], |
| 697 | ) -> DeltaMessage | None: |
| 698 | """Extract tool calls for pre-v11 Mistral models. |
| 699 | |
| 700 | Handles ``[TOOL_CALLS][{"name": "add", "arguments":{"a": 3.5}}]``. |
| 701 | """ |
| 702 | assert self.parse_coro is not None |
| 703 | content = None |
| 704 | delta_tool_calls: list[DeltaToolCall] = [] |
| 705 | current_tool_call: DeltaToolCall = DeltaToolCall( |
| 706 | index=self.current_tool_id, type="function" |
| 707 | ) |
| 708 | current_tool_call_modified = False |
| 709 | if self.bot_token_id in delta_token_ids or self.bot_token in delta_text: |
| 710 | # this is the first tool call |
| 711 | if not delta_text.startswith(self.bot_token): |
| 712 | content = delta_text.split(self.bot_token)[0] |
| 713 | delta_text = "".join(delta_text.split(self.bot_token)[1:]) |
| 714 | |
| 715 | # ijson gives no text index per event, so split the delta manually |
| 716 | # to know where each event is emitted from. |
| 717 | while len(delta_text) > 0: |
| 718 | streaming_state_before_parse = self.streaming_state |
| 719 | |
| 720 | if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START: |
| 721 | delta_to_be_parsed, delta_text = self._split_delta( |
| 722 | delta_text=delta_text, |
| 723 | stop_after_opening_curly_braces=1, |
| 724 | ) |
| 725 | elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY: |
| 726 | delta_to_be_parsed, delta_text = self._split_delta( |
| 727 | delta_text=delta_text, |
| 728 | stop_after_colon=1, |
| 729 | stop_after_opening_curly_braces=1, |
| 730 | ) |
| 731 | elif self.streaming_state == StreamingState.PARSING_NAME: |
| 732 | delta_to_be_parsed, delta_text = self._split_delta( |
| 733 | delta_text=delta_text, |
| 734 | stop_after_comma=1, |
| 735 | stop_after_closing_brackets=1, |
| 736 | ) |
| 737 | elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START: |
| 738 | delta_to_be_parsed, delta_text = self._split_delta( |
| 739 | delta_text=delta_text, |
| 740 | stop_after_opening_curly_braces=1, |
| 741 | ) |
| 742 | elif self.streaming_state == StreamingState.PARSING_ARGUMENTS: |
| 743 | delta_to_be_parsed, delta_text = self._split_delta( |
| 744 | delta_text=delta_text, |
| 745 | stop_after_closing_curly_braces=1, |
| 746 | ) |
| 747 | elif self.streaming_state in [ |
| 748 | StreamingState.PARSING_ARGUMENTS_COMPLETED, |
| 749 | StreamingState.PARSING_NAME_COMPLETED, |
| 750 | ]: |
| 751 | delta_to_be_parsed, delta_text = self._split_delta( |
| 752 | delta_text=delta_text, |
| 753 | stop_after_closing_curly_braces=1, |
| 754 | stop_after_closing_brackets=1, |
| 755 | ) |
| 756 | elif self.streaming_state == StreamingState.TOOL_COMPLETE: |
| 757 | delta_to_be_parsed, delta_text = self._split_delta( |
| 758 | delta_text=delta_text, |
| 759 | stop_after_opening_curly_braces=1, |
| 760 | stop_after_closing_brackets=1, |
| 761 | ) |
| 762 | elif self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE: |
| 763 | content = delta_text |
| 764 | delta_text = "" |
| 765 | else: |
| 766 | delta_to_be_parsed = delta_text |
| 767 | delta_text = "" |
| 768 | |
| 769 | if self.streaming_state != StreamingState.ALL_TOOLS_COMPLETE: |
| 770 | self.parse_coro.send(delta_to_be_parsed.encode("utf-8")) |
| 771 | |
| 772 | # start_map is the authoritative new-tool signal and survives |
| 773 | # batched deltas, unlike comparing pre/post streaming states. |
| 774 | if self.starting_new_tool: |
| 775 | self.starting_new_tool = False |
| 776 | if current_tool_call_modified: |
| 777 | if self.current_tool_mistral_id is not None: |
| 778 | current_tool_call.id = self.current_tool_mistral_id |
| 779 | self.current_tool_mistral_id = None |
| 780 | self._track_streamed_args_pre_v11(current_tool_call) |
| 781 | delta_tool_calls.append(current_tool_call) |
| 782 | current_tool_call_modified = False |
| 783 | self.current_tool_id += 1 |
| 784 | self.streamed_args_for_tool.append("") |
| 785 | self.prev_tool_call_arr.append({}) |
| 786 | self.current_tool_mistral_id = MistralToolCall.generate_random_id() |
| 787 | current_tool_call = DeltaToolCall( |
| 788 | index=self.current_tool_id, |
| 789 | type="function", |
| 790 | ) |
| 791 | if current_tool_call.function is None: |
| 792 | current_tool_call.function = DeltaFunctionCall() |
| 793 | |
| 794 | if self.current_tool_name is not None: |
| 795 | current_tool_call_modified = True |
| 796 | current_tool_call.function.name = self.current_tool_name |
| 797 | self.prev_tool_call_arr[self.current_tool_id]["name"] = ( |
| 798 | self.current_tool_name |
| 799 | ) |
| 800 | self.current_tool_name = None |
| 801 | if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED: |
| 802 | self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY |
| 803 | if self.streaming_state in [ |
| 804 | StreamingState.PARSING_ARGUMENTS, |
| 805 | StreamingState.PARSING_ARGUMENTS_COMPLETED, |
| 806 | ]: |
| 807 | if self.streaming_state == StreamingState.PARSING_ARGUMENTS_COMPLETED: |
| 808 | self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY |
| 809 | current_tool_call_modified = True |
| 810 | if current_tool_call.function.arguments is None: |
| 811 | current_tool_call.function.arguments = delta_to_be_parsed |
| 812 | else: |
| 813 | current_tool_call.function.arguments += delta_to_be_parsed |
| 814 | if streaming_state_before_parse != StreamingState.PARSING_ARGUMENTS: |
| 815 | # It's the first chunk of arg. let's lstrip it |
| 816 | current_tool_call.function.arguments = ( |
| 817 | current_tool_call.function.arguments.lstrip() |
| 818 | ) |
| 819 | |
| 820 | if current_tool_call_modified: |
| 821 | if self.current_tool_mistral_id is not None: |
| 822 | current_tool_call.id = self.current_tool_mistral_id |
| 823 | self.current_tool_mistral_id = None |
| 824 | self._track_streamed_args_pre_v11(current_tool_call) |
| 825 | delta_tool_calls.append(current_tool_call) |
| 826 | |
| 827 | if content or len(delta_tool_calls) > 0: |
| 828 | delta_message = DeltaMessage() |
| 829 | if content: |
| 830 | delta_message.content = content |
| 831 | if len(delta_tool_calls) > 0: |
| 832 | delta_message.tool_calls = delta_tool_calls |
| 833 | return delta_message |
| 834 | else: |
| 835 | if self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE: |
| 836 | return DeltaMessage() |
| 837 | else: |
| 838 | return None |
| 839 | |
| 840 | def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None: |
| 841 | r"""Accumulate `tool_call` arguments into the streaming state.""" |
| 842 | if tool_call.function is not None and tool_call.function.arguments is not None: |
| 843 | self.streamed_args_for_tool[self.current_tool_id] += ( |
| 844 | tool_call.function.arguments |
| 845 | ) |
| 846 | self.prev_tool_call_arr[self.current_tool_id]["arguments"] = ( |
| 847 | self.streamed_args_for_tool[self.current_tool_id] |
| 848 | ) |
| 849 | |
| 850 | def _split_delta( |
| 851 | self, |
| 852 | delta_text: str, |
| 853 | stop_after_quotes: int = -1, |
| 854 | stop_after_opening_curly_braces: int = -1, |
| 855 | stop_after_closing_curly_braces: int = -1, |
| 856 | stop_after_closing_brackets: int = -1, |
| 857 | stop_after_colon: int = -1, |
| 858 | stop_after_comma: int = -1, |
| 859 | ) -> tuple[str, str]: |
| 860 | delta_to_be_parsed = "" |
| 861 | for i, c in enumerate(delta_text): |
| 862 | if c in ['"', "'"]: |
| 863 | delta_to_be_parsed += c |
| 864 | stop_after_quotes -= 1 |
| 865 | if stop_after_quotes == 0: |
| 866 | return (delta_to_be_parsed, delta_text[i + 1 :]) |
| 867 | elif c == "{": |
| 868 | delta_to_be_parsed += c |
| 869 | stop_after_opening_curly_braces -= 1 |
| 870 | if stop_after_opening_curly_braces == 0: |
| 871 | return (delta_to_be_parsed, delta_text[i + 1 :]) |
| 872 | elif c == "}": |
| 873 | delta_to_be_parsed += c |
| 874 | stop_after_closing_curly_braces -= 1 |
| 875 | if stop_after_closing_curly_braces == 0: |
| 876 | return (delta_to_be_parsed, delta_text[i + 1 :]) |
| 877 | elif c == "]": |
| 878 | delta_to_be_parsed += c |
| 879 | stop_after_closing_brackets -= 1 |
| 880 | if stop_after_closing_brackets == 0: |
| 881 | return (delta_to_be_parsed, delta_text[i + 1 :]) |
| 882 | elif c == ":": |
| 883 | delta_to_be_parsed += c |
| 884 | stop_after_colon -= 1 |
| 885 | if stop_after_colon == 0: |
| 886 | return (delta_to_be_parsed, delta_text[i + 1 :]) |
| 887 | elif c == ",": |
| 888 | delta_to_be_parsed += c |
| 889 | stop_after_comma -= 1 |
| 890 | if stop_after_comma == 0: |
| 891 | return (delta_to_be_parsed, delta_text[i + 1 :]) |
| 892 | else: |
| 893 | delta_to_be_parsed += c |
| 894 | |
| 895 | return (delta_to_be_parsed, "") |
| 896 | |
| 897 | def extract_reasoning( |
| 898 | self, |
| 899 | model_output: str, |
| 900 | request: ChatCompletionRequest | ResponsesRequest, |
| 901 | ) -> tuple[str | None, str | None]: |
| 902 | if self._reasoning_encoding == "none": |
| 903 | return None, model_output |
| 904 | return super().extract_reasoning(model_output, request) |
| 905 | |
| 906 | def _accept_tool_name(self, name: str) -> bool: |
| 907 | # Once `[ARGS]` or the opening `{` has moved the slot past its name, |
| 908 | # that name is final, empty included. Emitting the call with "" |
| 909 | # surfaces the malformed generation instead of dropping it. |
| 910 | return self._is_valid_tool_name(name) |
| 911 | |
| 912 | def _try_extract_name(self, idx: int) -> str | None: |
| 913 | # No JSON-embedded "name" key in this format; the slot name is final. |
| 914 | return self._tool_slots[idx].name |
| 915 | |
| 916 | def _extract_name_and_args(self, raw_body: str) -> tuple[str, str]: |
| 917 | # Never a {"name": ..., "arguments": ...} envelope -- a literal "name" |
| 918 | # argument key must not be mistaken for the tool name. |
| 919 | return "", self._extract_args_json(raw_body, "") |
| 920 | |
| 921 | def _handle_arg_chunk( |
| 922 | self, |
| 923 | event: SemanticEvent, |
| 924 | deltas: list[DeltaToolCall], |
| 925 | ) -> None: |
| 926 | """Emit the opening ``{`` as an arg delta when the name is first sent. |
| 927 | |
| 928 | When the TOOL_NAME→TOOL_ARGS transition fires, ``{`` arrives as an |
| 929 | ARG_VALUE_CHUNK before ``name_sent`` is True. The parent emits the |
| 930 | name delta but not the ``{`` arg delta. This override re-emits the |
| 931 | current chunk so streaming clients receive a valid JSON prefix. |
| 932 | """ |
| 933 | idx = event.tool_index |
| 934 | name_sent_before = ( |
| 935 | 0 <= idx < len(self._tool_slots) and self._tool_slots[idx].name_sent |
| 936 | ) |
| 937 | super()._handle_arg_chunk(event, deltas) |
| 938 | if ( |
| 939 | event.value |
| 940 | and not name_sent_before |
| 941 | and 0 <= idx < len(self._tool_slots) |
| 942 | and self._tool_slots[idx].name_sent |
| 943 | ): |
| 944 | deltas.append( |
| 945 | DeltaToolCall( |
| 946 | index=idx, |
| 947 | function=DeltaFunctionCall(arguments=event.value), |
| 948 | ) |
| 949 | ) |
| 950 | |
| 951 | def _extract_args_json(self, raw_args: str, func_name: str) -> str: |
| 952 | """Return the first complete JSON value in ``raw_args``. |
| 953 | |
| 954 | v11+ tool calls are emitted as ``name{args}`` with no terminator, so a |
| 955 | model may append ordinary text after the closing brace. Parse the first |
| 956 | JSON value with ``raw_decode`` and drop any trailing output, mirroring |
| 957 | the pre-v11 path (fixes gh#48975). |
| 958 | """ |
| 959 | stripped = raw_args.strip() |
| 960 | if not stripped: |
| 961 | return "{}" |
| 962 | try: |
| 963 | _, end = json.JSONDecoder().raw_decode(stripped) |
| 964 | except json.JSONDecodeError: |
| 965 | return stripped |
| 966 | return stripped[:end] |