| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Qwen3 parser for tool calls and reasoning. |
| 4 | |
| 5 | Qwen3 XML tool call format:: |
| 6 | |
| 7 | <tool_call> |
| 8 | <function=func_name> |
| 9 | <parameter=key>value</parameter> |
| 10 | </function> |
| 11 | </tool_call> |
| 12 | |
| 13 | The argument body consists of ``<parameter=NAME>VALUE</parameter>`` tags. |
| 14 | The ``_qwen3_arg_converter`` parses these into a JSON object. |
| 15 | """ |
| 16 | |
| 17 | from __future__ import annotations |
| 18 | |
| 19 | import functools |
| 20 | import json |
| 21 | from typing import TYPE_CHECKING |
| 22 | |
| 23 | import regex as re |
| 24 | |
| 25 | from vllm.parser.engine.events import EventType |
| 26 | from vllm.parser.engine.parser_engine import ParserEngine |
| 27 | from vllm.parser.engine.parser_engine_config import ( |
| 28 | ParserEngineConfig, |
| 29 | ParserState, |
| 30 | Transition, |
| 31 | ) |
| 32 | |
| 33 | if TYPE_CHECKING: |
| 34 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 35 | ChatCompletionRequest, |
| 36 | ) |
| 37 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 38 | from vllm.tokenizers import TokenizerLike |
| 39 | from vllm.tool_parsers.abstract_tool_parser import Tool |
| 40 | |
| 41 | THINK_START = "<think>" |
| 42 | THINK_END = "</think>" |
| 43 | TOOL_CALL_START = "<tool_call>" |
| 44 | TOOL_CALL_END = "</tool_call>" |
| 45 | CHATML_TURN_BOUNDARIES = frozenset(("<|im_start|>", "<|im_end|>")) |
| 46 | FUNC_PREFIX = "<function=" |
| 47 | FUNC_END = "</function>" |
| 48 | PARAM_START = "<parameter=" |
| 49 | PARAM_END = "</parameter>" |
| 50 | |
| 51 | _PARAM_RE = re.compile( |
| 52 | r"<\s*parameter\s*=\s*([^>]*)>" |
| 53 | r"(.*?)" |
| 54 | r"(?:<\s*/\s*parameter\s*>|(?=<\s*parameter\s*=))", |
| 55 | re.DOTALL, |
| 56 | ) |
| 57 | _PARTIAL_PARAM_RE = re.compile(r"<\s*parameter\s*=\s*([^>]+)>(.*)$", re.DOTALL) |
| 58 | |
| 59 | |
| 60 | def _trim_wrapping_newlines(value: str) -> str: |
| 61 | """Strip one leading and one trailing newline (the Qwen3 template markup).""" |
| 62 | if value.startswith("\n"): |
| 63 | value = value[1:] |
| 64 | if value.endswith("\n"): |
| 65 | value = value[:-1] |
| 66 | return value |
| 67 | |
| 68 | |
| 69 | def _qwen3_arg_converter(raw_args: str, partial: bool) -> str: |
| 70 | params: dict[str, object] = {} |
| 71 | |
| 72 | for match in _PARAM_RE.finditer(raw_args): |
| 73 | name = match.group(1) |
| 74 | value = match.group(2) |
| 75 | params[name] = _trim_wrapping_newlines(value) |
| 76 | |
| 77 | if partial: |
| 78 | remaining = _PARAM_RE.sub("", raw_args) |
| 79 | m = _PARTIAL_PARAM_RE.search(remaining) |
| 80 | if m: |
| 81 | name = m.group(1) |
| 82 | value = m.group(2) |
| 83 | if name: |
| 84 | params[name] = _trim_wrapping_newlines(value) |
| 85 | |
| 86 | return json.dumps(params, ensure_ascii=False) |
| 87 | |
| 88 | |
| 89 | @functools.cache |
| 90 | def qwen3_config( |
| 91 | thinking: bool = True, |
| 92 | *, |
| 93 | name: str = "qwen3", |
| 94 | think_start: str = THINK_START, |
| 95 | think_end: str = THINK_END, |
| 96 | tool_start: str = TOOL_CALL_START, |
| 97 | tool_end: str = TOOL_CALL_END, |
| 98 | turn_boundary_tokens: frozenset[str] = frozenset(), |
| 99 | ) -> ParserEngineConfig: |
| 100 | return ParserEngineConfig( |
| 101 | name=name, |
| 102 | initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, |
| 103 | wait_for_reasoning=thinking, |
| 104 | turn_boundary_tokens=turn_boundary_tokens, |
| 105 | terminals={ |
| 106 | # Reasoning terminals |
| 107 | "THINK_START": think_start, |
| 108 | "THINK_END": think_end, |
| 109 | # Tool call terminals |
| 110 | "TOOL_START": tool_start, |
| 111 | "TOOL_END": tool_end, |
| 112 | "FUNC_PREFIX": FUNC_PREFIX, |
| 113 | "FUNC_END": FUNC_END, |
| 114 | "PARAM_START": PARAM_START, |
| 115 | "PARAM_END": PARAM_END, |
| 116 | "CLOSE_ANGLE": ">", |
| 117 | }, |
| 118 | token_id_terminals={ |
| 119 | "THINK_START": think_start, |
| 120 | "THINK_END": think_end, |
| 121 | "TOOL_START": tool_start, |
| 122 | "TOOL_END": tool_end, |
| 123 | }, |
| 124 | transitions={ |
| 125 | # -- Reasoning transitions -- |
| 126 | (ParserState.REASONING, "THINK_START"): Transition( |
| 127 | ParserState.REASONING, |
| 128 | (), |
| 129 | ), |
| 130 | (ParserState.REASONING, "THINK_END"): Transition( |
| 131 | ParserState.CONTENT, |
| 132 | (EventType.REASONING_END,), |
| 133 | ), |
| 134 | # Absorb duplicate </think> — model may emit it after |
| 135 | # already transitioning to CONTENT; drop it silently. |
| 136 | (ParserState.CONTENT, "THINK_END"): Transition( |
| 137 | ParserState.CONTENT, |
| 138 | (), |
| 139 | ), |
| 140 | # Tool call directly from reasoning (implicit end) |
| 141 | (ParserState.REASONING, "TOOL_START"): Transition( |
| 142 | ParserState.TOOL_PREAMBLE, |
| 143 | (EventType.REASONING_END, EventType.TOOL_CALL_START), |
| 144 | ), |
| 145 | # -- Tool call transitions -- |
| 146 | (ParserState.CONTENT, "TOOL_START"): Transition( |
| 147 | ParserState.TOOL_PREAMBLE, |
| 148 | (EventType.REASONING_END, EventType.TOOL_CALL_START), |
| 149 | ), |
| 150 | # Fallback: <function= without a preceding <tool_call> |
| 151 | (ParserState.CONTENT, "FUNC_PREFIX"): Transition( |
| 152 | ParserState.TOOL_NAME, |
| 153 | (EventType.TOOL_CALL_START,), |
| 154 | ), |
| 155 | (ParserState.TOOL_PREAMBLE, "TOOL_END"): Transition( |
| 156 | ParserState.CONTENT, |
| 157 | (EventType.TOOL_CALL_END,), |
| 158 | ), |
| 159 | (ParserState.TOOL_PREAMBLE, "FUNC_PREFIX"): Transition( |
| 160 | ParserState.TOOL_NAME, |
| 161 | (), |
| 162 | ), |
| 163 | (ParserState.TOOL_NAME, "CLOSE_ANGLE"): Transition( |
| 164 | ParserState.TOOL_ARGS, |
| 165 | (), |
| 166 | ), |
| 167 | # Malformed: </function> while still in TOOL_NAME (no closing >) |
| 168 | (ParserState.TOOL_NAME, "FUNC_END"): Transition( |
| 169 | ParserState.TOOL_BETWEEN, |
| 170 | (EventType.TOOL_CALL_END,), |
| 171 | ), |
| 172 | (ParserState.TOOL_ARGS, "FUNC_END"): Transition( |
| 173 | ParserState.TOOL_BETWEEN, |
| 174 | (EventType.TOOL_CALL_END,), |
| 175 | ), |
| 176 | (ParserState.TOOL_ARGS, "PARAM_START"): Transition( |
| 177 | ParserState.TOOL_ARGS, |
| 178 | (EventType.ARG_VALUE_CHUNK,), |
| 179 | ), |
| 180 | (ParserState.TOOL_ARGS, "PARAM_END"): Transition( |
| 181 | ParserState.TOOL_ARGS, |
| 182 | (EventType.ARG_VALUE_CHUNK,), |
| 183 | ), |
| 184 | (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( |
| 185 | ParserState.CONTENT, |
| 186 | (), |
| 187 | ), |
| 188 | # Consecutive tool call without closing </tool_call> |
| 189 | (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition( |
| 190 | ParserState.TOOL_PREAMBLE, |
| 191 | (EventType.TOOL_CALL_START,), |
| 192 | ), |
| 193 | (ParserState.TOOL_BETWEEN, "FUNC_PREFIX"): Transition( |
| 194 | ParserState.TOOL_NAME, |
| 195 | (EventType.TOOL_CALL_START,), |
| 196 | ), |
| 197 | }, |
| 198 | arg_converter=_qwen3_arg_converter, |
| 199 | stream_arg_deltas=True, |
| 200 | strip_trailing_reasoning_whitespace=False, |
| 201 | tool_args_json=False, |
| 202 | ) |
| 203 | |
| 204 | |
| 205 | class Qwen3Parser(ParserEngine): |
| 206 | """Qwen3 parser: ``<think>``/``</think>`` reasoning + |
| 207 | ``<tool_call>`` XML tool calls in a single engine. |
| 208 | |
| 209 | - ``<tool_call>`` as implicit reasoning end (a grammar transition, so it |
| 210 | also feeds ``is_reasoning_end`` and the structured-output gate) |
| 211 | |
| 212 | Subclasses that share the grammar but differ only in the four wrapper |
| 213 | token strings (reasoning + tool-call) override the class attributes |
| 214 | below; everything else is inherited unchanged. |
| 215 | """ |
| 216 | |
| 217 | CONFIG_NAME = "qwen3" |
| 218 | THINK_START = THINK_START |
| 219 | THINK_END = THINK_END |
| 220 | TOOL_START = TOOL_CALL_START |
| 221 | TOOL_END = TOOL_CALL_END |
| 222 | TURN_BOUNDARIES: frozenset[str] = CHATML_TURN_BOUNDARIES |
| 223 | |
| 224 | def __init__( |
| 225 | self, |
| 226 | tokenizer: TokenizerLike, |
| 227 | tools: list[Tool] | None = None, |
| 228 | **kwargs, |
| 229 | ) -> None: |
| 230 | chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} |
| 231 | self.thinking_enabled = chat_kwargs.get("enable_thinking", True) |
| 232 | kwargs.setdefault( |
| 233 | "parser_engine_config", |
| 234 | qwen3_config( |
| 235 | thinking=self.thinking_enabled, |
| 236 | name=self.CONFIG_NAME, |
| 237 | think_start=self.THINK_START, |
| 238 | think_end=self.THINK_END, |
| 239 | tool_start=self.TOOL_START, |
| 240 | tool_end=self.TOOL_END, |
| 241 | turn_boundary_tokens=self.TURN_BOUNDARIES, |
| 242 | ), |
| 243 | ) |
| 244 | super().__init__( |
| 245 | tokenizer, |
| 246 | tools, |
| 247 | **kwargs, |
| 248 | ) |
| 249 | |
| 250 | def extract_reasoning( |
| 251 | self, |
| 252 | model_output: str, |
| 253 | request: ChatCompletionRequest | ResponsesRequest, |
| 254 | ) -> tuple[str | None, str | None]: |
| 255 | if not self.thinking_enabled: |
| 256 | return None, model_output |
| 257 | return super().extract_reasoning(model_output, request) |