| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """DeepSeek V4 parser: ``<think>``/``</think>`` |
| 4 | reasoning plus DSML tool calls in a single state machine. |
| 5 | |
| 6 | DeepSeek V4 output format:: |
| 7 | |
| 8 | <think> |
| 9 | ...reasoning... |
| 10 | </think> |
| 11 | <｜DSML｜tool_calls> |
| 12 | <｜DSML｜invoke name="func_name"> |
| 13 | <｜DSML｜parameter name="location" string="true">杭州</｜DSML｜parameter> |
| 14 | <｜DSML｜parameter name="count" string="false">5</｜DSML｜parameter> |
| 15 | </｜DSML｜invoke> |
| 16 | </｜DSML｜tool_calls> |
| 17 | """ |
| 18 | |
| 19 | from __future__ import annotations |
| 20 | |
| 21 | import contextlib |
| 22 | import functools |
| 23 | import json |
| 24 | from typing import TYPE_CHECKING |
| 25 | |
| 26 | import regex as re |
| 27 | |
| 28 | from vllm.parser.engine.events import EventType |
| 29 | from vllm.parser.engine.parser_engine import ParserEngine |
| 30 | from vllm.parser.engine.parser_engine_config import ( |
| 31 | ParserEngineConfig, |
| 32 | ParserState, |
| 33 | Transition, |
| 34 | ) |
| 35 | from vllm.tool_parsers.utils import find_tool_properties |
| 36 | |
| 37 | if TYPE_CHECKING: |
| 38 | from vllm.tokenizers import TokenizerLike |
| 39 | from vllm.tool_parsers.abstract_tool_parser import Tool |
| 40 | |
| 41 | _DSML = "｜DSML｜" |
| 42 | |
| 43 | DSML_THINK_START = "<think>" |
| 44 | DSML_THINK_END = "</think>" |
| 45 | DSML_TOOL_START = f"<{_DSML}tool_calls>" |
| 46 | DSML_TOOL_END = f"</{_DSML}tool_calls>" |
| 47 | DSML_INVOKE_PREFIX = f'<{_DSML}invoke name="' |
| 48 | DSML_INVOKE_NAME_END = '">' |
| 49 | DSML_INVOKE_END = f"</{_DSML}invoke>" |
| 50 | DSML_PARAM_START = f"<{_DSML}parameter" |
| 51 | DSML_PARAM_CLOSE = f"</{_DSML}parameter>" |
| 52 | |
| 53 | # Spellings variants of ``DSML_TOOL_START`` observed in production. |
| 54 | DSML_TOOL_START_VARIANTS: tuple[str, ...] = ( |
| 55 | f"<{_DSML}toolcalls>", |
| 56 | f"<{_DSML}tool>", |
| 57 | ) |
| 58 | |
| 59 | |
| 60 | def _param_patterns( |
| 61 | param_start: str, param_close: str |
| 62 | ) -> tuple[re.Pattern, re.Pattern]: |
| 63 | """Complete and trailing-partial parameter regexes for one DSML dialect. |
| 64 | |
| 65 | The ``string`` attribute is optional: the model sometimes omits it, and |
| 66 | dropping such a parameter hands the client a tool call with no arguments. |
| 67 | """ |
| 68 | start, close = re.escape(param_start), re.escape(param_close) |
| 69 | head = rf'{start}\s+name="([^"]+)"(?:\s+string="([^"]*)")?\s*>' |
| 70 | return ( |
| 71 | re.compile(rf"{head}(.*?)(?:{close}|(?={start}\s+name=))", re.DOTALL), |
| 72 | re.compile(rf"{head}(.*)$", re.DOTALL), |
| 73 | ) |
| 74 | |
| 75 | |
| 76 | _PARAM_RE, _PARTIAL_PARAM_RE = _param_patterns(DSML_PARAM_START, DSML_PARAM_CLOSE) |
| 77 | |
| 78 | |
| 79 | def _dsml_arg_converter( |
| 80 | raw_args: str, |
| 81 | partial: bool, |
| 82 | *, |
| 83 | param_re: re.Pattern = _PARAM_RE, |
| 84 | partial_param_re: re.Pattern = _PARTIAL_PARAM_RE, |
| 85 | ) -> str: |
| 86 | params: dict[str, object] = {} |
| 87 | |
| 88 | last_end = 0 |
| 89 | # A missing ``string`` attribute is treated like ``string="false"``: try |
| 90 | # JSON, fall back to the literal text. |
| 91 | for m in param_re.finditer(raw_args): |
| 92 | name, is_str, value = m.group(1), m.group(2), m.group(3) |
| 93 | if is_str == "true": |
| 94 | params[name] = value |
| 95 | else: |
| 96 | try: |
| 97 | params[name] = json.loads(value) |
| 98 | except (json.JSONDecodeError, ValueError): |
| 99 | params[name] = value |
| 100 | last_end = m.end() |
| 101 | |
| 102 | if partial: |
| 103 | pm = partial_param_re.search(raw_args, last_end) |
| 104 | if pm: |
| 105 | name, is_str, value = pm.group(1), pm.group(2), pm.group(3) |
| 106 | if is_str == "true": |
| 107 | params[name] = value |
| 108 | else: |
| 109 | with contextlib.suppress(json.JSONDecodeError, ValueError): |
| 110 | params[name] = json.loads(value) |
| 111 | |
| 112 | return json.dumps(params, ensure_ascii=False) |
| 113 | |
| 114 | |
| 115 | def _unwrap_wrapper_args( |
| 116 | args_json: str, |
| 117 | tools: list[Tool] | None, |
| 118 | func_name: str | None, |
| 119 | ) -> str: |
| 120 | if not tools or not func_name: |
| 121 | return args_json |
| 122 | try: |
| 123 | args = json.loads(args_json) |
| 124 | except (json.JSONDecodeError, ValueError): |
| 125 | return args_json |
| 126 | if not isinstance(args, dict): |
| 127 | return args_json |
| 128 | properties = find_tool_properties(tools, func_name) |
| 129 | if not properties: |
| 130 | return args_json |
| 131 | allowed = set(properties.keys()) |
| 132 | for wrapper in ("arguments", "input"): |
| 133 | if set(args.keys()) != {wrapper} or wrapper in allowed: |
| 134 | continue |
| 135 | inner = args[wrapper] |
| 136 | if isinstance(inner, str): |
| 137 | try: |
| 138 | inner = json.loads(inner) |
| 139 | except json.JSONDecodeError: |
| 140 | return args_json |
| 141 | if isinstance(inner, dict) and set(inner.keys()).issubset(allowed): |
| 142 | return json.dumps(inner, ensure_ascii=False) |
| 143 | return args_json |
| 144 | |
| 145 | |
| 146 | @functools.cache |
| 147 | def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig: |
| 148 | return ParserEngineConfig( |
| 149 | name="deepseek_v4", |
| 150 | initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, |
| 151 | wait_for_reasoning=thinking, |
| 152 | terminals={ |
| 153 | "THINK_START": DSML_THINK_START, |
| 154 | "THINK_END": DSML_THINK_END, |
| 155 | "TOOL_START": (DSML_TOOL_START, *DSML_TOOL_START_VARIANTS), |
| 156 | "TOOL_END": DSML_TOOL_END, |
| 157 | "INVOKE_PREFIX": DSML_INVOKE_PREFIX, |
| 158 | "INVOKE_NAME_END": DSML_INVOKE_NAME_END, |
| 159 | "INVOKE_END": DSML_INVOKE_END, |
| 160 | "PARAM_START": DSML_PARAM_START, |
| 161 | "PARAM_CLOSE": DSML_PARAM_CLOSE, |
| 162 | }, |
| 163 | token_id_terminals={ |
| 164 | "THINK_START": DSML_THINK_START, |
| 165 | "THINK_END": DSML_THINK_END, |
| 166 | "TOOL_START": DSML_TOOL_START, |
| 167 | "TOOL_END": DSML_TOOL_END, |
| 168 | }, |
| 169 | transitions={ |
| 170 | (ParserState.CONTENT, "THINK_START"): Transition( |
| 171 | ParserState.REASONING, |
| 172 | (EventType.REASONING_START,), |
| 173 | ), |
| 174 | # Absorb a bare </think> with no prior <think> |
| 175 | (ParserState.CONTENT, "THINK_END"): Transition( |
| 176 | ParserState.CONTENT, |
| 177 | (), |
| 178 | ), |
| 179 | # Absorb a duplicate <think> while already reasoning |
| 180 | (ParserState.REASONING, "THINK_START"): Transition( |
| 181 | ParserState.REASONING, |
| 182 | (), |
| 183 | ), |
| 184 | (ParserState.REASONING, "THINK_END"): Transition( |
| 185 | ParserState.CONTENT, |
| 186 | (EventType.REASONING_END,), |
| 187 | ), |
| 188 | # Tool call beginning while still inside <think> |
| 189 | (ParserState.REASONING, "TOOL_START"): Transition( |
| 190 | ParserState.TOOL_PREAMBLE, |
| 191 | (EventType.REASONING_END,), |
| 192 | ), |
| 193 | (ParserState.CONTENT, "TOOL_START"): Transition( |
| 194 | ParserState.TOOL_PREAMBLE, |
| 195 | (), |
| 196 | ), |
| 197 | (ParserState.TOOL_PREAMBLE, "INVOKE_PREFIX"): Transition( |
| 198 | ParserState.TOOL_NAME, |
| 199 | (EventType.TOOL_CALL_START,), |
| 200 | ), |
| 201 | (ParserState.CONTENT, "INVOKE_PREFIX"): Transition( |
| 202 | ParserState.TOOL_NAME, |
| 203 | (EventType.TOOL_CALL_START,), |
| 204 | ), |
| 205 | (ParserState.TOOL_NAME, "INVOKE_NAME_END"): Transition( |
| 206 | ParserState.TOOL_ARGS, |
| 207 | (), |
| 208 | ), |
| 209 | (ParserState.TOOL_ARGS, "INVOKE_END"): Transition( |
| 210 | ParserState.TOOL_BETWEEN, |
| 211 | (EventType.TOOL_CALL_END,), |
| 212 | ), |
| 213 | (ParserState.TOOL_ARGS, "TOOL_END"): Transition( |
| 214 | ParserState.TOOL_BETWEEN, |
| 215 | (EventType.TOOL_CALL_END,), |
| 216 | ), |
| 217 | # Parallel tool calls |
| 218 | (ParserState.TOOL_BETWEEN, "INVOKE_PREFIX"): Transition( |
| 219 | ParserState.TOOL_NAME, |
| 220 | (EventType.TOOL_CALL_START,), |
| 221 | ), |
| 222 | # A tool call ends the turn: stay in TOOL_BETWEEN so any text |
| 223 | # after the block is dropped. |
| 224 | (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( |
| 225 | ParserState.TOOL_BETWEEN, |
| 226 | (), |
| 227 | ), |
| 228 | (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition( |
| 229 | ParserState.TOOL_PREAMBLE, |
| 230 | (), |
| 231 | ), |
| 232 | }, |
| 233 | content_events={ |
| 234 | ParserState.CONTENT: EventType.TEXT_CHUNK, |
| 235 | ParserState.REASONING: EventType.REASONING_CHUNK, |
| 236 | ParserState.TOOL_NAME: EventType.TOOL_NAME, |
| 237 | ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, |
| 238 | }, |
| 239 | arg_converter=_dsml_arg_converter, |
| 240 | arg_structural_chars=frozenset(">"), |
| 241 | strip_content_whitespace_with_tools=False, |
| 242 | tool_args_json=False, |
| 243 | ) |
| 244 | |
| 245 | |
| 246 | class DeepSeekV4Parser(ParserEngine): |
| 247 | parser_config = staticmethod(deepseek_v4_config) |
| 248 | |
| 249 | def __init__( |
| 250 | self, |
| 251 | tokenizer: TokenizerLike, |
| 252 | tools: list[Tool] | None = None, |
| 253 | **kwargs, |
| 254 | ) -> None: |
| 255 | chat_kwargs = kwargs.pop("chat_template_kwargs", None) or {} |
| 256 | thinking = bool( |
| 257 | chat_kwargs.get("thinking") or chat_kwargs.get("enable_thinking") |
| 258 | ) |
| 259 | if "thinking" not in chat_kwargs and "enable_thinking" not in chat_kwargs: |
| 260 | thinking = True |
| 261 | thinking = thinking and chat_kwargs.get("reasoning_effort") != "none" |
| 262 | super().__init__( |
| 263 | tokenizer, |
| 264 | tools, |
| 265 | parser_engine_config=self.parser_config(thinking=thinking), |
| 266 | **kwargs, |
| 267 | ) |
| 268 | self._arg_converter = self._convert_args |
| 269 | |
| 270 | def _convert_args(self, raw_args: str, partial: bool) -> str: |
| 271 | converter = self.parser_engine_config.arg_converter |
| 272 | assert converter is not None |
| 273 | result = converter(raw_args, partial) |
| 274 | if not self._tools: |
| 275 | return result |
| 276 | func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None) |
| 277 | return _unwrap_wrapper_args(result, self._tools, func_name) |