@tduncan / Nvidia-Optimization-Stack-VLLM / commits / fb5291b35b

[Frontend] [Parser] Port DeepSeek V4 to streaming parser engine framework (#45877)

Signed-off-by: Ben Browning <bbrownin@redhat.com>

Ben Browning committed Jul 3, 2026 at 20:55 UTC fb5291b35b0b1b1ec5840438da0f62f127417710
22 files changed +1900 -671
tests/parser/engine/test_deepseek_v32.py new
+268
@@ -0,0 +1,268 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for DeepSeek V3.2 parser engine semantics.
4 +
5 +V3.2 uses the same DSML parameter format as V4 but wraps tool calls in
6 +``<｜DSML｜function_calls>`` instead of ``<｜DSML｜tool_calls>`` and has
7 +no reasoning (``<think>``/``</think>``) support.
8 +"""
9 +
10 +import json
11 +
12 +import pytest
13 +
14 +from tests.parser.engine.conftest import make_mock_tokenizer
15 +from tests.parser.engine.streaming_helpers import (
16 + collect_content,
17 + collect_function_name,
18 + collect_tool_arguments,
19 + simulate_tool_streaming,
20 +)
21 +from vllm.parser.deepseek_v4 import (
22 + DSML_INVOKE_END,
23 + DSML_INVOKE_NAME_END,
24 + DSML_INVOKE_PREFIX,
25 +)
26 +from vllm.parser.deepseek_v32 import (
27 + DSML_FUNC_END,
28 + DSML_FUNC_START,
29 + DeepSeekV32Parser,
30 +)
31 +from vllm.parser.engine.parser_engine_config import ParserState
32 +
33 +_PARAM_OPEN = '｜DSML｜parameter name="{name}" string="{is_str}">'
34 +_PARAM_CLOSE = "</｜DSML｜parameter>"
35 +
36 +
37 +def _param(name: str, is_str: str, value: str) -> str:
38 + return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}"
39 +
40 +
41 +def _invoke(name: str, *params: str) -> str:
42 + body = "\n".join(params)
43 + return (
44 + f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}"
45 + )
46 +
47 +
48 +def _func_calls(*invocations: str) -> str:
49 + body = "\n".join(invocations)
50 + return f"{DSML_FUNC_START}\n{body}\n{DSML_FUNC_END}"
51 +
52 +
53 +def _make_tool(name, properties):
54 + from vllm.entrypoints.openai.chat_completion.protocol import (
55 + ChatCompletionToolsParam,
56 + )
57 +
58 + return ChatCompletionToolsParam(
59 + type="function",
60 + function={
61 + "name": name,
62 + "parameters": {
63 + "type": "object",
64 + "properties": properties,
65 + },
66 + },
67 + )
68 +
69 +
70 +@pytest.fixture
71 +def mock_tokenizer():
72 + return make_mock_tokenizer({})
73 +
74 +
75 +@pytest.fixture
76 +def mock_request():
77 + from unittest.mock import MagicMock
78 +
79 + from vllm.entrypoints.openai.chat_completion.protocol import (
80 + ChatCompletionRequest,
81 + )
82 +
83 + req = MagicMock(spec=ChatCompletionRequest)
84 + req.tools = []
85 + req.tool_choice = "auto"
86 + return req
87 +
88 +
89 +# ── Non-streaming extraction ────────────────────────────────────────
90 +
91 +
92 +class TestNonStreaming:
93 + def test_no_tool_call(self, mock_tokenizer, mock_request):
94 + parser = DeepSeekV32Parser(mock_tokenizer)
95 + result = parser.extract_tool_calls("Hello world", mock_request)
96 + assert not result.tools_called
97 + assert result.content == "Hello world"
98 +
99 + def test_single_tool(self, mock_tokenizer, mock_request):
100 + text = _func_calls(
101 + _invoke("get_weather", _param("city", "true", "SF")),
102 + )
103 + parser = DeepSeekV32Parser(mock_tokenizer)
104 + result = parser.extract_tool_calls(text, mock_request)
105 + assert result.tools_called
106 + assert len(result.tool_calls) == 1
107 + assert result.tool_calls[0].function.name == "get_weather"
108 + args = json.loads(result.tool_calls[0].function.arguments)
109 + assert args == {"city": "SF"}
110 +
111 + def test_parallel_tools(self, mock_tokenizer, mock_request):
112 + text = _func_calls(
113 + _invoke("get_weather", _param("city", "true", "SF")),
114 + _invoke("get_weather", _param("city", "true", "NYC")),
115 + )
116 + parser = DeepSeekV32Parser(mock_tokenizer)
117 + result = parser.extract_tool_calls(text, mock_request)
118 + assert result.tools_called
119 + assert len(result.tool_calls) == 2
120 + assert json.loads(result.tool_calls[0].function.arguments) == {"city": "SF"}
121 + assert json.loads(result.tool_calls[1].function.arguments) == {"city": "NYC"}
122 +
123 + def test_content_before_tool_call(self, mock_tokenizer, mock_request):
124 + text = "Let me check. " + _func_calls(
125 + _invoke("search", _param("q", "true", "vllm")),
126 + )
127 + parser = DeepSeekV32Parser(mock_tokenizer)
128 + result = parser.extract_tool_calls(text, mock_request)
129 + assert result.tools_called
130 + assert result.content is not None
131 + assert "Let me check" in result.content
132 +
133 + def test_non_string_params_json_parsed(self, mock_tokenizer, mock_request):
134 + text = _func_calls(
135 + _invoke(
136 + "toggle",
137 + _param("enabled", "false", "true"),
138 + _param("count", "false", "42"),
139 + ),
140 + )
141 + parser = DeepSeekV32Parser(mock_tokenizer)
142 + result = parser.extract_tool_calls(text, mock_request)
143 + args = json.loads(result.tool_calls[0].function.arguments)
144 + assert args["enabled"] is True
145 + assert args["count"] == 42
146 +
147 + def test_wrapper_unwrapping(self, mock_tokenizer, mock_request):
148 + tool = _make_tool("get_weather", {"location": {"type": "string"}})
149 + mock_request.tools = [tool]
150 + text = _func_calls(
151 + _invoke(
152 + "get_weather",
153 + _param("arguments", "false", '{"location":"Beijing"}'),
154 + ),
155 + )
156 + parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
157 + result = parser.extract_tool_calls(text, mock_request)
158 + args = json.loads(result.tool_calls[0].function.arguments)
159 + assert args == {"location": "Beijing"}
160 +
161 +
162 +# ── Initial state ────────────────────────────────────────────────────
163 +
164 +
165 +class TestInitialState:
166 + def test_always_content(self, mock_tokenizer):
167 + parser = DeepSeekV32Parser(mock_tokenizer)
168 + cfg = parser.parser_engine_config
169 + assert cfg.initial_state == ParserState.CONTENT
170 +
171 + def test_ignores_thinking_kwargs(self, mock_tokenizer):
172 + parser = DeepSeekV32Parser(
173 + mock_tokenizer,
174 + chat_template_kwargs={"thinking": True, "enable_thinking": True},
175 + )
176 + cfg = parser.parser_engine_config
177 + assert cfg.initial_state == ParserState.CONTENT
178 +
179 +
180 +# ── Streaming ────────────────────────────────────────────────────────
181 +
182 +
183 +class TestStreaming:
184 + def test_single_tool_streaming(self, mock_tokenizer, mock_request):
185 + text = _func_calls(
186 + _invoke("get_weather", _param("city", "true", "SF")),
187 + )
188 + parser = DeepSeekV32Parser(mock_tokenizer)
189 + results = simulate_tool_streaming(parser, mock_request, list(text))
190 + assert collect_function_name(results) == "get_weather"
191 + args_json = collect_tool_arguments(results)
192 + assert json.loads(args_json) == {"city": "SF"}
193 +
194 + def test_content_before_tool_streaming(self, mock_tokenizer, mock_request):
195 + text = "Checking... " + _func_calls(
196 + _invoke("fn", _param("k", "true", "v")),
197 + )
198 + parser = DeepSeekV32Parser(mock_tokenizer)
199 + results = simulate_tool_streaming(parser, mock_request, list(text))
200 + content = collect_content(results)
201 + assert "Checking" in content
202 +
203 + def test_parallel_tools_streaming(self, mock_tokenizer, mock_request):
204 + text = _func_calls(
205 + _invoke("fn_a", _param("x", "true", "1")),
206 + _invoke("fn_b", _param("y", "true", "2")),
207 + )
208 + parser = DeepSeekV32Parser(mock_tokenizer)
209 + results = simulate_tool_streaming(parser, mock_request, list(text))
210 +
211 + names = []
212 + for delta, _ in results:
213 + if delta and delta.tool_calls:
214 + for tc in delta.tool_calls:
215 + if tc.function and tc.function.name:
216 + names.append(tc.function.name)
217 + assert "fn_a" in names
218 + assert "fn_b" in names
219 +
220 + def test_no_tool_content_only(self, mock_tokenizer, mock_request):
221 + text = "Just some text, no tools."
222 + parser = DeepSeekV32Parser(mock_tokenizer)
223 + results = simulate_tool_streaming(parser, mock_request, list(text))
224 + content = collect_content(results)
225 + assert "Just some text" in content
226 + args = collect_tool_arguments(results)
227 + assert args == ""
228 +
229 + def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
230 + tool = _make_tool("get_weather", {"location": {"type": "string"}})
231 + mock_request.tools = [tool]
232 + parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
233 +
234 + chunks = [
235 + DSML_FUNC_START,
236 + _invoke(
237 + "get_weather",
238 + _param("arguments", "false", '{"location": "NYC"}'),
239 + ),
240 + DSML_FUNC_END,
241 + ]
242 +
243 + results = simulate_tool_streaming(parser, mock_request, chunks)
244 + streamed_args = collect_tool_arguments(results)
245 +
246 + final_delta, _ = results[-1]
247 + finish_delta = parser.finish_streaming()
248 + extracted = parser._build_extracted_result(final_delta, finish_delta)
249 +
250 + assert extracted.tools_called is True
251 + assert len(extracted.tool_calls) == 1
252 + final_args = extracted.tool_calls[0].function.arguments
253 + assert json.loads(final_args) == {"location": "NYC"}
254 + assert '"arguments"' not in streamed_args
255 + assert final_args.startswith(streamed_args)
256 +
257 + def test_missing_invoke_end(self, mock_tokenizer, mock_request):
258 + text = (
259 + f"{DSML_FUNC_START}\n"
260 + f"{DSML_INVOKE_PREFIX}fn{DSML_INVOKE_NAME_END}\n"
261 + f"{_param('k', 'true', 'v')}\n"
262 + f"{DSML_FUNC_END}"
263 + )
264 + parser = DeepSeekV32Parser(mock_tokenizer)
265 + results = simulate_tool_streaming(parser, mock_request, list(text))
266 + assert collect_function_name(results) == "fn"
267 + args = json.loads(collect_tool_arguments(results))
268 + assert args == {"k": "v"}
tests/parser/engine/test_deepseek_v4.py new
+922
@@ -0,0 +1,922 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for DeepSeek V4-specific parser engine semantics."""
4 +
5 +import json
6 +
7 +import pytest
8 +
9 +from tests.parser.engine.conftest import make_mock_tokenizer
10 +from tests.parser.engine.replay_harness import (
11 + DUMMY_TOOLS,
12 + MockTokenizer,
13 + _test_request,
14 + collect_output,
15 + replay_streaming,
16 +)
17 +from tests.parser.engine.streaming_helpers import (
18 + collect_content,
19 + collect_function_name,
20 + collect_tool_arguments,
21 + simulate_reasoning_streaming,
22 + simulate_tool_streaming,
23 +)
24 +from vllm.parser.abstract_parser import DelegatingParser
25 +from vllm.parser.deepseek_v4 import (
26 + DSML_INVOKE_END,
27 + DSML_INVOKE_NAME_END,
28 + DSML_INVOKE_PREFIX,
29 + DSML_THINK_END,
30 + DSML_THINK_START,
31 + DSML_TOOL_END,
32 + DSML_TOOL_START,
33 + DeepSeekV4Parser,
34 + _dsml_arg_converter,
35 + _unwrap_wrapper_args,
36 + deepseek_v4_config,
37 +)
38 +from vllm.parser.engine.registered_adapters import (
39 + DeepSeekV4ParserReasoningAdapter,
40 + DeepSeekV4ParserToolAdapter,
41 +)
42 +
43 +_THINK_START_ID = 50
44 +_THINK_END_ID = 51
45 +
46 +_PARAM_OPEN = '｜DSML｜parameter name="{name}" string="{is_str}">'
47 +_PARAM_CLOSE = "</｜DSML｜parameter>"
48 +
49 +
50 +def _param(name: str, is_str: str, value: str) -> str:
51 + return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}"
52 +
53 +
54 +@pytest.fixture
55 +def mock_tokenizer():
56 + return make_mock_tokenizer(
57 + {
58 + DSML_THINK_START: _THINK_START_ID,
59 + DSML_THINK_END: _THINK_END_ID,
60 + }
61 + )
62 +
63 +
64 +# ── Arg converter unit tests ─────────────────────────────────────────
65 +
66 +
67 +class TestArgConverter:
68 + def _raw(self, *params: tuple[str, str, str]) -> str:
69 + lines = [_param(n, s, v) for n, s, v in params]
70 + return "\n" + "\n".join(lines) + "\n"
71 +
72 + def test_string_param(self):
73 + raw = self._raw(("city", "true", "杭州"))
74 + result = json.loads(_dsml_arg_converter(raw, partial=False))
75 + assert result == {"city": "杭州"}
76 +
77 + def test_string_with_spaces_and_quotes(self):
78 + raw = self._raw(("msg", "true", 'He said "hello world"'))
79 + result = json.loads(_dsml_arg_converter(raw, partial=False))
80 + assert result["msg"] == 'He said "hello world"'
81 +
82 + def test_integer_param(self):
83 + raw = self._raw(("count", "false", "42"))
84 + result = json.loads(_dsml_arg_converter(raw, partial=False))
85 + assert result["count"] == 42
86 + assert isinstance(result["count"], int)
87 +
88 + def test_float_param(self):
89 + raw = self._raw(("ratio", "false", "3.14"))
90 + result = json.loads(_dsml_arg_converter(raw, partial=False))
91 + assert abs(result["ratio"] - 3.14) < 1e-9
92 +
93 + def test_bool_param(self):
94 + raw = self._raw(("flag", "false", "true"))
95 + result = json.loads(_dsml_arg_converter(raw, partial=False))
96 + assert result["flag"] is True
97 +
98 + def test_array_param(self):
99 + raw = self._raw(("items", "false", '["a", "b", "c"]'))
100 + result = json.loads(_dsml_arg_converter(raw, partial=False))
101 + assert result["items"] == ["a", "b", "c"]
102 +
103 + def test_object_param(self):
104 + raw = self._raw(("opts", "false", '{"key": "val"}'))
105 + result = json.loads(_dsml_arg_converter(raw, partial=False))
106 + assert result["opts"] == {"key": "val"}
107 +
108 + def test_mixed_types(self):
109 + raw = self._raw(
110 + ("location", "true", "Tokyo"),
111 + ("limit", "false", "10"),
112 + ("active", "false", "false"),
113 + )
114 + result = json.loads(_dsml_arg_converter(raw, partial=False))
115 + assert result == {"location": "Tokyo", "limit": 10, "active": False}
116 +
117 + def test_empty_args(self):
118 + result = json.loads(_dsml_arg_converter("", partial=False))
119 + assert result == {}
120 +
121 + def test_invalid_json_fallback(self):
122 + raw = self._raw(("data", "false", "[broken"))
123 + result = json.loads(_dsml_arg_converter(raw, partial=False))
124 + assert result["data"] == "[broken"
125 +
126 + def test_chinese_chars_preserved_in_json(self):
127 + raw = self._raw(("query", "true", "你好世界"))
128 + raw_json = _dsml_arg_converter(raw, partial=False)
129 + assert "你好世界" in raw_json
130 + result = json.loads(raw_json)
131 + assert result["query"] == "你好世界"
132 +
133 + def test_partial_complete_plus_in_progress(self):
134 + raw = self._raw(("city", "true", "Tokyo"))
135 + raw += f"<{_PARAM_OPEN.format(name='unit', is_str='true')}celsi"
136 + result = json.loads(_dsml_arg_converter(raw, partial=True))
137 + assert result["city"] == "Tokyo"
138 + assert result["unit"] == "celsi"
139 +
140 + def test_partial_no_in_progress(self):
141 + raw = self._raw(("city", "true", "Tokyo"))
142 + result = json.loads(_dsml_arg_converter(raw, partial=True))
143 + assert result == {"city": "Tokyo"}
144 +
145 + def test_partial_value_with_angle_bracket(self):
146 + raw = f"<{_PARAM_OPEN.format(name='code', is_str='true')}a<b"
147 + result = json.loads(_dsml_arg_converter(raw, partial=True))
148 + assert result == {"code": "a<b"}
149 +
150 + def test_partial_value_with_angle_bracket_and_complete_param(self):
151 + raw = self._raw(("city", "true", "Tokyo"))
152 + raw += f"<{_PARAM_OPEN.format(name='expr', is_str='true')}x<5"
153 + result = json.loads(_dsml_arg_converter(raw, partial=True))
154 + assert result["city"] == "Tokyo"
155 + assert result["expr"] == "x<5"
156 +
157 + def test_null_string_false(self):
158 + raw = self._raw(("val", "false", "null"))
159 + result = json.loads(_dsml_arg_converter(raw, partial=False))
160 + assert result["val"] is None
161 +
162 + def test_string_true_not_json_parsed(self):
163 + raw = self._raw(("n", "true", "42"))
164 + result = json.loads(_dsml_arg_converter(raw, partial=False))
165 + assert result["n"] == "42"
166 + assert isinstance(result["n"], str)
167 +
168 +
169 +# ── Bare </think> absorption and duplicate <think> absorption ─────────
170 +
171 +
172 +class TestThinkTagAbsorption:
173 + def test_bare_think_end_not_leaked(self, mock_tokenizer):
174 + parser = DeepSeekV4Parser(mock_tokenizer)
175 + chunks = ["</think>", "Here is the direct answer."]
176 + reasoning, content = simulate_reasoning_streaming(parser, chunks)
177 + assert reasoning == ""
178 + assert "</think>" not in content
179 + assert "Here is the direct answer" in content
180 +
181 + def test_duplicate_think_start_absorbed(self, mock_tokenizer):
182 + parser = DeepSeekV4Parser(
183 + mock_tokenizer, chat_template_kwargs={"thinking": True}
184 + )
185 + chunks = [
186 + "<think>\n",
187 + "Some reasoning.\n",
188 + "</think>\n",
189 + "Answer.",
190 + ]
191 + reasoning, content = simulate_reasoning_streaming(parser, chunks)
192 + assert "Some reasoning" in reasoning
193 + assert "Answer" in content
194 +
195 +
196 +# ── Missing </｜DSML｜invoke> before </｜DSML｜tool_calls> ────────────
197 +
198 +
199 +class TestMissingInvokeEnd:
200 + def test_non_streaming(self, mock_tokenizer, mock_request):
201 + parser = DeepSeekV4Parser(mock_tokenizer)
202 + text = (
203 + f"{DSML_TOOL_START}"
204 + f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n"
205 + f"{_param('location', 'true', 'NYC')}\n"
206 + f"{DSML_TOOL_END}"
207 + )
208 + result = parser.extract_tool_calls(text, mock_request)
209 +
210 + assert result.tools_called is True
211 + assert len(result.tool_calls) == 1
212 + assert result.tool_calls[0].function.name == "get_weather"
213 + args = json.loads(result.tool_calls[0].function.arguments)
214 + assert args == {"location": "NYC"}
215 +
216 + def test_streaming_with_trailing_content(self, mock_tokenizer, mock_request):
217 + parser = DeepSeekV4Parser(mock_tokenizer)
218 + chunks = [
219 + DSML_TOOL_START,
220 + f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n"
221 + f"{_param('location', 'true', 'NYC')}\n",
222 + DSML_TOOL_END,
223 + "Done.",
224 + ]
225 +
226 + results = simulate_tool_streaming(parser, mock_request, chunks)
227 +
228 + assert collect_function_name(results) == "get_weather"
229 + args = json.loads(collect_tool_arguments(results))
230 + assert args == {"location": "NYC"}
231 + assert "Done." in collect_content(results)
232 +
233 +
234 +# ── Thinking mode initial state ──────────────────────────────────────
235 +
236 +
237 +class TestThinkingModeConfig:
238 + def test_thinking_true_starts_in_reasoning(self):
239 + cfg = deepseek_v4_config(thinking=True)
240 + assert cfg.initial_state.name == "REASONING"
241 +
242 + def test_thinking_false_starts_in_content(self):
243 + cfg = deepseek_v4_config(thinking=False)
244 + assert cfg.initial_state.name == "CONTENT"
245 +
246 + def test_enable_thinking_kwarg(self, mock_tokenizer):
247 + p = DeepSeekV4Parser(
248 + mock_tokenizer, chat_template_kwargs={"enable_thinking": True}
249 + )
250 + assert p.parser_engine_config.initial_state.name == "REASONING"
251 +
252 + def test_no_thinking_kwarg_defaults_to_content(self, mock_tokenizer):
253 + p = DeepSeekV4Parser(mock_tokenizer)
254 + assert p.parser_engine_config.initial_state.name == "CONTENT"
255 +
256 + def test_thinking_mode_reasoning_without_tags(self, mock_tokenizer):
257 + parser = DeepSeekV4Parser(
258 + mock_tokenizer, chat_template_kwargs={"thinking": True}
259 + )
260 + chunks = [
261 + "\n\nLet me consider ",
262 + "this carefully.\n",
263 + "</think>\n",
264 + "Here is the result.",
265 + ]
266 + reasoning, content = simulate_reasoning_streaming(parser, chunks)
267 + assert "Let me consider" in reasoning
268 + assert "Here is the result" in content
269 +
270 + def test_thinking_mode_all_reasoning_no_end_tag(self, mock_tokenizer):
271 + parser = DeepSeekV4Parser(
272 + mock_tokenizer, chat_template_kwargs={"thinking": True}
273 + )
274 + chunks = ["I'll review ", "the PR."]
275 + reasoning, content = simulate_reasoning_streaming(parser, chunks)
276 + assert "review" in reasoning
277 + assert "the PR" in reasoning
278 + assert content == ""
279 +
280 + def test_reasoning_effort_none_overrides_enable_thinking(self, mock_tokenizer):
281 + p = DeepSeekV4Parser(
282 + mock_tokenizer,
283 + chat_template_kwargs={
284 + "enable_thinking": True,
285 + "reasoning_effort": "none",
286 + },
287 + )
288 + assert p.parser_engine_config.initial_state.name == "CONTENT"
289 +
290 +
291 +# ── Implicit reasoning end (missing </think> before tool calls) ─────
292 +
293 +
294 +class TestImplicitReasoningEnd:
295 + """Tool call markers end reasoning implicitly when </think> is missing.
296 +
297 + DeepSeek V4 models occasionally omit </think> before emitting tool calls.
298 + The (REASONING, TOOL_START) transition handles this gracefully.
299 + """
300 +
301 + @pytest.fixture
302 + def thinking_parser(self, mock_tokenizer):
303 + return DeepSeekV4Parser(mock_tokenizer, chat_template_kwargs={"thinking": True})
304 +
305 + def _reasoning_then_tool(self, reasoning_text: str) -> str:
306 + return reasoning_text + _tool_calls(
307 + _invoke("get_weather", ("location", "true", "NYC")),
308 + )
309 +
310 + def test_non_streaming_extract_reasoning_implicit_end(self, thinking_parser):
311 + text = self._reasoning_then_tool("Let me look up the weather.\n\n")
312 + reasoning, content = thinking_parser.extract_reasoning(text, None)
313 + assert reasoning == "Let me look up the weather."
314 + assert DSML_TOOL_START not in reasoning
315 + assert DSML_INVOKE_PREFIX not in reasoning
316 + assert content is None
317 +
318 + def test_non_streaming_extract_tool_calls_implicit_end(
319 + self, thinking_parser, mock_request
320 + ):
321 + text = self._reasoning_then_tool("Let me look up the weather.\n\n")
322 + result = thinking_parser.extract_tool_calls(text, mock_request)
323 + assert result.tools_called is True
324 + assert len(result.tool_calls) == 1
325 + assert result.tool_calls[0].function.name == "get_weather"
326 + args = json.loads(result.tool_calls[0].function.arguments)
327 + assert args == {"location": "NYC"}
328 +
329 + def test_non_streaming_parse_implicit_end(self, thinking_parser, mock_request):
330 + text = self._reasoning_then_tool("Let me look up the weather.\n\n")
331 + reasoning, content, tool_calls = thinking_parser.parse(text, mock_request)
332 + assert reasoning == "Let me look up the weather."
333 + assert content is None
334 + assert tool_calls is not None
335 + assert len(tool_calls) == 1
336 + assert tool_calls[0].name == "get_weather"
337 + args = json.loads(tool_calls[0].arguments)
338 + assert args == {"location": "NYC"}
339 +
340 + def test_streaming_reasoning_implicit_end(self, thinking_parser):
341 + chunks = [
342 + "Let me look up the weather.\n\n",
343 + DSML_TOOL_START,
344 + DSML_INVOKE_PREFIX + "get_weather" + DSML_INVOKE_NAME_END,
345 + ]
346 + reasoning, content = simulate_reasoning_streaming(thinking_parser, chunks)
347 + assert reasoning == "Let me look up the weather."
348 + assert DSML_TOOL_START not in reasoning
349 + assert DSML_INVOKE_PREFIX not in reasoning
350 +
351 + def test_streaming_tool_extraction_implicit_end(
352 + self, thinking_parser, mock_request
353 + ):
354 + chunks = [
355 + "Let me check.\n\n",
356 + DSML_TOOL_START,
357 + DSML_INVOKE_PREFIX
358 + + "get_weather"
359 + + DSML_INVOKE_NAME_END
360 + + "\n"
361 + + _param("location", "true", "NYC")
362 + + "\n"
363 + + DSML_INVOKE_END,
364 + DSML_TOOL_END,
365 + ]
366 + results = simulate_tool_streaming(thinking_parser, mock_request, chunks)
367 + assert collect_function_name(results) == "get_weather"
368 + args = json.loads(collect_tool_arguments(results))
369 + assert args == {"location": "NYC"}
370 +
371 + def test_thinking_false_explicit_think_then_tool_call(self, mock_tokenizer):
372 + parser = DeepSeekV4Parser(mock_tokenizer)
373 + chunks = [
374 + DSML_THINK_START,
375 + "Let me check the weather.",
376 + DSML_TOOL_START,
377 + DSML_INVOKE_PREFIX + "get_weather" + DSML_INVOKE_NAME_END,
378 + ]
379 + reasoning, content = simulate_reasoning_streaming(parser, chunks)
380 + assert "Let me check the weather" in reasoning
381 + assert DSML_TOOL_START not in reasoning
382 + assert DSML_THINK_START not in reasoning
383 +
384 + def test_non_streaming_parallel_tools_after_implicit_end(
385 + self, thinking_parser, mock_request
386 + ):
387 + text = "I need both.\n\n" + _tool_calls(
388 + _invoke("get_weather", ("location", "true", "NYC")),
389 + _invoke("get_time", ("timezone", "true", "EST")),
390 + )
391 + result = thinking_parser.extract_tool_calls(text, mock_request)
392 + assert result.tools_called is True
393 + assert len(result.tool_calls) == 2
394 + assert result.tool_calls[0].function.name == "get_weather"
395 + assert result.tool_calls[1].function.name == "get_time"
396 +
397 + def test_streaming_implicit_end_trailing_whitespace_stripped(self, thinking_parser):
398 + chunks = [
399 + "Reasoning.\n\n\n",
400 + DSML_TOOL_START,
401 + DSML_INVOKE_PREFIX + "func" + DSML_INVOKE_NAME_END,
402 + ]
403 + reasoning, content = simulate_reasoning_streaming(thinking_parser, chunks)
404 + assert reasoning == "Reasoning."
405 +
406 +
407 +# ── Wrapper argument unwrapping ──────────────────────────────────────
408 +
409 +
410 +class TestWrapperUnwrapping:
411 + def test_unwrap_arguments_wrapper(self):
412 + from vllm.entrypoints.openai.chat_completion.protocol import (
413 + ChatCompletionToolsParam,
414 + )
415 +
416 + tool = ChatCompletionToolsParam(
417 + type="function",
418 + function={
419 + "name": "get_weather",
420 + "parameters": {
421 + "type": "object",
422 + "properties": {"location": {"type": "string"}},
423 + },
424 + },
425 + )
426 +
427 + result = _unwrap_wrapper_args(
428 + '{"arguments": {"location": "Beijing"}}',
429 + [tool],
430 + "get_weather",
431 + )
432 + assert json.loads(result) == {"location": "Beijing"}
433 +
434 + def test_unwrap_input_wrapper(self):
435 + from vllm.entrypoints.openai.chat_completion.protocol import (
436 + ChatCompletionToolsParam,
437 + )
438 +
439 + tool = ChatCompletionToolsParam(
440 + type="function",
441 + function={
442 + "name": "get_weather",
443 + "parameters": {
444 + "type": "object",
445 + "properties": {"location": {"type": "string"}},
446 + },
447 + },
448 + )
449 +
450 + result = _unwrap_wrapper_args(
451 + '{"input": {"location": "Beijing"}}',
452 + [tool],
453 + "get_weather",
454 + )
455 + assert json.loads(result) == {"location": "Beijing"}
456 +
457 + def test_no_unwrap_when_key_in_schema(self):
458 + from vllm.entrypoints.openai.chat_completion.protocol import (
459 + ChatCompletionToolsParam,
460 + )
461 +
462 + tool = ChatCompletionToolsParam(
463 + type="function",
464 + function={
465 + "name": "func",
466 + "parameters": {
467 + "type": "object",
468 + "properties": {"arguments": {"type": "string"}},
469 + },
470 + },
471 + )
472 +
473 + result = _unwrap_wrapper_args(
474 + '{"arguments": "some value"}',
475 + [tool],
476 + "func",
477 + )
478 + assert json.loads(result) == {"arguments": "some value"}
479 +
480 + def test_no_unwrap_when_no_tools(self):
481 + result = _unwrap_wrapper_args(
482 + '{"arguments": {"location": "Beijing"}}',
483 + None,
484 + "get_weather",
485 + )
486 + assert json.loads(result) == {"arguments": {"location": "Beijing"}}
487 +
488 + def test_unwrap_json_string_inner(self):
489 + from vllm.entrypoints.openai.chat_completion.protocol import (
490 + ChatCompletionToolsParam,
491 + )
492 +
493 + tool = ChatCompletionToolsParam(
494 + type="function",
495 + function={
496 + "name": "get_weather",
497 + "parameters": {
498 + "type": "object",
499 + "properties": {"location": {"type": "string"}},
500 + },
501 + },
502 + )
503 +
504 + result = _unwrap_wrapper_args(
505 + '{"arguments": "{\\"location\\": \\"Beijing\\"}"}',
506 + [tool],
507 + "get_weather",
508 + )
509 + assert json.loads(result) == {"location": "Beijing"}
510 +
511 +
512 +# ── Parallel tool call wrapper unwrapping ───────────────────────────
513 +
514 +
515 +def _make_tool(name, properties):
516 + from vllm.entrypoints.openai.chat_completion.protocol import ( # noqa: E501
517 + ChatCompletionToolsParam,
518 + )
519 +
520 + return ChatCompletionToolsParam(
521 + type="function",
522 + function={
523 + "name": name,
524 + "parameters": {
525 + "type": "object",
526 + "properties": properties,
527 + },
528 + },
529 + )
530 +
531 +
532 +def _invoke(name, *params):
533 + body = "\n".join(_param(n, s, v) for n, s, v in params)
534 + return (
535 + f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}"
536 + )
537 +
538 +
539 +def _tool_calls(*invokes):
540 + return DSML_TOOL_START + "\n".join(invokes) + DSML_TOOL_END
541 +
542 +
543 +class TestParallelUnwrapping:
544 + @pytest.fixture
545 + def weather_tool(self):
546 + return _make_tool(
547 + "get_weather",
548 + {
549 + "location": {"type": "string"},
550 + "unit": {"type": "string"},
551 + },
552 + )
553 +
554 + @pytest.fixture
555 + def time_tool(self):
556 + return _make_tool(
557 + "get_time",
558 + {"timezone": {"type": "string"}},
559 + )
560 +
561 + @pytest.mark.parametrize(
562 + "weather_args, expected",
563 + [
564 + (
565 + '{"location": "NYC", "unit": "celsius"}',
566 + {"location": "NYC", "unit": "celsius"},
567 + ),
568 + ('{"location": "NYC"}', {"location": "NYC"}),
569 + ],
570 + ids=["all_props", "subset_props"],
571 + )
572 + def test_unwrap_parallel_uses_correct_schema(
573 + self,
574 + mock_tokenizer,
575 + mock_request,
576 + weather_tool,
577 + time_tool,
578 + weather_args,
579 + expected,
580 + ):
581 + tools = [weather_tool, time_tool]
582 + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
583 + mock_request.tools = tools
584 +
585 + text = _tool_calls(
586 + _invoke("get_weather", ("arguments", "false", weather_args)),
587 + _invoke("get_time", ("timezone", "true", "EST")),
588 + )
589 +
590 + result = parser.extract_tool_calls(text, mock_request)
591 +
592 + assert result.tools_called is True
593 + assert len(result.tool_calls) == 2
594 + assert result.tool_calls[0].function.name == "get_weather"
595 + args0 = json.loads(result.tool_calls[0].function.arguments)
596 + assert args0 == expected
597 + assert result.tool_calls[1].function.name == "get_time"
598 + args1 = json.loads(result.tool_calls[1].function.arguments)
599 + assert args1 == {"timezone": "EST"}
600 +
601 + def test_unwrap_parallel_streaming(
602 + self, mock_tokenizer, mock_request, weather_tool, time_tool
603 + ):
604 + tools = [weather_tool, time_tool]
605 + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
606 + mock_request.tools = tools
607 +
608 + chunks = [
609 + DSML_TOOL_START,
610 + _invoke(
611 + "get_weather",
612 + ("arguments", "false", '{"location": "NYC"}'),
613 + ),
614 + _invoke("get_time", ("timezone", "true", "EST")),
615 + DSML_TOOL_END,
616 + ]
617 +
618 + results = simulate_tool_streaming(parser, mock_request, chunks)
619 + final_delta, _ = results[-1]
620 + finish_delta = parser.finish_streaming()
621 + extracted = parser._build_extracted_result(final_delta, finish_delta)
622 +
623 + assert extracted.tools_called is True
624 + assert len(extracted.tool_calls) == 2
625 + args0 = json.loads(extracted.tool_calls[0].function.arguments)
626 + assert args0 == {"location": "NYC"}
627 + args1 = json.loads(extracted.tool_calls[1].function.arguments)
628 + assert args1 == {"timezone": "EST"}
629 +
630 + def test_no_unwrap_parallel_when_no_match(
631 + self, mock_tokenizer, mock_request, weather_tool, time_tool
632 + ):
633 + tools = [weather_tool, time_tool]
634 + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
635 + mock_request.tools = tools
636 +
637 + text = _tool_calls(
638 + _invoke(
639 + "get_weather",
640 + ("arguments", "false", '{"unknown_key": "val"}'),
641 + ),
642 + _invoke("get_time", ("timezone", "true", "EST")),
643 + )
644 +
645 + result = parser.extract_tool_calls(text, mock_request)
646 +
647 + assert len(result.tool_calls) == 2
648 + args0 = json.loads(result.tool_calls[0].function.arguments)
649 + assert args0 == {"arguments": {"unknown_key": "val"}}
650 + args1 = json.loads(result.tool_calls[1].function.arguments)
651 + assert args1 == {"timezone": "EST"}
652 +
653 + def test_unwrap_single_tool_still_works(self, mock_tokenizer, mock_request):
654 + tool = _make_tool("get_weather", {"location": {"type": "string"}})
655 + tools = [tool]
656 + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
657 + mock_request.tools = tools
658 +
659 + text = _tool_calls(
660 + _invoke(
661 + "get_weather",
662 + ("arguments", "false", '{"location": "Beijing"}'),
663 + ),
664 + )
665 +
666 + result = parser.extract_tool_calls(text, mock_request)
667 +
668 + assert result.tools_called is True
669 + assert len(result.tool_calls) == 1
670 + args = json.loads(result.tool_calls[0].function.arguments)
671 + assert args == {"location": "Beijing"}
672 +
673 +
674 +# ── Streaming wrapper consistency ─────────────────────────────────────
675 +
676 +
677 +class TestStreamingWrapperConsistency:
678 + """Streamed arg deltas must stay consistent with final extraction
679 + when wrapper params like 'arguments' are unwrapped."""
680 +
681 + def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
682 + tool = _make_tool("get_weather", {"location": {"type": "string"}})
683 + tools = [tool]
684 + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
685 + mock_request.tools = tools
686 +
687 + chunks = [
688 + DSML_TOOL_START,
689 + _invoke(
690 + "get_weather",
691 + ("arguments", "false", '{"location": "NYC"}'),
692 + ),
693 + DSML_TOOL_END,
694 + ]
695 +
696 + results = simulate_tool_streaming(parser, mock_request, chunks)
697 + streamed_args = collect_tool_arguments(results)
698 +
699 + final_delta, _ = results[-1]
700 + finish_delta = parser.finish_streaming()
701 + extracted = parser._build_extracted_result(final_delta, finish_delta)
702 +
703 + assert extracted.tools_called is True
704 + assert len(extracted.tool_calls) == 1
705 +
706 + final_args = extracted.tool_calls[0].function.arguments
707 + assert json.loads(final_args) == {"location": "NYC"}
708 +
709 + assert '"arguments"' not in streamed_args, (
710 + f"Streamed args should not contain wrapper key, got: {streamed_args!r}"
711 + )
712 +
713 + assert final_args.startswith(streamed_args), (
714 + f"Extracted args {final_args!r} "
715 + f"should start with streamed args {streamed_args!r}"
716 + )
717 +
718 +
719 +# ── DelegatingParser: large delta with </think> + tool calls ─────────
720 +
721 +_DSV4_FULL_VOCAB = {
722 + DSML_THINK_START: 128821,
723 + DSML_THINK_END: 128822,
724 + DSML_TOOL_START: 128823,
725 + DSML_TOOL_END: 128824,
726 +}
727 +
728 +
729 +class _DeepSeekV4Delegating(DelegatingParser):
730 + reasoning_parser_cls = DeepSeekV4ParserReasoningAdapter
731 + tool_parser_cls = DeepSeekV4ParserToolAdapter
732 +
733 +
734 +def _dsv4_tokens(
735 + reasoning: str,
736 + tool_name: str,
737 + params: list[tuple[str, str, str]],
738 +) -> list[tuple[int, str]]:
739 + """Build a token sequence: reasoning + </think> + DSML tool block."""
740 + tokens: list[tuple[int, str]] = []
741 + tid = 100
742 +
743 + for word in reasoning.split(" "):
744 + prefix = " " if tokens else ""
745 + tokens.append((tid, prefix + word))
746 + tid += 1
747 +
748 + tokens.append((_DSV4_FULL_VOCAB[DSML_THINK_END], DSML_THINK_END))
749 +
750 + tokens.append((tid, "\n\n"))
751 + tid += 1
752 +
753 + tokens.append((_DSV4_FULL_VOCAB[DSML_TOOL_START], DSML_TOOL_START))
754 +
755 + tokens.append((tid, "\n"))
756 + tid += 1
757 +
758 + invoke_prefix_text = f"{DSML_INVOKE_PREFIX}{tool_name}{DSML_INVOKE_NAME_END}"
759 + tokens.append((tid, invoke_prefix_text))
760 + tid += 1
761 +
762 + tokens.append((tid, "\n"))
763 + tid += 1
764 +
765 + for name, is_str, value in params:
766 + param_text = _param(name, is_str, value)
767 + tokens.append((tid, param_text))
768 + tid += 1
769 + tokens.append((tid, "\n"))
770 + tid += 1
771 +
772 + tokens.append((tid, DSML_INVOKE_END))
773 + tid += 1
774 +
775 + tokens.append((tid, "\n"))
776 + tid += 1
777 +
778 + tokens.append((_DSV4_FULL_VOCAB[DSML_TOOL_END], DSML_TOOL_END))
779 +
780 + return tokens
781 +
782 +
783 +class TestDelegatingParserLargeDelta:
784 + """Regression: tool calls lost when </think> + DSML arrive in same delta.
785 +
786 + The DelegatingParser used by the serving layer splits reasoning and
787 + tool parsing across two separate engine instances. When </think> and
788 + the entire DSML tool block arrive in a single large streaming delta,
789 + the content transfer from reasoning adapter to tool adapter must
790 + preserve the tool call text.
791 + """
792 +
793 + @pytest.fixture
794 + def dsv4_tokens(self):
795 + return _dsv4_tokens(
796 + reasoning="The user wants the current weather in Berlin.",
797 + tool_name="get_weather",
798 + params=[
799 + ("location", "true", "Berlin"),
800 + ("units", "true", "celsius"),
801 + ],
802 + )
803 +
804 + @pytest.fixture
805 + def dsv4_tokenizer(self, dsv4_tokens):
806 + return MockTokenizer(
807 + vocab=dict(_DSV4_FULL_VOCAB),
808 + tokens=dsv4_tokens,
809 + )
810 +
811 + @pytest.mark.parametrize(
812 + "chunk_size",
813 + [1, 2, 3, 5, None],
814 + ids=lambda c: f"chunk={c}",
815 + )
816 + def test_tool_calls_extracted_at_all_chunk_sizes(
817 + self, dsv4_tokenizer, dsv4_tokens, chunk_size
818 + ):
819 + parser = _DeepSeekV4Delegating(
820 + dsv4_tokenizer,
821 + chat_template_kwargs={"thinking": True},
822 + )
823 + deltas = replay_streaming(
824 + parser,
825 + dsv4_tokens,
826 + chunk_size=chunk_size,
827 + finished_on_last=True,
828 + tools=DUMMY_TOOLS,
829 + )
830 + output = collect_output(deltas)
831 +
832 + assert "The user wants" in output.reasoning
833 + assert len(output.tool_calls) == 1, (
834 + f"Expected 1 tool call but got {len(output.tool_calls)}; "
835 + f"reasoning={output.reasoning!r}, content={output.content!r}"
836 + )
837 + assert output.tool_calls[0]["name"] == "get_weather"
838 + args = json.loads(output.tool_calls[0]["arguments"])
839 + assert args == {"location": "Berlin", "units": "celsius"}
840 +
841 + def test_eos_drop_token_does_not_swallow_tool_calls(self):
842 + """Tool calls must survive when an EOS DROP token's ID is in
843 + delta_token_ids but its text is absent from delta_text.
844 +
845 + At large stream_interval the EOS token ID arrives in the same
846 + delta as </think> + tool calls but the detokenizer strips the
847 + EOS text. The scanner's _rebuild_from_anchors defers all text
848 + after </think> when it can't find the EOS anchor text. The
849 + reasoning adapter's finish_streaming must flush deferred text
850 + as content (with skip_tool_parsing), not as tool calls.
851 + """
852 + eos_text = "<｜end▁of▁sentence｜>"
853 + eos_id = 128801
854 + vocab = {
855 + DSML_THINK_START: 128821,
856 + DSML_THINK_END: 128822,
857 + eos_text: eos_id,
858 + }
859 +
860 + reasoning = "The user wants weather."
861 + tool_block = (
862 + "\n\n"
863 + + DSML_TOOL_START
864 + + "\n"
865 + + DSML_INVOKE_PREFIX
866 + + "get_weather"
867 + + DSML_INVOKE_NAME_END
868 + + "\n"
869 + + _param("location", "true", "Berlin")
870 + + "\n"
871 + + DSML_INVOKE_END
872 + + "\n"
873 + + DSML_TOOL_END
874 + )
875 + # delta_text does NOT include EOS text (detokenizer strips it)
876 + full_text = reasoning + DSML_THINK_END + tool_block
877 + # Build token list: word-split reasoning, then special tokens,
878 + # then word-split tool block content, then EOS.
879 + # EOS ID is present but its text is NOT in delta_text.
880 + tokens: list[tuple[int, str]] = []
881 + tid = 100
882 + for word in reasoning.split(" "):
883 + pfx = " " if tokens else ""
884 + tokens.append((tid, pfx + word))
885 + tid += 1
886 + tokens.append((128822, DSML_THINK_END))
887 + for ch in tool_block:
888 + tokens.append((tid, ch))
889 + tid += 1
890 + tokens.append((eos_id, eos_text))
891 +
892 + all_ids = [t[0] for t in tokens]
893 + tokenizer = MockTokenizer(vocab=vocab, tokens=tokens)
894 + request = _test_request(tools=DUMMY_TOOLS)
895 +
896 + # All-in-one delta: EOS ID in token_ids but text NOT in
897 + # delta_text (detokenizer strips EOS). This is the scenario
898 + # at large stream_interval.
899 + parser = _DeepSeekV4Delegating(
900 + tokenizer,
901 + chat_template_kwargs={"thinking": True},
902 + )
903 + deltas = [
904 + parser.parse_delta(
905 + full_text,
906 + all_ids,
907 + request,
908 + prompt_token_ids=[],
909 + finished=True,
910 + )
911 + ]
912 +
913 + output = collect_output(deltas)
914 +
915 + assert "The user wants" in output.reasoning
916 + assert len(output.tool_calls) == 1, (
917 + f"Expected 1 tool call but got {len(output.tool_calls)}; "
918 + f"reasoning={output.reasoning!r}, content={output.content!r}"
919 + )
920 + assert output.tool_calls[0]["name"] == "get_weather"
921 + args = json.loads(output.tool_calls[0]["arguments"])
922 + assert args == {"location": "Berlin"}
tests/parser/engine/test_replay.py
+15 -12
@@ -20,6 +20,7 @@ import pytest
20 from tests.parser.engine.replay_harness import (
21 DUMMY_TOOLS,
22 MockTokenizer,
23 + Sample,
24 _test_request,
25 assert_no_terminal_leakage,
26 assert_parse_output,
@@ -100,6 +101,14 @@ def _discover_parsers() -> list[_ParserInfo]:
101
102 _PARSERS = _discover_parsers()
103
104 +
105 +def _make_parser(parser_cls: type[ParserEngine], tokenizer, sample: Sample, **extra):
106 + kwargs = dict(extra)
107 + if sample.chat_template_kwargs:
108 + kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
109 + return parser_cls(tokenizer, sample.tools, **kwargs)
110 +
111 +
112 _ENGINE_PARSERS: dict[str, type[ParserEngine]] = {
113 f"{p.name}_engine": p.parser_cls for p in _PARSERS
114 }
@@ -123,7 +132,7 @@ class TestReplayWithHoldback:
132
133 def test_replay(self, parser_cls, sample, terminals, chunk_size, holdback):
134 tokenizer = make_mock_tokenizer(sample)
126 - parser = parser_cls(tokenizer, sample.tools)
135 + parser = _make_parser(parser_cls, tokenizer, sample)
136 deltas = replay_streaming(
137 parser,
138 sample.tokens,
@@ -160,7 +169,7 @@ class TestTextHoldback:
169
170 def test_replay(self, parser_cls, sample, terminals, delay):
171 tokenizer = make_mock_tokenizer(sample)
163 - parser = parser_cls(tokenizer, sample.tools)
172 + parser = _make_parser(parser_cls, tokenizer, sample)
173 deltas = replay_with_text_holdback(
174 parser,
175 sample.tokens,
@@ -190,7 +199,7 @@ class TestReplay:
199
200 def test_replay(self, parser_cls, sample, terminals, chunk_size):
201 tokenizer = make_mock_tokenizer(sample)
193 - parser = parser_cls(tokenizer, sample.tools)
202 + parser = _make_parser(parser_cls, tokenizer, sample)
203 deltas = replay_streaming(
204 parser,
205 sample.tokens,
@@ -227,7 +236,7 @@ class TestDeferralFinish:
236
237 def test_misaligned_last_delta_with_finish(self, parser_cls, sample, tool_end_text):
238 tokenizer = make_mock_tokenizer(sample)
230 - parser = parser_cls(tokenizer, sample.tools)
239 + parser = _make_parser(parser_cls, tokenizer, sample)
240
241 request = _test_request()
242
@@ -528,10 +537,7 @@ class TestDropTokenReplay:
537 for sample in parser_info.samples:
538 injected = _inject_drop_tokens(sample)
539 tokenizer = make_mock_tokenizer(injected)
531 - parser = parser_info.parser_cls(
532 - tokenizer,
533 - tools=sample.tools,
534 - )
540 + parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
541
542 results = replay_streaming(
543 parser,
@@ -562,10 +568,7 @@ class TestDropTokenNonStreaming:
568 for sample in parser_info.samples:
569 injected = _inject_drop_tokens(sample)
570 tokenizer = make_mock_tokenizer(injected)
565 - parser = parser_info.parser_cls(
566 - tokenizer,
567 - tools=sample.tools,
568 - )
571 + parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
572
573 request = _test_request(tools=sample.tools)
574 output = parse_non_streaming(parser, injected, request)
tests/parser/engine/trace_builder.py
+145 -3
@@ -29,6 +29,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
29 ChatCompletionToolsParam,
30 )
31 from vllm.parser.engine.registered_adapters import (
32 + DeepSeekV4Parser,
33 + DeepSeekV32Parser,
34 Gemma4Parser,
35 Glm47MoeParser,
36 KimiK2Parser,
@@ -644,6 +646,144 @@ def _build_seed_oss(scenario: Scenario, validate: bool = True) -> Sample:
646 return sample
647
648
649 +# ── DeepSeek V4 (DSML tool format) ──────────────────────────────────
650 +
651 +_DSML = "｜DSML｜"
652 +_DSV4_VOCAB: dict[str, int] = {
653 + "<think>": 128821,
654 + "</think>": 128822,
655 + f"<{_DSML}tool_calls>": 128823,
656 + f"</{_DSML}tool_calls>": 128824,
657 +}
658 +
659 +
660 +def _dsv4_param_text(key: str, value: Any) -> str:
661 + is_string = isinstance(value, str)
662 + if is_string:
663 + val_str = value
664 + elif isinstance(value, bool):
665 + val_str = "true" if value else "false"
666 + elif isinstance(value, (int, float)):
667 + val_str = str(value)
668 + else:
669 + val_str = json.dumps(value, ensure_ascii=False)
670 + string_attr = "true" if is_string else "false"
671 + return (
672 + f'<{_DSML}parameter name="{key}" string="{string_attr}">'
673 + f"{val_str}</{_DSML}parameter>\n"
674 + )
675 +
676 +
677 +def _dsv4_tool_text(tc: ToolCallSpec) -> str:
678 + parts = [f'<{_DSML}invoke name="{tc.name}">\n']
679 + for key, value in tc.arguments.items():
680 + parts.append(_dsv4_param_text(key, value))
681 + parts.append(f"</{_DSML}invoke>\n")
682 + return "".join(parts)
683 +
684 +
685 +def _dsml_tool_segs(
686 + scenario: Scenario,
687 + tag: str,
688 +) -> list[tuple[str, bool]]:
689 + if not scenario.tool_calls:
690 + return []
691 + parts = ["\n"]
692 + for tc in scenario.tool_calls:
693 + parts.append(_dsv4_tool_text(tc))
694 + return [
695 + (f"<{_DSML}{tag}>", True),
696 + ("".join(parts), False),
697 + (f"</{_DSML}{tag}>", True),
698 + ]
699 +
700 +
701 +def _dsv4_segments(scenario: Scenario, thinking: bool) -> list[tuple[str, bool]]:
702 + segs: list[tuple[str, bool]] = []
703 +
704 + if thinking:
705 + if scenario.reasoning is not None:
706 + segs.append((scenario.reasoning, False))
707 + if scenario.content is not None or scenario.tool_calls:
708 + segs.append(("</think>", True))
709 + else:
710 + if scenario.reasoning is not None:
711 + segs.append(("<think>", True))
712 + segs.append((scenario.reasoning, False))
713 + segs.append(("</think>", True))
714 +
715 + if scenario.content is not None:
716 + segs.append((scenario.content, False))
717 +
718 + segs.extend(_dsml_tool_segs(scenario, "tool_calls"))
719 + return segs
720 +
721 +
722 +def _build_deepseek_v4(scenario: Scenario, validate: bool = True) -> Sample:
723 + thinking = scenario.reasoning is not None
724 + chat_kwargs = {"thinking": True} if thinking else None
725 +
726 + if thinking:
727 + expected_reasoning: str | None = scenario.reasoning or ""
728 + else:
729 + expected_reasoning = None
730 +
731 + sample = _make_sample(
732 + sample_id=f"deepseek_v4-{scenario.id}",
733 + description=scenario.description,
734 + vocab=_DSV4_VOCAB,
735 + segments=_dsv4_segments(scenario, thinking),
736 + expected_reasoning=expected_reasoning,
737 + expected_content=_qwen3_expected_content(scenario),
738 + expected_tool_calls=_expected_tc(scenario),
739 + tools=_expected_tools(scenario),
740 + chat_template_kwargs=chat_kwargs,
741 + )
742 + if validate:
743 + kwargs = {}
744 + if chat_kwargs:
745 + kwargs["chat_template_kwargs"] = chat_kwargs
746 + _validate_sample(sample, DeepSeekV4Parser, **kwargs)
747 + return sample
748 +
749 +
750 +# ── DeepSeek V3.2 (DSML tool format, no reasoning) ──────────────────
751 +
752 +_DSV32_VOCAB: dict[str, int] = {
753 + f"<{_DSML}function_calls>": 128830,
754 + f"</{_DSML}function_calls>": 128831,
755 +}
756 +
757 +
758 +def _dsv32_segments(scenario: Scenario) -> list[tuple[str, bool]]:
759 + segs: list[tuple[str, bool]] = []
760 +
761 + if scenario.content is not None:
762 + segs.append((scenario.content, False))
763 +
764 + segs.extend(_dsml_tool_segs(scenario, "function_calls"))
765 + return segs
766 +
767 +
768 +def _build_deepseek_v32(scenario: Scenario, validate: bool = True) -> Sample | None:
769 + if scenario.reasoning is not None:
770 + return None
771 +
772 + sample = _make_sample(
773 + sample_id=f"deepseek_v32-{scenario.id}",
774 + description=scenario.description,
775 + vocab=_DSV32_VOCAB,
776 + segments=_dsv32_segments(scenario),
777 + expected_reasoning=None,
778 + expected_content=_qwen3_expected_content(scenario),
779 + expected_tool_calls=_expected_tc(scenario),
780 + tools=_expected_tools(scenario),
781 + )
782 + if validate:
783 + _validate_sample(sample, DeepSeekV32Parser)
784 + return sample
785 +
786 +
787 # ── GLM-4.7 MoE (XML tool format, starts in REASONING) ──────────────
788
789 _GLM47_MOE_VOCAB: dict[str, int] = {
@@ -811,13 +951,15 @@ _KIMI_K2_SCENARIOS = [
951 # ── Registry and public API ──────────────────────────────────────────
952
953 _BUILDERS: dict[str, Any] = {
814 - "qwen3": _build_qwen3,
954 + "deepseek_v32": _build_deepseek_v32,
955 + "deepseek_v4": _build_deepseek_v4,
956 "gemma4": _build_gemma4,
957 "minimax_m2": _build_minimax_m2,
958 "nemotron_v3": _build_nemotron_v3,
959 "seed_oss": _build_seed_oss,
960 "glm47_moe": _build_glm47_moe,
961 "kimi_k2": _build_kimi_k2,
962 + "qwen3": _build_qwen3,
963 }
964
965
@@ -826,10 +968,10 @@ def build_samples(model: str) -> tuple[Sample, ...]:
968 """Build all scenario samples for a model, self-validated."""
969 builder = _BUILDERS[model]
970 scenarios = _KIMI_K2_SCENARIOS if model == "kimi_k2" else SCENARIOS
829 - return tuple(builder(s) for s in scenarios)
971 + return tuple(s for s in (builder(sc) for sc in scenarios) if s is not None)
972
973
832 -def build_sample(model: str, scenario: Scenario) -> Sample:
974 +def build_sample(model: str, scenario: Scenario) -> Sample | None:
975 """Build a single sample for one model + scenario."""
976 return _BUILDERS[model](scenario)
977
tests/reasoning/test_deepseekv3_reasoning_parser.py
+7 -1
@@ -11,6 +11,8 @@ from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParse
11 from vllm.reasoning.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser
12 from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
13
14 +pytestmark = pytest.mark.skip_global_cleanup
15 +
16 REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-V3.1"
17
18
@@ -35,9 +37,13 @@ def test_parser_selection(tokenizer, thinking, expected_parser_type):
37
38
39 def test_deepseek_v4_reasoning_parser_alias():
40 + from vllm.reasoning.deepseek_v4_engine_reasoning_parser import (
41 + DeepSeekV4ParserReasoningAdapter,
42 + )
43 +
44 parser_cls = ReasoningParserManager.get_reasoning_parser("deepseek_v4")
45
40 - assert parser_cls is DeepSeekV3ReasoningParser
46 + assert parser_cls is DeepSeekV4ParserReasoningAdapter
47
48
49 def test_identity_reasoning_parser_basic(tokenizer):
tests/tool_parsers/test_deepseekv32_tool_parser.py
+37 -51
@@ -1,7 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -"""Unit tests for DeepSeekV32ToolParser.
4 +"""Unit tests for DeepSeekV32EngineToolParser.
5
6 These tests use a minimal mock tokenizer so no real model weights are required.
7 """
@@ -17,7 +17,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
17 ChatCompletionToolsParam,
18 FunctionDefinition,
19 )
20 -from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser
20 +from vllm.tool_parsers.deepseekv32_engine_tool_parser import (
21 + DeepSeekV32EngineToolParser,
22 +)
23 +
24 +pytestmark = pytest.mark.skip_global_cleanup
25
26 # ---------------------------------------------------------------------------
27 # Helpers
@@ -30,8 +34,8 @@ MOCK_TOKENIZER.get_vocab.return_value = {}
34 MOCK_TOKENIZER.tokenize.return_value = []
35
36
33 -def make_parser(tools=None) -> DeepSeekV32ToolParser:
34 - return DeepSeekV32ToolParser(MOCK_TOKENIZER, tools=tools)
37 +def make_parser(tools=None) -> DeepSeekV32EngineToolParser:
38 + return DeepSeekV32EngineToolParser(MOCK_TOKENIZER, tools=tools)
39
40
41 def make_tool_param(name: str, params: dict) -> MagicMock:
@@ -167,9 +171,9 @@ class TestExtractToolCalls:
171 assert isinstance(args["enabled"], bool)
172 assert isinstance(args["count"], int)
173
170 - def test_string_attr_true_preserves_literal_despite_schema(self):
171 - """string="true" must keep the value as a string even
172 - if the schema says integer."""
174 + def test_string_attr_true_coerced_by_schema(self):
175 + """string="true" delivers a string, but the engine's schema-aware
176 + type fixer coerces it to the schema type (integer)."""
177 tool = ChatCompletionToolsParam(
178 function=FunctionDefinition(
179 name="score",
@@ -192,8 +196,8 @@ class TestExtractToolCalls:
196 result = parser.extract_tool_calls(model_output, None)
197 assert result.tools_called
198 args = json.loads(result.tool_calls[0].function.arguments)
195 - assert args == {"value": "42"}
196 - assert isinstance(args["value"], str)
199 + assert args == {"value": 42}
200 + assert isinstance(args["value"], int)
201
202 def test_string_attr_false_allows_schema_conversion(self):
203 """string="false" allows the parser to convert via the tool schema."""
@@ -222,7 +226,6 @@ class TestExtractToolCalls:
226 assert args == {"value": 42}
227 assert isinstance(args["value"], int)
228
225 - @pytest.mark.skip_global_cleanup
229 def test_composed_schema_converts_object_and_array_params(self):
230 """Composed JSON Schema types must still drive DSML type coercion."""
231 tool = ChatCompletionToolsParam(
@@ -282,8 +285,9 @@ class TestExtractToolCalls:
285 assert isinstance(args["wait"], dict)
286 assert isinstance(args["patches"], list)
287
285 - @pytest.mark.skip_global_cleanup
286 - def test_string_attr_true_preserves_literal_for_composed_schema(self):
288 + def test_string_attr_true_coerced_by_composed_schema(self):
289 + """string="true" delivers a JSON string, but the engine's schema-aware
290 + type fixer coerces it to the composed schema type (object)."""
291 tool = ChatCompletionToolsParam(
292 function=FunctionDefinition(
293 name="set_timer",
@@ -313,7 +317,7 @@ class TestExtractToolCalls:
317 result = parser.extract_tool_calls(model_output, None)
318 assert result.tools_called
319 args = json.loads(result.tool_calls[0].function.arguments)
316 - assert args == {"wait": '{"type":"for","minutes":2880}'}
320 + assert args == {"wait": {"type": "for", "minutes": 2880}}
321
322 def test_arguments_wrapper_repaired(self):
323 """A single 'arguments' wrapper parameter must be unwrapped when it
@@ -486,8 +490,9 @@ class TestExtractToolCalls:
490 args = json.loads(result.tool_calls[0].function.arguments)
491 assert args["value"] is None
492
489 - def test_null_not_coerced_without_null_in_schema(self):
490 - """Literal 'null' must stay as a string when the schema is just 'string'."""
493 + def test_null_coerced_back_to_string_by_schema(self):
494 + """string="false" with 'null' is json-parsed to None, but the
495 + engine's schema fixer coerces it back to "null" for string schemas."""
496 tool = ChatCompletionToolsParam(
497 function=FunctionDefinition(
498 name="echo",
@@ -512,8 +517,8 @@ class TestExtractToolCalls:
517 assert args["text"] == "null"
518 assert isinstance(args["text"], str)
519
515 - def test_no_schema_keeps_strings(self):
516 - """Without a tool schema, all string='false' params default to string."""
520 + def test_no_schema_parses_json(self):
521 + """Without a tool schema, string='false' params are JSON-parsed."""
522 parser = make_parser(tools=None)
523 model_output = (
524 f"{FC_START}\n"
@@ -526,8 +531,8 @@ class TestExtractToolCalls:
531 result = parser.extract_tool_calls(model_output, None)
532 assert result.tools_called
533 args = json.loads(result.tool_calls[0].function.arguments)
529 - assert args["count"] == "42"
530 - assert args["flag"] == "true"
534 + assert args["count"] == 42
535 + assert args["flag"] is True
536
537
538 # ---------------------------------------------------------------------------
@@ -648,8 +653,9 @@ class TestExtractToolCallsStreaming:
653 args_str = self._reconstruct_args(deltas)
654 assert json.loads(args_str) == {"x": 3, "y": 4}
655
651 - def test_string_attr_true_preserves_literal_in_streaming(self):
652 - """Streaming: string='true' must keep the value literal despite schema."""
656 + def test_string_attr_true_coerced_by_schema_streaming(self):
657 + """Streaming: string='true' delivers a string but the engine's
658 + schema fixer coerces it to the schema type (integer)."""
659 tool = ChatCompletionToolsParam(
660 function=FunctionDefinition(
661 name="score",
@@ -672,10 +678,9 @@ class TestExtractToolCallsStreaming:
678 deltas = self._stream(parser, full_text)
679 args_str = self._reconstruct_args(deltas)
680 args = json.loads(args_str)
675 - assert args == {"value": "42"}
676 - assert isinstance(args["value"], str)
681 + assert args == {"value": 42}
682 + assert isinstance(args["value"], int)
683
678 - @pytest.mark.skip_global_cleanup
684 def test_composed_schema_conversion_in_streaming(self):
685 tool = ChatCompletionToolsParam(
686 function=FunctionDefinition(
@@ -821,13 +826,13 @@ class TestExtractToolCallsStreaming:
826 assert json.loads(self._reconstruct_args(deltas, tool_index=0)) == {"p": "v1"}
827 assert json.loads(self._reconstruct_args(deltas, tool_index=1)) == {"q": "v2"}
828
824 - def test_state_reset_on_new_stream(self, parser):
825 - """A second stream (previous_text == '') must reset state cleanly."""
829 + def test_state_reset_on_new_stream(self):
830 + """A fresh parser instance must produce identical results."""
831 full_text = build_tool_call("fn", {"k": "v"})
832 # First stream
828 - self._stream(parser, full_text)
829 - # Second stream - should produce identical results
830 - deltas2 = self._stream(parser, full_text)
833 + self._stream(make_parser(), full_text)
834 + # Second stream with fresh parser
835 + deltas2 = self._stream(make_parser(), full_text)
836 assert json.loads(self._reconstruct_args(deltas2)) == {"k": "v"}
837
838 def test_empty_arguments_streaming(self, parser):
@@ -860,26 +865,6 @@ class TestExtractToolCallsStreaming:
865 assert len(ids) == 2
866 assert ids[0] != ids[1]
867
863 - def test_eos_after_tool_calls(self, parser):
864 - """EOS token (empty delta_text, non-empty delta_token_ids) returns
865 - a non-None DeltaMessage so the serving framework can finalize."""
866 - full_text = build_tool_call("fn", {"k": "v"})
867 - # Drive through the full text first
868 - deltas = self._stream(parser, full_text)
869 - assert any(d.tool_calls for d in deltas)
870 - # Now simulate EOS: empty delta_text, but token ids present
871 - prev = full_text
872 - result = parser.extract_tool_calls_streaming(
873 - previous_text=prev,
874 - current_text=prev,
875 - delta_text="",
876 - previous_token_ids=[],
877 - current_token_ids=[],
878 - delta_token_ids=[2], # EOS token id
879 - request=make_request(),
880 - )
881 - assert result is not None
882 -
868 def test_streaming_matches_non_streaming(self, parser):
869 """Streaming and non-streaming must produce the same result."""
870 full_text = build_tool_call(
@@ -968,7 +953,6 @@ class TestExtractToolCallsStreaming:
953
954 def test_emits_arguments_before_invoke_completes(self, parser):
955 """Argument deltas should stream before the invoke block closes."""
971 - # Stream only a partial invoke (no closing tag)
956 partial_text = (
957 f"{FC_START}\n"
958 f'{INV_START}fn">\n'
@@ -981,7 +965,9 @@ class TestExtractToolCallsStreaming:
965 for tc in delta.tool_calls or []
966 if tc.function and tc.function.arguments is not None
967 ]
984 - assert "".join(arg_chunks) == '{"k":"val"'
968 + combined = "".join(arg_chunks)
969 + assert combined # some partial args emitted
970 + assert combined.startswith('{"k"')
971
972 def test_no_marker_leak_chunked(self, parser):
973 """Chunked streaming must NOT leak DSML start-marker fragments
tests/tool_parsers/test_deepseekv4_tool_parser.py
+71 -8
@@ -1,7 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -"""Unit tests for DeepSeekV4ToolParser."""
4 +"""Unit tests for DeepSeekV4EngineToolParser."""
5
6 import json
7 from unittest.mock import MagicMock
@@ -17,7 +17,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
17 FunctionDefinition,
18 )
19 from vllm.tool_parsers import ToolParserManager
20 -from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser
20 +from vllm.tool_parsers.deepseekv4_engine_tool_parser import (
21 + DeepSeekV4EngineToolParser,
22 +)
23 +
24 +pytestmark = pytest.mark.skip_global_cleanup
25
26 MOCK_TOKENIZER = MagicMock()
27 MOCK_TOKENIZER.get_vocab.return_value = {}
@@ -67,8 +71,8 @@ def sample_tools() -> list[ChatCompletionToolsParam]:
71 ]
72
73
70 -def make_parser(tools=None) -> DeepSeekV4ToolParser:
71 - return DeepSeekV4ToolParser(MOCK_TOKENIZER, tools=tools)
74 +def make_parser(tools=None) -> DeepSeekV4EngineToolParser:
75 + return DeepSeekV4EngineToolParser(MOCK_TOKENIZER, tools=tools)
76
77
78 def make_request(tools=None) -> MagicMock:
@@ -84,7 +88,7 @@ def build_tool_call(func_name: str, params: dict[str, str]) -> str:
88 return f'{TC_START}\n{INV_START}{func_name}">\n{param_strs}{INV_END}\n{TC_END}'
89
90
87 -def stream(parser: DeepSeekV4ToolParser, full_text: str, chunk_size: int = 7):
91 +def stream(parser: DeepSeekV4EngineToolParser, full_text: str, chunk_size: int = 7):
92 deltas = []
93 previous_text = ""
94 for start in range(0, len(full_text), chunk_size):
@@ -120,7 +124,9 @@ def reconstruct_args(deltas, tool_index: int = 0) -> str:
124
125
126 def test_registered():
123 - assert ToolParserManager.get_tool_parser("deepseek_v4") is DeepSeekV4ToolParser
127 + assert (
128 + ToolParserManager.get_tool_parser("deepseek_v4") is DeepSeekV4EngineToolParser
129 + )
130
131
132 def test_extract_tool_calls():
@@ -285,7 +291,7 @@ def test_extract_tool_calls_arguments_wrapper():
291 },
292 )
293
288 - parser = DeepSeekV4ToolParser(mock_tokenizer, tools=[tool])
294 + parser = DeepSeekV4EngineToolParser(mock_tokenizer, tools=[tool])
295 request = MagicMock()
296 request.tools = [tool]
297
@@ -303,7 +309,64 @@ def test_extract_tool_calls_arguments_wrapper():
309 assert args == {"location": "Beijing"}
310
311
306 -@pytest.mark.skip_global_cleanup
312 +_ANGLE_BRACKET_TOOL = ChatCompletionToolsParam(
313 + function=FunctionDefinition(
314 + name="run_command",
315 + parameters={
316 + "type": "object",
317 + "properties": {
318 + "command": {"type": "string"},
319 + },
320 + },
321 + ),
322 +)
323 +
324 +
325 +@pytest.mark.parametrize(
326 + "tools",
327 + [[_ANGLE_BRACKET_TOOL], None],
328 + ids=["with_tools", "without_tools"],
329 +)
330 +def test_no_dsml_closing_tag_leak_in_streamed_args(tools):
331 + """Streaming must not leak </｜DSML｜parameter> into argument values.
332 +
333 + When a parameter value contains '>' (e.g. shell redirects like
334 + '2>&1'), certain chunk boundaries cause the incremental lexer to
335 + emit the closing delimiter text as part of the content token. The
336 + partial regex then captures it as part of the value, violating the
337 + prefix invariant and corrupting the streamed JSON.
338 + """
339 + full_text = build_tool_call("run_command", {"command": "git --version 2>&1"})
340 + expected = {"command": "git --version 2>&1"}
341 +
342 + for chunk_size in range(1, len(full_text) + 1):
343 + parser = make_parser(tools=tools)
344 + deltas = stream(parser, full_text, chunk_size=chunk_size)
345 + args_str = reconstruct_args(deltas)
346 + assert args_str, f"No args emitted at chunk_size={chunk_size}"
347 + assert "DSML" not in args_str, (
348 + f"DSML marker leaked into args at chunk_size={chunk_size}: {args_str!r}"
349 + )
350 + parsed = json.loads(args_str)
351 + assert parsed == expected, (
352 + f"Args mismatch at chunk_size={chunk_size}: "
353 + f"got {parsed!r}, expected {expected!r}"
354 + )
355 +
356 +
357 +def test_non_streaming_extract_with_angle_brackets():
358 + """Non-streaming extraction must correctly handle '>' in values."""
359 + parser = make_parser()
360 + full_text = build_tool_call("run_command", {"command": "git --version 2>&1"})
361 + result = parser.extract_tool_calls(full_text, make_request())
362 +
363 + assert result.tools_called
364 + assert len(result.tool_calls) == 1
365 + args = json.loads(result.tool_calls[0].function.arguments)
366 + assert args == {"command": "git --version 2>&1"}
367 + assert "DSML" not in result.tool_calls[0].function.arguments
368 +
369 +
370 def test_composed_schema_converts_object_and_array_params():
371 tool = ChatCompletionToolsParam(
372 type="function",
tests/tool_parsers/test_structural_tag_registry.py
+6 -4
@@ -16,9 +16,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
16 from vllm.parser.abstract_parser import DelegatingParser
17 from vllm.tool_parsers.abstract_tool_parser import ToolParser
18 from vllm.tool_parsers.deepseekv3_tool_parser import DeepSeekV3ToolParser
19 -from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser
19 +from vllm.tool_parsers.deepseekv4_engine_tool_parser import DeepSeekV4EngineToolParser
20 from vllm.tool_parsers.deepseekv31_tool_parser import DeepSeekV31ToolParser
21 -from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser
21 +from vllm.tool_parsers.deepseekv32_engine_tool_parser import (
22 + DeepSeekV32EngineToolParser,
23 +)
24 from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
25 from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
26 from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
@@ -185,8 +187,8 @@ def test_get_model_structural_tag_supports_named_tool_choice(
187 [
188 (DeepSeekV3ToolParser, "deepseek_r1"),
189 (DeepSeekV31ToolParser, "deepseek_v3_1"),
188 - (DeepSeekV32ToolParser, "deepseek_v3_2"),
189 - (DeepSeekV4ToolParser, "deepseek_v4"),
190 + (DeepSeekV32EngineToolParser, "deepseek_v3_2"),
191 + (DeepSeekV4EngineToolParser, "deepseek_v4"),
192 (Glm47MoeModelToolParser, "glm_4_7"),
193 (Hermes2ProToolParser, "hermes"),
194 (KimiK2ToolParser, "kimi"),
vllm/parser/deepseek_v32.py new
+131
@@ -0,0 +1,131 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""DeepSeek V3.2 parser: DSML tool calls with ``function_calls`` wrapper.
4 +
5 +DeepSeek V3.2 output format::
6 +
7 + <｜DSML｜function_calls>
8 + <｜DSML｜invoke name="func_name">
9 + <｜DSML｜parameter name="location" string="true">杭州</｜DSML｜parameter>
10 + <｜DSML｜parameter name="count" string="false">5</｜DSML｜parameter>
11 + </｜DSML｜invoke>
12 + </｜DSML｜function_calls>
13 +
14 +This is identical to DeepSeek V4 except for the outer wrapper
15 +(``function_calls`` instead of ``tool_calls``) and the absence of
16 +``<think>``/``</think>`` reasoning tags.
17 +"""
18 +
19 +from __future__ import annotations
20 +
21 +import functools
22 +from typing import TYPE_CHECKING
23 +
24 +from vllm.parser.deepseek_v4 import (
25 + DSML_INVOKE_END,
26 + DSML_INVOKE_NAME_END,
27 + DSML_INVOKE_PREFIX,
28 + DSML_PARAM_CLOSE,
29 + _dsml_arg_converter,
30 + _unwrap_wrapper_args,
31 +)
32 +from vllm.parser.engine.events import EventType
33 +from vllm.parser.engine.parser_engine import ParserEngine
34 +from vllm.parser.engine.parser_engine_config import (
35 + ParserEngineConfig,
36 + ParserState,
37 + Transition,
38 +)
39 +
40 +if TYPE_CHECKING:
41 + from vllm.tokenizers import TokenizerLike
42 + from vllm.tool_parsers.abstract_tool_parser import Tool
43 +
44 +_DSML = "｜DSML｜"
45 +
46 +DSML_FUNC_START = f"<{_DSML}function_calls>"
47 +DSML_FUNC_END = f"</{_DSML}function_calls>"
48 +
49 +
50 +@functools.cache
51 +def deepseek_v32_config() -> ParserEngineConfig:
52 + return ParserEngineConfig(
53 + name="deepseek_v32",
54 + initial_state=ParserState.CONTENT,
55 + terminals={
56 + "TOOL_START": DSML_FUNC_START,
57 + "TOOL_END": DSML_FUNC_END,
58 + "INVOKE_PREFIX": DSML_INVOKE_PREFIX,
59 + "INVOKE_NAME_END": DSML_INVOKE_NAME_END,
60 + "INVOKE_END": DSML_INVOKE_END,
61 + "PARAM_CLOSE": DSML_PARAM_CLOSE,
62 + },
63 + token_id_terminals={
64 + "TOOL_START": DSML_FUNC_START,
65 + "TOOL_END": DSML_FUNC_END,
66 + },
67 + transitions={
68 + (ParserState.CONTENT, "TOOL_START"): Transition(
69 + ParserState.TOOL_PREAMBLE,
70 + (),
71 + ),
72 + (ParserState.TOOL_PREAMBLE, "INVOKE_PREFIX"): Transition(
73 + ParserState.TOOL_NAME,
74 + (EventType.TOOL_CALL_START,),
75 + ),
76 + (ParserState.TOOL_NAME, "INVOKE_NAME_END"): Transition(
77 + ParserState.TOOL_ARGS,
78 + (),
79 + ),
80 + (ParserState.TOOL_ARGS, "INVOKE_END"): Transition(
81 + ParserState.TOOL_BETWEEN,
82 + (EventType.TOOL_CALL_END,),
83 + ),
84 + (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
85 + ParserState.CONTENT,
86 + (EventType.TOOL_CALL_END,),
87 + ),
88 + # Parallel tool calls
89 + (ParserState.TOOL_BETWEEN, "INVOKE_PREFIX"): Transition(
90 + ParserState.TOOL_NAME,
91 + (EventType.TOOL_CALL_START,),
92 + ),
93 + (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
94 + ParserState.CONTENT,
95 + (),
96 + ),
97 + },
98 + content_events={
99 + ParserState.CONTENT: EventType.TEXT_CHUNK,
100 + ParserState.TOOL_NAME: EventType.TOOL_NAME,
101 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
102 + },
103 + arg_converter=_dsml_arg_converter,
104 + arg_structural_chars=frozenset(">"),
105 + strip_content_whitespace_with_tools=False,
106 + tool_args_json=False,
107 + )
108 +
109 +
110 +class DeepSeekV32Parser(ParserEngine):
111 + def __init__(
112 + self,
113 + tokenizer: TokenizerLike,
114 + tools: list[Tool] | None = None,
115 + **kwargs,
116 + ) -> None:
117 + kwargs.pop("chat_template_kwargs", None)
118 + super().__init__(
119 + tokenizer,
120 + tools,
121 + parser_engine_config=deepseek_v32_config(),
122 + **kwargs,
123 + )
124 + self._arg_converter = self._convert_args
125 +
126 + def _convert_args(self, raw_args: str, partial: bool) -> str:
127 + result = _dsml_arg_converter(raw_args, partial)
128 + if not self._tools:
129 + return result
130 + func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None)
131 + return _unwrap_wrapper_args(result, self._tools, func_name)
vllm/parser/deepseek_v4.py new
+237
@@ -0,0 +1,237 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""DeepSeek V4 parser: ``<think>``/``</think>``
4 +reasoning plus DSML tool calls in a single state machine.
5 +
6 +DeepSeek V4 output format::
7 +
8 + <think>
9 + ...reasoning...
10 + </think>
11 + <｜DSML｜tool_calls>
12 + <｜DSML｜invoke name="func_name">
13 + <｜DSML｜parameter name="location" string="true">杭州</｜DSML｜parameter>
14 + <｜DSML｜parameter name="count" string="false">5</｜DSML｜parameter>
15 + </｜DSML｜invoke>
16 + </｜DSML｜tool_calls>
17 +"""
18 +
19 +from __future__ import annotations
20 +
21 +import contextlib
22 +import functools
23 +import json
24 +from typing import TYPE_CHECKING
25 +
26 +import regex as re
27 +
28 +from vllm.parser.engine.events import EventType
29 +from vllm.parser.engine.parser_engine import ParserEngine
30 +from vllm.parser.engine.parser_engine_config import (
31 + ParserEngineConfig,
32 + ParserState,
33 + Transition,
34 +)
35 +from vllm.tool_parsers.utils import find_tool_properties
36 +
37 +if TYPE_CHECKING:
38 + from vllm.tokenizers import TokenizerLike
39 + from vllm.tool_parsers.abstract_tool_parser import Tool
40 +
41 +_DSML = "｜DSML｜"
42 +
43 +DSML_THINK_START = "<think>"
44 +DSML_THINK_END = "</think>"
45 +DSML_TOOL_START = f"<{_DSML}tool_calls>"
46 +DSML_TOOL_END = f"</{_DSML}tool_calls>"
47 +DSML_INVOKE_PREFIX = f'<{_DSML}invoke name="'
48 +DSML_INVOKE_NAME_END = '">'
49 +DSML_INVOKE_END = f"</{_DSML}invoke>"
50 +DSML_PARAM_CLOSE = f"</{_DSML}parameter>"
51 +
52 +_ESCAPED_DSML = re.escape(_DSML)
53 +_PARAM_RE = re.compile(
54 + rf'<{_ESCAPED_DSML}parameter\s+name="([^"]+)"\s+string="(true|false)">'
55 + rf"(.*?)</{_ESCAPED_DSML}parameter>",
56 + re.DOTALL,
57 +)
58 +_PARTIAL_PARAM_RE = re.compile(
59 + rf'<{_ESCAPED_DSML}parameter\s+name="([^"]+)"\s+string="(true|false)">'
60 + rf"(.*)$",
61 + re.DOTALL,
62 +)
63 +
64 +
65 +def _dsml_arg_converter(raw_args: str, partial: bool) -> str:
66 + params: dict[str, object] = {}
67 +
68 + last_end = 0
69 + for m in _PARAM_RE.finditer(raw_args):
70 + name, is_str, value = m.group(1), m.group(2), m.group(3)
71 + if is_str == "true":
72 + params[name] = value
73 + else:
74 + try:
75 + params[name] = json.loads(value)
76 + except (json.JSONDecodeError, ValueError):
77 + params[name] = value
78 + last_end = m.end()
79 +
80 + if partial:
81 + pm = _PARTIAL_PARAM_RE.search(raw_args, last_end)
82 + if pm:
83 + name, is_str, value = pm.group(1), pm.group(2), pm.group(3)
84 + if is_str == "true":
85 + params[name] = value
86 + else:
87 + with contextlib.suppress(json.JSONDecodeError, ValueError):
88 + params[name] = json.loads(value)
89 +
90 + return json.dumps(params, ensure_ascii=False)
91 +
92 +
93 +def _unwrap_wrapper_args(
94 + args_json: str,
95 + tools: list[Tool] | None,
96 + func_name: str | None,
97 +) -> str:
98 + if not tools or not func_name:
99 + return args_json
100 + try:
101 + args = json.loads(args_json)
102 + except (json.JSONDecodeError, ValueError):
103 + return args_json
104 + if not isinstance(args, dict):
105 + return args_json
106 + properties = find_tool_properties(tools, func_name)
107 + if not properties:
108 + return args_json
109 + allowed = set(properties.keys())
110 + for wrapper in ("arguments", "input"):
111 + if set(args.keys()) != {wrapper} or wrapper in allowed:
112 + continue
113 + inner = args[wrapper]
114 + if isinstance(inner, str):
115 + try:
116 + inner = json.loads(inner)
117 + except json.JSONDecodeError:
118 + return args_json
119 + if isinstance(inner, dict) and set(inner.keys()).issubset(allowed):
120 + return json.dumps(inner, ensure_ascii=False)
121 + return args_json
122 +
123 +
124 +@functools.cache
125 +def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig:
126 + return ParserEngineConfig(
127 + name="deepseek_v4",
128 + initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
129 + terminals={
130 + "THINK_START": DSML_THINK_START,
131 + "THINK_END": DSML_THINK_END,
132 + "TOOL_START": DSML_TOOL_START,
133 + "TOOL_END": DSML_TOOL_END,
134 + "INVOKE_PREFIX": DSML_INVOKE_PREFIX,
135 + "INVOKE_NAME_END": DSML_INVOKE_NAME_END,
136 + "INVOKE_END": DSML_INVOKE_END,
137 + "PARAM_CLOSE": DSML_PARAM_CLOSE,
138 + },
139 + token_id_terminals={
140 + "THINK_START": DSML_THINK_START,
141 + "THINK_END": DSML_THINK_END,
142 + "TOOL_START": DSML_TOOL_START,
143 + "TOOL_END": DSML_TOOL_END,
144 + },
145 + transitions={
146 + (ParserState.CONTENT, "THINK_START"): Transition(
147 + ParserState.REASONING,
148 + (EventType.REASONING_START,),
149 + ),
150 + # Absorb a bare </think> with no prior <think>
151 + (ParserState.CONTENT, "THINK_END"): Transition(
152 + ParserState.CONTENT,
153 + (),
154 + ),
155 + # Absorb a duplicate <think> while already reasoning
156 + (ParserState.REASONING, "THINK_START"): Transition(
157 + ParserState.REASONING,
158 + (),
159 + ),
160 + (ParserState.REASONING, "THINK_END"): Transition(
161 + ParserState.CONTENT,
162 + (EventType.REASONING_END,),
163 + ),
164 + # Tool call beginning while still inside <think>
165 + (ParserState.REASONING, "TOOL_START"): Transition(
166 + ParserState.TOOL_PREAMBLE,
167 + (EventType.REASONING_END,),
168 + ),
169 + (ParserState.CONTENT, "TOOL_START"): Transition(
170 + ParserState.TOOL_PREAMBLE,
171 + (),
172 + ),
173 + (ParserState.TOOL_PREAMBLE, "INVOKE_PREFIX"): Transition(
174 + ParserState.TOOL_NAME,
175 + (EventType.TOOL_CALL_START,),
176 + ),
177 + (ParserState.TOOL_NAME, "INVOKE_NAME_END"): Transition(
178 + ParserState.TOOL_ARGS,
179 + (),
180 + ),
181 + (ParserState.TOOL_ARGS, "INVOKE_END"): Transition(
182 + ParserState.TOOL_BETWEEN,
183 + (EventType.TOOL_CALL_END,),
184 + ),
185 + (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
186 + ParserState.CONTENT,
187 + (EventType.TOOL_CALL_END,),
188 + ),
189 + # Parallel tool calls
190 + (ParserState.TOOL_BETWEEN, "INVOKE_PREFIX"): Transition(
191 + ParserState.TOOL_NAME,
192 + (EventType.TOOL_CALL_START,),
193 + ),
194 + (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
195 + ParserState.CONTENT,
196 + (),
197 + ),
198 + },
199 + content_events={
200 + ParserState.CONTENT: EventType.TEXT_CHUNK,
201 + ParserState.REASONING: EventType.REASONING_CHUNK,
202 + ParserState.TOOL_NAME: EventType.TOOL_NAME,
203 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
204 + },
205 + arg_converter=_dsml_arg_converter,
206 + arg_structural_chars=frozenset(">"),
207 + strip_content_whitespace_with_tools=False,
208 + tool_args_json=False,
209 + )
210 +
211 +
212 +class DeepSeekV4Parser(ParserEngine):
213 + def __init__(
214 + self,
215 + tokenizer: TokenizerLike,
216 + tools: list[Tool] | None = None,
217 + **kwargs,
218 + ) -> None:
219 + chat_kwargs = kwargs.pop("chat_template_kwargs", None) or {}
220 + thinking = (
221 + bool(chat_kwargs.get("thinking") or chat_kwargs.get("enable_thinking"))
222 + and chat_kwargs.get("reasoning_effort") != "none"
223 + )
224 + super().__init__(
225 + tokenizer,
226 + tools,
227 + parser_engine_config=deepseek_v4_config(thinking=thinking),
228 + **kwargs,
229 + )
230 + self._arg_converter = self._convert_args
231 +
232 + def _convert_args(self, raw_args: str, partial: bool) -> str:
233 + result = _dsml_arg_converter(raw_args, partial)
234 + if not self._tools:
235 + return result
236 + func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None)
237 + return _unwrap_wrapper_args(result, self._tools, func_name)
vllm/parser/engine/adapters.py
+2 -1
@@ -111,7 +111,8 @@ class ParserEngineReasoningAdapter(ReasoningParser):
111 return self._parser_engine.reasoning_ended
112
113 def finish_streaming(self) -> DeltaMessage | None:
114 - return self._parser_engine.finish_streaming()
114 + with self._skip_tool_parsing():
115 + return self._parser_engine.finish_streaming()
116
117 def get_streaming_fallback_content(
118 self,
vllm/parser/engine/parser_engine.py
+7 -3
@@ -108,7 +108,11 @@ class ParserEngine(Parser):
108 parser_engine_config, tokenizer, vocab=self.vocab
109 )
110
111 - self._reasoning_ended: bool = False
111 + self._has_reasoning = (
112 + "THINK_END" in parser_engine_config.token_id_terminals
113 + or parser_engine_config.initial_state == ParserState.REASONING
114 + )
115 + self._reasoning_ended: bool = not self._has_reasoning
116 self._streaming_initialized: bool = False
117 self._prompt_streaming_prepared: bool = False
118
@@ -188,7 +192,7 @@ class ParserEngine(Parser):
192
193 def _reset(self, initial_state: ParserState | None = None) -> None:
194 self._engine.reset(initial_state=initial_state)
191 - self._reasoning_ended = False
195 + self._reasoning_ended = not self._has_reasoning
196 self._tool_slots.clear()
197 self._deferred_content = ""
198 self._deferred_reasoning = ""
@@ -255,7 +259,7 @@ class ParserEngine(Parser):
259 types = extract_types_from_schema(schema)
260 as_str = json.dumps(value, ensure_ascii=False)
261 coerced = coerce_to_schema_type(as_str, types)
258 - if coerced != value:
262 + if type(coerced) is not type(value) or coerced != value:
263 return coerced, True
264 return value, False
265
vllm/parser/engine/registered_adapters.py
+12
@@ -7,6 +7,8 @@ names so that :class:`ReasoningParserManager` and
7 :class:`ToolParserManager` can load them lazily.
8 """
9
10 +from vllm.parser.deepseek_v4 import DeepSeekV4Parser
11 +from vllm.parser.deepseek_v32 import DeepSeekV32Parser
12 from vllm.parser.engine.adapters import make_adapters
13 from vllm.parser.gemma4 import Gemma4Parser
14 from vllm.parser.glm47_moe import Glm47MoeParser
@@ -16,6 +18,16 @@ from vllm.parser.nemotron_v3 import NemotronV3Parser
18 from vllm.parser.qwen3 import Qwen3Parser
19 from vllm.parser.seed_oss import SeedOssParser
20
21 +(
22 + DeepSeekV32ParserReasoningAdapter,
23 + DeepSeekV32ParserToolAdapter,
24 +) = make_adapters(DeepSeekV32Parser)
25 +
26 +(
27 + DeepSeekV4ParserReasoningAdapter,
28 + DeepSeekV4ParserToolAdapter,
29 +) = make_adapters(DeepSeekV4Parser)
30 +
31 (
32 MinimaxM2ParserReasoningAdapter,
33 MinimaxM2ParserToolAdapter,
vllm/parser/engine/streaming_parser_engine.py
+1 -2
@@ -22,14 +22,13 @@ from vllm.parser.engine.parser_engine_config import (
22 Transition,
23 )
24 from vllm.parser.engine.token_id_scanner import (
25 + DROP_TERMINAL,
26 LexerInput,
27 PreLexedTerminal,
28 TextChunk,
29 TokenIDScanner,
30 )
31
31 -DROP_TERMINAL = "__DROP__"
32 -
32
33 @dataclass(slots=True)
34 class _DropInfo:
vllm/parser/engine/token_id_scanner.py
+11 -1
@@ -8,6 +8,8 @@ from __future__ import annotations
8 from collections.abc import Sequence
9 from dataclasses import dataclass
10
11 +DROP_TERMINAL = "__DROP__"
12 +
13
14 @dataclass(slots=True)
15 class TextChunk:
@@ -277,7 +279,15 @@ class TokenIDScanner:
279 consumed = pos + len(anchor.text)
280 else:
281 has_later_valid = any(p >= 0 for p in positions[i + 1 :])
280 - if not has_later_valid and consumed < len(delta_text):
282 + # DROP anchors (EOS, etc.) may have text that never
283 + # arrives in delta_text (stripped by detokenizer).
284 + # Don't defer remaining content waiting for text
285 + # that will never come.
286 + if (
287 + not has_later_valid
288 + and consumed < len(delta_text)
289 + and anchor.terminal != DROP_TERMINAL
290 + ):
291 self._deferred_post_text += delta_text[consumed:]
292 consumed = len(delta_text)
293 self._deferred_terminals.append(anchor)
vllm/reasoning/__init__.py
+2 -2
@@ -29,8 +29,8 @@ _REASONING_PARSERS_TO_REGISTER = {
29 "DeepSeekV3ReasoningParser",
30 ),
31 "deepseek_v4": (
32 - "deepseek_v3_reasoning_parser",
33 - "DeepSeekV3ReasoningParser",
32 + "deepseek_v4_engine_reasoning_parser",
33 + "DeepSeekV4ParserReasoningAdapter",
34 ),
35 "poolside_v1": (
36 "poolside_v1_reasoning_parser",
vllm/reasoning/deepseek_v4_engine_reasoning_parser.py new
+6
@@ -0,0 +1,6 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.engine.registered_adapters import DeepSeekV4ParserReasoningAdapter
5 +
6 +__all__ = ["DeepSeekV4ParserReasoningAdapter"]
vllm/tool_parsers/__init__.py
+4 -4
@@ -31,12 +31,12 @@ _TOOL_PARSERS_TO_REGISTER = {
31 "DeepSeekV31ToolParser",
32 ),
33 "deepseek_v32": (
34 - "deepseekv32_tool_parser",
35 - "DeepSeekV32ToolParser",
34 + "deepseekv32_engine_tool_parser",
35 + "DeepSeekV32EngineToolParser",
36 ),
37 "deepseek_v4": (
38 - "deepseekv4_tool_parser",
39 - "DeepSeekV4ToolParser",
38 + "deepseekv4_engine_tool_parser",
39 + "DeepSeekV4EngineToolParser",
40 ),
41 "cohere_command3": (
42 "cohere_command_tool_parser",
vllm/tool_parsers/deepseekv32_engine_tool_parser.py new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.engine.registered_adapters import DeepSeekV32ParserToolAdapter
5 +
6 +
7 +class DeepSeekV32EngineToolParser(DeepSeekV32ParserToolAdapter): # type: ignore[valid-type, misc]
8 + structural_tag_model = "deepseek_v3_2"
vllm/tool_parsers/deepseekv32_tool_parser.py deleted
-562
@@ -1,562 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import json
5 -import uuid
6 -from collections.abc import Sequence
7 -from typing import Any, Literal
8 -
9 -import regex as re
10 -
11 -from vllm.entrypoints.openai.chat_completion.protocol import (
12 - ChatCompletionRequest,
13 -)
14 -from vllm.entrypoints.openai.engine.protocol import (
15 - DeltaFunctionCall,
16 - DeltaMessage,
17 - DeltaToolCall,
18 - ExtractedToolCallInformation,
19 - FunctionCall,
20 - ToolCall,
21 -)
22 -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
23 -from vllm.logger import init_logger
24 -from vllm.tokenizers import TokenizerLike
25 -from vllm.tool_parsers.abstract_tool_parser import (
26 - Tool,
27 - ToolParser,
28 -)
29 -from vllm.tool_parsers.utils import (
30 - coerce_to_schema_type,
31 - extract_types_from_schema,
32 - find_tool_properties,
33 - partial_tag_overlap,
34 -)
35 -
36 -logger = init_logger(__name__)
37 -
38 -
39 -class DeepSeekV32ToolParser(ToolParser):
40 - """
41 - example tool call content:
42 - <｜DSML｜function_calls>
43 - <｜DSML｜invoke name="get_weather">
44 - <｜DSML｜parameter name="location" string="true">杭州</｜DSML｜parameter>
45 - <｜DSML｜parameter name="date" string="true">2024-01-16</｜DSML｜parameter>
46 - </｜DSML｜invoke>
47 - <｜DSML｜invoke name="get_weather">
48 - <｜DSML｜parameter name="location" string="true">北京</｜DSML｜parameter>
49 - <｜DSML｜parameter name="date" string="true">2024-01-16</｜DSML｜parameter>
50 - </｜DSML｜invoke>
51 - </｜DSML｜function_calls>
52 - """
53 -
54 - tool_call_start_token: str = "<｜DSML｜function_calls>"
55 - tool_call_end_token: str = "</｜DSML｜function_calls>"
56 - structural_tag_model = "deepseek_v3_2"
57 -
58 - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
59 - super().__init__(tokenizer, tools)
60 -
61 - self.prev_tool_call_arr: list[dict] = []
62 -
63 - # Streaming state
64 - self.current_tool_index: int = 0
65 - self._sent_content_idx: int = 0
66 - self._buffer: str = ""
67 - self._in_tool_calls: bool = False
68 - self._active_tool_index: int | None = None
69 - self._active_tool_name: str | None = None
70 - self._active_param_name: str | None = None
71 - self._active_param_string_attr: str | None = None
72 - self._active_param_mode: str | None = None
73 - self._active_param_parts: list[str] = []
74 - self._args_started: list[bool] = []
75 -
76 - # Regex patterns for complete parsing
77 - self.tool_call_complete_regex = re.compile(
78 - re.escape(self.tool_call_start_token)
79 - + r"(.*?)"
80 - + re.escape(self.tool_call_end_token),
81 - re.DOTALL,
82 - )
83 - self.invoke_complete_regex = re.compile(
84 - r'<｜DSML｜invoke\s+name="([^"]+)"\s*>(.*?)</｜DSML｜invoke>', re.DOTALL
85 - )
86 - self.parameter_complete_regex = re.compile(
87 - r'<｜DSML｜parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>(.*?)</｜DSML｜parameter>',
88 - re.DOTALL,
89 - )
90 - self.invoke_start_regex = re.compile(r'<｜DSML｜invoke\s+name="([^"]+)"\s*>')
91 - self.parameter_start_regex = re.compile(
92 - r'<｜DSML｜parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>'
93 - )
94 -
95 - if not self.model_tokenizer:
96 - raise ValueError(
97 - "The model tokenizer must be passed to the ToolParser "
98 - "constructor during construction."
99 - )
100 -
101 - logger.debug(
102 - "vLLM Successfully import tool parser %s !", self.__class__.__name__
103 - )
104 -
105 - def adjust_request(
106 - self, request: ChatCompletionRequest | ResponsesRequest
107 - ) -> ChatCompletionRequest | ResponsesRequest:
108 - request = super().adjust_request(request)
109 - if request.tools and request.tool_choice != "none":
110 - # Ensure tool call tokens
111 - # (e.g. <｜DSML｜function_calls>, </｜DSML｜function_calls>)
112 - # are not skippedduring decoding.
113 - # Even though they are not marked as special tokens,
114 - # setting skip_special_tokens=False ensures proper handling in
115 - # transformers 5.x where decoding behavior may have changed.
116 - request.skip_special_tokens = False
117 - return request
118 -
119 - def _generate_tool_call_id(self) -> str:
120 - """Generate a unique tool call ID."""
121 - return f"call_{uuid.uuid4().hex[:24]}"
122 -
123 - def _parse_invoke_params(self, invoke_str: str) -> dict[str, tuple[str, str]]:
124 - param_dict: dict[str, tuple[str, str]] = {}
125 - for param_name, string_attr, param_val in self.parameter_complete_regex.findall(
126 - invoke_str
127 - ):
128 - param_dict[param_name] = (param_val, string_attr)
129 - return param_dict
130 -
131 - @staticmethod
132 - def _repair_param_dict(
133 - param_dict: dict[str, Any],
134 - param_config: dict[str, Any],
135 - ) -> dict[str, Any]:
136 - """Unwrap single 'arguments' / 'input' wrappers when the wrapper
137 - is not part of the requested tool schema and the wrapped object
138 - matches the schema fields."""
139 - allowed = set(param_config.keys())
140 - for wrapper in ("arguments", "input"):
141 - if set(param_dict.keys()) != {wrapper} or wrapper in allowed:
142 - continue
143 - inner = param_dict[wrapper]
144 - if isinstance(inner, str):
145 - try:
146 - inner = json.loads(inner)
147 - except json.JSONDecodeError:
148 - return param_dict
149 - if isinstance(inner, dict) and set(inner.keys()).issubset(allowed):
150 - return inner
151 - return param_dict
152 -
153 - def _convert_params_with_schema(
154 - self,
155 - function_name: str,
156 - param_dict: dict[str, tuple[str, str]],
157 - ) -> dict[str, Any]:
158 - """Convert raw string param values using the tool schema types."""
159 - param_config = find_tool_properties(self.tools, function_name)
160 -
161 - converted: dict[str, Any] = {}
162 - for name, (value, string_attr) in param_dict.items():
163 - if string_attr == "true":
164 - converted[name] = value
165 - continue
166 -
167 - param_types = extract_types_from_schema(param_config.get(name, {}))
168 - converted[name] = coerce_to_schema_type(value, param_types)
169 - return self._repair_param_dict(converted, param_config)
170 -
171 - def _get_param_config(self, function_name: str | None) -> dict[str, Any]:
172 - if not function_name or not self.tools:
173 - return {}
174 - return find_tool_properties(self.tools, function_name)
175 -
176 - @staticmethod
177 - def _json_escape_string_content(text: str) -> str:
178 - return json.dumps(text, ensure_ascii=False)[1:-1]
179 -
180 - def extract_tool_calls(
181 - self,
182 - model_output: str,
183 - request: ChatCompletionRequest,
184 - ) -> ExtractedToolCallInformation:
185 - """Extract tool calls from complete model output (non-streaming)."""
186 - # Quick check
187 - if self.tool_call_start_token not in model_output:
188 - return ExtractedToolCallInformation(
189 - tools_called=False, tool_calls=[], content=model_output
190 - )
191 -
192 - try:
193 - tool_calls = []
194 -
195 - # Find all complete tool_call blocks
196 - for tool_call_match in self.tool_call_complete_regex.findall(model_output):
197 - # Find all invokes within this tool_call
198 - for invoke_name, invoke_content in self.invoke_complete_regex.findall(
199 - tool_call_match
200 - ):
201 - param_dict = self._parse_invoke_params(invoke_content)
202 - params = self._convert_params_with_schema(invoke_name, param_dict)
203 - tool_calls.append(
204 - ToolCall(
205 - type="function",
206 - function=FunctionCall(
207 - name=invoke_name,
208 - arguments=json.dumps(params, ensure_ascii=False),
209 - ),
210 - )
211 - )
212 -
213 - if not tool_calls:
214 - return ExtractedToolCallInformation(
215 - tools_called=False, tool_calls=[], content=model_output
216 - )
217 -
218 - # Extract content before first tool call
219 - first_tool_idx = model_output.find(self.tool_call_start_token)
220 - content = model_output[:first_tool_idx] if first_tool_idx > 0 else None
221 -
222 - return ExtractedToolCallInformation(
223 - tools_called=True, tool_calls=tool_calls, content=content
224 - )
225 -
226 - except Exception:
227 - logger.exception("Error extracting tool calls")
228 - return ExtractedToolCallInformation(
229 - tools_called=False, tool_calls=[], content=model_output
230 - )
231 -
232 - def _reset_streaming_state(self):
233 - """Reset all streaming state."""
234 - self.current_tool_index = 0
235 - self._sent_content_idx = 0
236 - self._buffer = ""
237 - self._in_tool_calls = False
238 - self._active_tool_index = None
239 - self._active_tool_name = None
240 - self._active_param_name = None
241 - self._active_param_string_attr = None
242 - self._active_param_mode = None
243 - self._active_param_parts.clear()
244 - self.prev_tool_call_arr.clear()
245 - self.streamed_args_for_tool.clear()
246 - self._args_started.clear()
247 -
248 - def _add_tool_call_delta(
249 - self,
250 - tool_call_deltas: dict[int, DeltaToolCall],
251 - index: int,
252 - *,
253 - call_id: str | None = None,
254 - call_type: Literal["function"] | None = None,
255 - name: str | None = None,
256 - arguments: str | None = None,
257 - ) -> None:
258 - if arguments:
259 - self.streamed_args_for_tool[index] += arguments
260 -
261 - if index not in tool_call_deltas:
262 - tool_call_deltas[index] = DeltaToolCall(
263 - index=index,
264 - id=call_id,
265 - type=call_type,
266 - function=DeltaFunctionCall(name=name, arguments=arguments),
267 - )
268 - return
269 -
270 - delta = tool_call_deltas[index]
271 - if call_id is not None:
272 - delta.id = call_id
273 - if call_type is not None:
274 - delta.type = call_type
275 - if delta.function is None:
276 - delta.function = DeltaFunctionCall()
277 - if name is not None:
278 - delta.function.name = name
279 - if arguments is not None:
280 - delta.function.arguments = (delta.function.arguments or "") + arguments
281 -
282 - def _begin_streaming_tool_call(
283 - self,
284 - name: str,
285 - tool_call_deltas: dict[int, DeltaToolCall],
286 - ) -> None:
287 - index = self.current_tool_index
288 - self.current_tool_index += 1
289 - self._active_tool_index = index
290 - self._active_tool_name = name
291 - self.prev_tool_call_arr.append({"name": name, "arguments": {}})
292 - self.streamed_args_for_tool.append("")
293 - self._args_started.append(False)
294 - self._add_tool_call_delta(
295 - tool_call_deltas,
296 - index,
297 - call_id=self._generate_tool_call_id(),
298 - call_type="function",
299 - name=name,
300 - arguments="",
301 - )
302 -
303 - def _append_param_prefix(
304 - self,
305 - tool_call_deltas: dict[int, DeltaToolCall],
306 - index: int,
307 - key: str,
308 - *,
309 - as_string: bool,
310 - ) -> None:
311 - prefix = "{" if not self._args_started[index] else ","
312 - self._args_started[index] = True
313 - arguments = prefix + json.dumps(key, ensure_ascii=False) + ":"
314 - if as_string:
315 - arguments += '"'
316 - self._add_tool_call_delta(tool_call_deltas, index, arguments=arguments)
317 -
318 - def _append_json_param_value(
319 - self,
320 - tool_call_deltas: dict[int, DeltaToolCall],
321 - index: int,
322 - key: str,
323 - value: Any,
324 - ) -> None:
325 - self._append_param_prefix(tool_call_deltas, index, key, as_string=False)
326 - self._add_tool_call_delta(
327 - tool_call_deltas,
328 - index,
329 - arguments=json.dumps(value, ensure_ascii=False),
330 - )
331 -
332 - def _param_types_for_name(self, name: str) -> list[str]:
333 - param_config = self._get_param_config(self._active_tool_name)
334 - if name in param_config and isinstance(param_config[name], dict):
335 - return extract_types_from_schema(param_config[name])
336 - return ["string"]
337 -
338 - @staticmethod
339 - def _can_stream_raw_param(param_types: list[str]) -> bool:
340 - # Scalars and unions need the complete value so streaming and
341 - # non-streaming share the same coercion fallback behavior.
342 - return set(param_types).issubset({"object", "array"})
343 -
344 - def _should_buffer_wrapper_param(self, name: str) -> bool:
345 - if (
346 - self._active_tool_index is None
347 - or self._args_started[self._active_tool_index]
348 - ):
349 - return False
350 - param_config = self._get_param_config(self._active_tool_name)
351 - return bool(
352 - param_config and name in ("arguments", "input") and name not in param_config
353 - )
354 -
355 - def _finish_buffered_param(
356 - self,
357 - tool_call_deltas: dict[int, DeltaToolCall],
358 - index: int,
359 - ) -> None:
360 - assert self._active_param_name is not None
361 - assert self._active_param_string_attr is not None
362 - raw_value = "".join(self._active_param_parts)
363 - converted = self._convert_params_with_schema(
364 - self._active_tool_name or "",
365 - {self._active_param_name: (raw_value, self._active_param_string_attr)},
366 - )
367 - for key, value in converted.items():
368 - self._append_json_param_value(tool_call_deltas, index, key, value)
369 -
370 - def _close_streaming_tool_call(
371 - self,
372 - tool_call_deltas: dict[int, DeltaToolCall],
373 - ) -> None:
374 - index = self._active_tool_index
375 - if index is None:
376 - return
377 -
378 - suffix = "}" if self._args_started[index] else "{}"
379 - self._add_tool_call_delta(tool_call_deltas, index, arguments=suffix)
380 - try:
381 - self.prev_tool_call_arr[index] = {
382 - "name": self._active_tool_name,
383 - "arguments": json.loads(self.streamed_args_for_tool[index]),
384 - }
385 - except (json.JSONDecodeError, IndexError):
386 - logger.exception("Failed to finalize DeepSeek DSML streaming tool call")
387 -
388 - self._active_tool_index = None
389 - self._active_tool_name = None
390 - self._active_param_name = None
391 - self._active_param_string_attr = None
392 - self._active_param_mode = None
393 - self._active_param_parts.clear()
394 -
395 - def _process_streaming_buffer(
396 - self,
397 - content_parts: list[str],
398 - tool_call_deltas: dict[int, DeltaToolCall],
399 - ) -> None:
400 - parameter_end_token = "</｜DSML｜parameter>"
401 - invoke_end_token = "</｜DSML｜invoke>"
402 -
403 - while True:
404 - if not self._in_tool_calls:
405 - start_idx = self._buffer.find(self.tool_call_start_token)
406 - if start_idx == -1:
407 - overlap = partial_tag_overlap(
408 - self._buffer, self.tool_call_start_token
409 - )
410 - sendable_idx = len(self._buffer) - overlap
411 - if sendable_idx > 0:
412 - content_parts.append(self._buffer[:sendable_idx])
413 - self._buffer = self._buffer[sendable_idx:]
414 - return
415 -
416 - if start_idx > 0:
417 - content_parts.append(self._buffer[:start_idx])
418 - self._buffer = self._buffer[start_idx:]
419 - continue
420 -
421 - self._buffer = self._buffer[len(self.tool_call_start_token) :]
422 - self._in_tool_calls = True
423 - continue
424 -
425 - if self._active_tool_index is None:
426 - stripped_len = len(self._buffer) - len(self._buffer.lstrip())
427 - if stripped_len:
428 - self._buffer = self._buffer[stripped_len:]
429 - continue
430 -
431 - if self._buffer.startswith(self.tool_call_end_token):
432 - self._buffer = self._buffer[len(self.tool_call_end_token) :]
433 - self._in_tool_calls = False
434 - continue
435 -
436 - match = self.invoke_start_regex.match(self._buffer)
437 - if match is None:
438 - return
439 -
440 - self._buffer = self._buffer[match.end() :]
441 - self._begin_streaming_tool_call(match.group(1), tool_call_deltas)
442 - continue
443 -
444 - index = self._active_tool_index
445 -
446 - if self._active_param_mode is not None:
447 - end_pos = self._buffer.find(parameter_end_token)
448 - if end_pos != -1:
449 - raw_content = self._buffer[:end_pos]
450 - self._buffer = self._buffer[end_pos + len(parameter_end_token) :]
451 - if self._active_param_mode in ("wrapper", "buffered"):
452 - self._active_param_parts.append(raw_content)
453 - self._finish_buffered_param(tool_call_deltas, index)
454 - elif self._active_param_mode == "string":
455 - arguments = self._json_escape_string_content(raw_content) + '"'
456 - self._add_tool_call_delta(
457 - tool_call_deltas, index, arguments=arguments
458 - )
459 - else:
460 - self._add_tool_call_delta(
461 - tool_call_deltas, index, arguments=raw_content
462 - )
463 -
464 - self._active_param_name = None
465 - self._active_param_string_attr = None
466 - self._active_param_mode = None
467 - self._active_param_parts.clear()
468 - continue
469 -
470 - overlap = partial_tag_overlap(self._buffer, parameter_end_token)
471 - safe_len = len(self._buffer) - overlap
472 - if safe_len > 0:
473 - raw_content = self._buffer[:safe_len]
474 - self._buffer = self._buffer[safe_len:]
475 - if self._active_param_mode in ("wrapper", "buffered"):
476 - self._active_param_parts.append(raw_content)
477 - elif self._active_param_mode == "string":
478 - self._add_tool_call_delta(
479 - tool_call_deltas,
480 - index,
481 - arguments=self._json_escape_string_content(raw_content),
482 - )
483 - else:
484 - self._add_tool_call_delta(
485 - tool_call_deltas, index, arguments=raw_content
486 - )
487 - return
488 -
489 - stripped_len = len(self._buffer) - len(self._buffer.lstrip())
490 - if stripped_len:
491 - self._buffer = self._buffer[stripped_len:]
492 - continue
493 -
494 - if self._buffer.startswith(invoke_end_token):
495 - self._buffer = self._buffer[len(invoke_end_token) :]
496 - self._close_streaming_tool_call(tool_call_deltas)
497 - continue
498 -
499 - match = self.parameter_start_regex.match(self._buffer)
500 - if match is None:
501 - return
502 -
503 - self._buffer = self._buffer[match.end() :]
504 - name = match.group(1)
505 - string_attr = match.group(2)
506 - self._active_param_name = name
507 - self._active_param_string_attr = string_attr
508 -
509 - if self._should_buffer_wrapper_param(name):
510 - self._active_param_mode = "wrapper"
511 - continue
512 -
513 - if string_attr == "true":
514 - self._append_param_prefix(tool_call_deltas, index, name, as_string=True)
515 - self._active_param_mode = "string"
516 - continue
517 -
518 - param_types = self._param_types_for_name(name)
519 - if not self._can_stream_raw_param(param_types):
520 - self._active_param_mode = "buffered"
521 - continue
522 -
523 - self._append_param_prefix(tool_call_deltas, index, name, as_string=False)
524 - self._active_param_mode = "raw"
525 -
526 - def extract_tool_calls_streaming(
527 - self,
528 - previous_text: str,
529 - current_text: str,
530 - delta_text: str,
531 - previous_token_ids: Sequence[int], # pylint: disable=unused-argument
532 - current_token_ids: Sequence[int], # pylint: disable=unused-argument
533 - delta_token_ids: Sequence[int],
534 - request: ChatCompletionRequest,
535 - ) -> DeltaMessage | None:
536 - """Extract tool calls from streaming model output.
537 -
538 - Buffers DSML markup while streaming tool-call metadata and argument
539 - JSON fragments as soon as they are complete enough to be valid deltas.
540 - """
541 -
542 - # First chunk of a new stream — reset state from prior request.
543 - if not previous_text:
544 - self._reset_streaming_state()
545 -
546 - self._buffer += delta_text
547 - content_parts: list[str] = []
548 - tool_call_deltas: dict[int, DeltaToolCall] = {}
549 - self._process_streaming_buffer(content_parts, tool_call_deltas)
550 -
551 - if content_parts or tool_call_deltas:
552 - content = "".join(content_parts) or None
553 - return DeltaMessage(
554 - content=content, tool_calls=list(tool_call_deltas.values())
555 - )
556 -
557 - # Empty delta with token ids means EOS or closing tag; return
558 - # non-None so the serving framework can finalize finish_reason.
559 - if not delta_text and delta_token_ids and self.prev_tool_call_arr:
560 - return DeltaMessage(content="")
561 -
562 - return None
vllm/tool_parsers/deepseekv4_engine_tool_parser.py new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.engine.registered_adapters import DeepSeekV4ParserToolAdapter
5 +
6 +
7 +class DeepSeekV4EngineToolParser(DeepSeekV4ParserToolAdapter): # type: ignore[valid-type, misc]
8 + structural_tag_model = "deepseek_v4"
vllm/tool_parsers/deepseekv4_tool_parser.py deleted
-17
@@ -1,17 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser
5 -
6 -
7 -class DeepSeekV4ToolParser(DeepSeekV32ToolParser):
8 - """
9 - DeepSeek V4 DSML tool parser.
10 -
11 - V4 keeps the V3.2 DSML invoke/parameter grammar, but wraps tool calls in
12 - ``<｜DSML｜tool_calls>`` instead of ``<｜DSML｜function_calls>``.
13 - """
14 -
15 - tool_call_start_token: str = "<｜DSML｜tool_calls>"
16 - tool_call_end_token: str = "</｜DSML｜tool_calls>"
17 - structural_tag_model = "deepseek_v4"