A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[Bugfix] Fix Qwen3 tool parser for Responses API tools (#38848)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Apr 7, 2026 at 22:08 UTC
9ea7d670d85b5667ffa72e33415e3a40f8aedc81
4 files changed
+99
-113
tests/tool_parsers/test_qwen3coder_tool_parser.py
+73
-55
@@ -5,6 +5,7 @@ import json
5
from collections.abc import Generator
6
7
import pytest
8
+from openai.types.responses.function_tool import FunctionTool
9
10
from vllm.entrypoints.openai.chat_completion.protocol import (
11
ChatCompletionRequest,
@@ -49,41 +50,62 @@ def qwen3_tool_parser_parametrized(qwen3_tool_parser, qwen3_xml_tool_parser, req
50
return qwen3_xml_tool_parser
51
52
52
-@pytest.fixture
53
-def sample_tools():
54
- return [
55
- ChatCompletionToolsParam(
56
- type="function",
57
- function={
58
- "name": "get_current_weather",
59
- "description": "Get the current weather",
60
- "parameters": {
61
- "type": "object",
62
- "properties": {
63
- "city": {"type": "string", "description": "The city name"},
64
- "state": {"type": "string", "description": "The state code"},
65
- "unit": {"type": "string", "enum": ["fahrenheit", "celsius"]},
66
- },
67
- "required": ["city", "state"],
53
+WEATHER_PARAMS = {
54
+ "type": "object",
55
+ "properties": {
56
+ "city": {"type": "string", "description": "The city name"},
57
+ "state": {"type": "string", "description": "The state code"},
58
+ "unit": {"type": "string", "enum": ["fahrenheit", "celsius"]},
59
+ },
60
+ "required": ["city", "state"],
61
+}
62
+
63
+AREA_PARAMS = {
64
+ "type": "object",
65
+ "properties": {
66
+ "shape": {"type": "string"},
67
+ "dimensions": {"type": "object"},
68
+ "precision": {"type": "integer"},
69
+ },
70
+}
71
+
72
+
73
+@pytest.fixture(params=["chat_completion", "responses_api"])
74
+def sample_tools(request):
75
+ if request.param == "chat_completion":
76
+ return [
77
+ ChatCompletionToolsParam(
78
+ type="function",
79
+ function={
80
+ "name": "get_current_weather",
81
+ "description": "Get the current weather",
82
+ "parameters": WEATHER_PARAMS,
83
},
69
- },
70
- ),
71
- ChatCompletionToolsParam(
72
- type="function",
73
- function={
74
- "name": "calculate_area",
75
- "description": "Calculate area of a shape",
76
- "parameters": {
77
- "type": "object",
78
- "properties": {
79
- "shape": {"type": "string"},
80
- "dimensions": {"type": "object"},
81
- "precision": {"type": "integer"},
82
- },
84
+ ),
85
+ ChatCompletionToolsParam(
86
+ type="function",
87
+ function={
88
+ "name": "calculate_area",
89
+ "description": "Calculate area of a shape",
90
+ "parameters": AREA_PARAMS,
91
},
84
- },
85
- ),
86
- ]
92
+ ),
93
+ ]
94
+ else:
95
+ return [
96
+ FunctionTool(
97
+ type="function",
98
+ name="get_current_weather",
99
+ description="Get the current weather",
100
+ parameters=WEATHER_PARAMS,
101
+ ),
102
+ FunctionTool(
103
+ type="function",
104
+ name="calculate_area",
105
+ description="Calculate area of a shape",
106
+ parameters=AREA_PARAMS,
107
+ ),
108
+ ]
109
110
111
def assert_tool_calls(
@@ -337,12 +359,11 @@ circle
359
)
360
def test_extract_tool_calls(
361
qwen3_tool_parser_parametrized,
340
- sample_tools,
362
model_output,
363
expected_tool_calls,
364
expected_content,
365
):
345
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
366
+ request = ChatCompletionRequest(model=MODEL, messages=[])
367
extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
368
model_output, request=request
369
)
@@ -354,7 +375,7 @@ def test_extract_tool_calls(
375
376
377
def test_extract_tool_calls_fallback_no_tags(
357
- qwen3_tool_parser_parametrized, sample_tools
378
+ qwen3_tool_parser_parametrized,
379
):
380
"""Test fallback parsing when XML tags are missing"""
381
model_output = """<function=get_current_weather>
@@ -366,7 +387,7 @@ TX
387
</parameter>
388
</function>"""
389
369
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
390
+ request = ChatCompletionRequest(model=MODEL, messages=[])
391
extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
392
model_output, request=request
393
)
@@ -607,13 +628,12 @@ circle
628
def test_extract_tool_calls_streaming(
629
qwen3_tool_parser_parametrized,
630
qwen3_tokenizer,
610
- sample_tools,
631
model_output,
632
expected_tool_calls,
633
expected_content,
634
):
635
"""Test incremental streaming behavior including typed parameters"""
616
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
636
+ request = ChatCompletionRequest(model=MODEL, messages=[])
637
638
other_content = ""
639
tool_states = {} # Track state per tool index
@@ -683,7 +703,7 @@ def test_extract_tool_calls_streaming(
703
704
705
def test_extract_tool_calls_missing_closing_parameter_tag(
686
- qwen3_tool_parser_parametrized, sample_tools
706
+ qwen3_tool_parser_parametrized,
707
):
708
"""Test handling of missing closing </parameter> tag"""
709
# Using get_current_weather from sample_tools but with malformed XML
@@ -701,7 +721,7 @@ fahrenheit
721
</function>
722
</tool_call>"""
723
704
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
724
+ request = ChatCompletionRequest(model=MODEL, messages=[])
725
extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
726
model_output, request=request
727
)
@@ -725,7 +745,7 @@ fahrenheit
745
746
747
def test_extract_tool_calls_streaming_missing_closing_tag(
728
- qwen3_tool_parser_parametrized, qwen3_tokenizer, sample_tools
748
+ qwen3_tool_parser_parametrized, qwen3_tokenizer
749
):
750
"""Test streaming with missing closing </parameter> tag"""
751
# Using get_current_weather from sample_tools but with malformed XML
@@ -743,7 +763,7 @@ fahrenheit
763
</function>
764
</tool_call>"""
765
746
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
766
+ request = ChatCompletionRequest(model=MODEL, messages=[])
767
768
other_content = ""
769
tool_states = {}
@@ -800,7 +820,7 @@ fahrenheit
820
821
822
def test_extract_tool_calls_streaming_incremental(
803
- qwen3_tool_parser_parametrized, qwen3_tokenizer, sample_tools
823
+ qwen3_tool_parser_parametrized, qwen3_tokenizer
824
):
825
"""Test that streaming is truly incremental"""
826
model_output = """I'll check the weather.<tool_call>
@@ -814,7 +834,7 @@ TX
834
</function>
835
</tool_call>"""
836
817
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
837
+ request = ChatCompletionRequest(model=MODEL, messages=[])
838
839
chunks = []
840
for delta_message in stream_delta_message_generator(
@@ -897,7 +917,7 @@ def test_extract_tool_calls_complex_type_with_single_quote(
917
918
919
def test_extract_tool_calls_streaming_missing_opening_tag(
900
- qwen3_tool_parser_parametrized, qwen3_tokenizer, sample_tools
920
+ qwen3_tool_parser_parametrized, qwen3_tokenizer
921
):
922
"""Test streaming with missing opening <tool_call> tag
923
@@ -919,7 +939,7 @@ fahrenheit
939
</function>
940
</tool_call>"""
941
922
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
942
+ request = ChatCompletionRequest(model=MODEL, messages=[])
943
944
other_content = ""
945
tool_states = {}
@@ -976,7 +996,7 @@ fahrenheit
996
assert args["unit"] == "fahrenheit"
997
998
979
-def test_malformed_xml_no_gt_delimiter(qwen3_tool_parser, sample_tools):
999
+def test_malformed_xml_no_gt_delimiter(qwen3_tool_parser):
1000
"""Regression: malformed XML without '>' must not crash (PR #36774)."""
1001
model_output = (
1002
"<tool_call>\n"
@@ -986,14 +1006,14 @@ def test_malformed_xml_no_gt_delimiter(qwen3_tool_parser, sample_tools):
1006
"</tool_call>"
1007
)
1008
989
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1009
+ request = ChatCompletionRequest(model=MODEL, messages=[])
1010
result = qwen3_tool_parser.extract_tool_calls(model_output, request=request)
1011
assert result is not None
1012
assert isinstance(result.tool_calls, list)
1013
assert all(tc is not None for tc in result.tool_calls)
1014
1015
996
-def test_none_tool_calls_filtered(qwen3_tool_parser, sample_tools):
1016
+def test_none_tool_calls_filtered(qwen3_tool_parser):
1017
"""Regression: None tool calls filtered from output (PR #36774)."""
1018
model_output = (
1019
"<tool_call>\n"
@@ -1008,7 +1028,7 @@ def test_none_tool_calls_filtered(qwen3_tool_parser, sample_tools):
1028
"</tool_call>"
1029
)
1030
1011
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1031
+ request = ChatCompletionRequest(model=MODEL, messages=[])
1032
result = qwen3_tool_parser.extract_tool_calls(model_output, request=request)
1033
assert all(tc is not None for tc in result.tool_calls)
1034
assert result.tools_called
@@ -1058,11 +1078,9 @@ def test_anyof_parameter_not_double_encoded(qwen3_tokenizer):
1078
assert args["data"] == {"key": "value", "count": 42}
1079
1080
1061
-def test_streaming_multi_param_single_chunk(
1062
- qwen3_tool_parser, qwen3_tokenizer, sample_tools
1063
-):
1081
+def test_streaming_multi_param_single_chunk(qwen3_tool_parser, qwen3_tokenizer):
1082
"""Regression: speculative decode delivering multiple params at once (PR #35615)."""
1065
- request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools)
1083
+ request = ChatCompletionRequest(model=MODEL, messages=[])
1084
1085
deltas = [
1086
"<tool_call>",
vllm/tool_parsers/qwen3coder_tool_parser.py
+6
-31
@@ -25,6 +25,7 @@ from vllm.tool_parsers.abstract_tool_parser import (
25
Tool,
26
ToolParser,
27
)
28
+from vllm.tool_parsers.utils import find_tool_properties
29
30
logger = init_logger(__name__)
31
@@ -109,28 +110,6 @@ class Qwen3CoderToolParser(ToolParser):
110
self.accumulated_params = {}
111
self.streaming_request = None
112
112
- def _get_arguments_config(self, func_name: str, tools: list[Tool] | None) -> dict:
113
- """Extract argument configuration for a function."""
114
- if tools is None:
115
- return {}
116
- for config in tools:
117
- if not hasattr(config, "type") or not (
118
- hasattr(config, "function") and hasattr(config.function, "name")
119
- ):
120
- continue
121
- if config.type == "function" and config.function.name == func_name:
122
- if not hasattr(config.function, "parameters"):
123
- return {}
124
- params = config.function.parameters
125
- if isinstance(params, dict) and "properties" in params:
126
- return params["properties"]
127
- elif isinstance(params, dict):
128
- return params
129
- else:
130
- return {}
131
- logger.debug("Tool '%s' is not defined in the tools list.", func_name)
132
- return {}
133
-
113
def _convert_param_value(
114
self, param_value: str, param_name: str, param_config: dict, func_name: str
115
) -> Any:
@@ -243,16 +222,14 @@ class Qwen3CoderToolParser(ToolParser):
222
)
223
return param_value
224
246
- def _parse_xml_function_call(
247
- self, function_call_str: str, tools: list[Tool] | None
248
- ) -> ToolCall | None:
225
+ def _parse_xml_function_call(self, function_call_str: str) -> ToolCall | None:
226
# Extract function name
227
end_index = function_call_str.find(">")
228
# If there's no ">" character, this is not a valid xml function call
229
if end_index == -1:
230
return None
231
function_name = function_call_str[:end_index]
255
- param_config = self._get_arguments_config(function_name, tools)
232
+ param_config = find_tool_properties(self.tools, function_name)
233
parameters = function_call_str[end_index + 1 :]
234
param_dict = {}
235
for match_text in self.tool_call_parameter_regex.findall(parameters):
@@ -314,7 +291,7 @@ class Qwen3CoderToolParser(ToolParser):
291
)
292
293
tool_calls = [
317
- self._parse_xml_function_call(function_call_str, self.tools)
294
+ self._parse_xml_function_call(function_call_str)
295
for function_call_str in function_calls
296
]
297
# Populate prev_tool_call_arr for serving layer to set finish_reason
@@ -605,9 +582,8 @@ class Qwen3CoderToolParser(ToolParser):
582
self.current_param_name = current_param_name
583
self.accumulated_params[current_param_name] = param_value
584
608
- param_config = self._get_arguments_config(
609
- self.current_function_name or "",
610
- self.tools,
585
+ param_config = find_tool_properties(
586
+ self.tools, self.current_function_name or ""
587
)
588
589
converted_value = self._convert_param_value(
@@ -666,7 +642,6 @@ class Qwen3CoderToolParser(ToolParser):
642
try:
643
parsed_tool = self._parse_xml_function_call(
644
func_content,
669
- self.tools,
645
)
646
if parsed_tool and self.current_tool_index < len(
647
self.prev_tool_call_arr
vllm/tool_parsers/qwen3xml_tool_parser.py
+6
-27
@@ -26,6 +26,7 @@ from vllm.tool_parsers.abstract_tool_parser import (
26
Tool,
27
ToolParser,
28
)
29
+from vllm.tool_parsers.utils import find_tool_properties
30
31
logger = init_logger(__name__)
32
@@ -1000,33 +1001,11 @@ class StreamingXMLToolCallParser:
1001
if not self.tools or not self.current_function_name:
1002
return "string"
1003
1003
- for tool in self.tools:
1004
- if not hasattr(tool, "type") or not (
1005
- hasattr(tool, "function") and hasattr(tool.function, "name")
1006
- ):
1007
- continue
1008
- if (
1009
- tool.type == "function"
1010
- and tool.function.name == self.current_function_name
1011
- ):
1012
- if not hasattr(tool.function, "parameters"):
1013
- return "string"
1014
- params = tool.function.parameters
1015
- if isinstance(params, dict) and "properties" in params:
1016
- properties = params["properties"]
1017
- if param_name in properties and isinstance(
1018
- properties[param_name], dict
1019
- ):
1020
- return self.repair_param_type(
1021
- str(properties[param_name].get("type", "string"))
1022
- )
1023
- elif isinstance(params, dict) and param_name in params:
1024
- param_config = params[param_name]
1025
- if isinstance(param_config, dict):
1026
- return self.repair_param_type(
1027
- str(param_config.get("type", "string"))
1028
- )
1029
- break
1004
+ properties = find_tool_properties(self.tools, self.current_function_name)
1005
+ if param_name in properties and isinstance(properties[param_name], dict):
1006
+ return self.repair_param_type(
1007
+ str(properties[param_name].get("type", "string"))
1008
+ )
1009
return "string"
1010
1011
def repair_param_type(self, param_type: str) -> str:
vllm/tool_parsers/utils.py
+14
@@ -142,6 +142,20 @@ def _extract_tool_info(
142
raise TypeError(f"Unsupported tool type: {type(tool)}")
143
144
145
+def find_tool_properties(
146
+ tools: list[Tool] | None,
147
+ tool_name: str,
148
+) -> dict[str, Any]:
149
+ """Find a tool by name and return its properties dict, or {}."""
150
+ if not tools:
151
+ return {}
152
+ for tool in tools:
153
+ name, params = _extract_tool_info(tool)
154
+ if name == tool_name:
155
+ return (params or {}).get("properties", {})
156
+ return {}
157
+
158
+
159
def _get_tool_schema_from_tool(tool: Tool) -> dict:
160
name, params = _extract_tool_info(tool)
161
params = params if params else {"type": "object", "properties": {}}