@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 786835807b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Bugfix]: Qwen3 Coder Tool Parser (#23099)

Signed-off-by: Yiheng Xu <charlesyihengxu@gmail.com> Co-authored-by: Aaron Pham <contact@aarnphm.xyz>

Yiheng Xu committed Aug 27, 2025 at 10:58 UTC 786835807b491279af1fc5f565df9c6baedf3827
3 files changed +576 -248
examples/tool_chat_template_qwen3coder.jinja new
+117
@@ -0,0 +1,117 @@
1 +{% macro render_extra_keys(json_dict, handled_keys) %}
2 + {%- if json_dict is mapping %}
3 + {%- for json_key in json_dict if json_key not in handled_keys %}
4 + {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
5 + {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
6 + {%- else %}
7 + {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
8 + {%- endif %}
9 + {%- endfor %}
10 + {%- endif %}
11 +{% endmacro %}
12 +
13 +{%- if messages[0]["role"] == "system" %}
14 + {%- set system_message = messages[0]["content"] %}
15 + {%- set loop_messages = messages[1:] %}
16 +{%- else %}
17 + {%- set loop_messages = messages %}
18 +{%- endif %}
19 +
20 +{%- if not tools is defined %}
21 + {%- set tools = [] %}
22 +{%- endif %}
23 +
24 +{%- if system_message is defined %}
25 + {{- "<|im_start|>system\n" + system_message }}
26 +{%- else %}
27 + {%- if tools is iterable and tools | length > 0 %}
28 + {{- "<|im_start|>system\nYou are Qwen, a helpful AI assistant that can interact with a computer to solve tasks." }}
29 + {%- endif %}
30 +{%- endif %}
31 +{%- if tools is iterable and tools | length > 0 %}
32 + {{- "\n\n# Tools\n\nYou have access to the following functions:\n\n" }}
33 + {{- "<tools>" }}
34 + {%- for tool in tools %}
35 + {%- if tool.function is defined %}
36 + {%- set tool = tool.function %}
37 + {%- endif %}
38 + {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
39 + {%- if tool.description is defined %}
40 + {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
41 + {%- endif %}
42 + {{- '\n<parameters>' }}
43 + {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
44 + {%- for param_name, param_fields in tool.parameters.properties|items %}
45 + {{- '\n<parameter>' }}
46 + {{- '\n<name>' ~ param_name ~ '</name>' }}
47 + {%- if param_fields.type is defined %}
48 + {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
49 + {%- endif %}
50 + {%- if param_fields.description is defined %}
51 + {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
52 + {%- endif %}
53 + {%- set handled_keys = ['name', 'type', 'description'] %}
54 + {{- render_extra_keys(param_fields, handled_keys) }}
55 + {{- '\n</parameter>' }}
56 + {%- endfor %}
57 + {%- endif %}
58 + {% set handled_keys = ['type', 'properties'] %}
59 + {{- render_extra_keys(tool.parameters, handled_keys) }}
60 + {{- '\n</parameters>' }}
61 + {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
62 + {{- render_extra_keys(tool, handled_keys) }}
63 + {{- '\n</function>' }}
64 + {%- endfor %}
65 + {{- "\n</tools>" }}
66 + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
67 +{%- endif %}
68 +{%- if system_message is defined %}
69 + {{- '<|im_end|>\n' }}
70 +{%- else %}
71 + {%- if tools is iterable and tools | length > 0 %}
72 + {{- '<|im_end|>\n' }}
73 + {%- endif %}
74 +{%- endif %}
75 +{%- for message in loop_messages %}
76 + {%- if message.role == "assistant" and message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
77 + {{- '<|im_start|>' + message.role }}
78 + {%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
79 + {{- '\n' + message.content | trim + '\n' }}
80 + {%- endif %}
81 + {%- for tool_call in message.tool_calls %}
82 + {%- if tool_call.function is defined %}
83 + {%- set tool_call = tool_call.function %}
84 + {%- endif %}
85 + {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
86 + {%- if tool_call.arguments is defined %}
87 + {%- for args_name, args_value in tool_call.arguments|items %}
88 + {{- '<parameter=' + args_name + '>\n' }}
89 + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
90 + {{- args_value }}
91 + {{- '\n</parameter>\n' }}
92 + {%- endfor %}
93 + {%- endif %}
94 + {{- '</function>\n</tool_call>' }}
95 + {%- endfor %}
96 + {{- '<|im_end|>\n' }}
97 + {%- elif message.role == "user" or message.role == "system" or message.role == "assistant" %}
98 + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
99 + {%- elif message.role == "tool" %}
100 + {%- if loop.previtem and loop.previtem.role != "tool" %}
101 + {{- '<|im_start|>user\n' }}
102 + {%- endif %}
103 + {{- '<tool_response>\n' }}
104 + {{- message.content }}
105 + {{- '\n</tool_response>\n' }}
106 + {%- if not loop.last and loop.nextitem.role != "tool" %}
107 + {{- '<|im_end|>\n' }}
108 + {%- elif loop.last %}
109 + {{- '<|im_end|>\n' }}
110 + {%- endif %}
111 + {%- else %}
112 + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
113 + {%- endif %}
114 +{%- endfor %}
115 +{%- if add_generation_prompt %}
116 + {{- '<|im_start|>assistant\n' }}
117 +{%- endif %}
tests/tool_use/test_qwen3coder_tool_parser.py
+173 -5
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.tool_parsers.qwen3coder_tool_parser import (
16 from vllm.transformers_utils.detokenizer import detokenize_incrementally
17 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
18
19 -MODEL = "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8"
19 +MODEL = "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
20
21
22 @pytest.fixture(scope="module")
@@ -397,7 +397,9 @@ hello world
397 "no_tools",
398 "single_tool",
399 "single_tool_with_content",
400 + "single_tool_multiline_param",
401 "parallel_tools",
402 + "tool_with_typed_params", # Added this test case
403 ],
404 argnames=["model_output", "expected_tool_calls", "expected_content"],
405 argvalues=[
@@ -422,7 +424,7 @@ fahrenheit
424 "state": "TX",
425 "unit": "fahrenheit"
426 })))
425 - ], ""),
427 + ], None),
428 ('''Sure! Let me check the weather for you.<tool_call>
429 <function=get_current_weather>
430 <parameter=city>
@@ -445,6 +447,30 @@ fahrenheit
447 })))
448 ], "Sure! Let me check the weather for you."),
449 ('''<tool_call>
450 +<function=calculate_area>
451 +<parameter=shape>
452 +rectangle
453 +</parameter>
454 +<parameter=dimensions>
455 +{"width": 10,
456 + "height": 20}
457 +</parameter>
458 +<parameter=precision>
459 +2
460 +</parameter>
461 +</function>
462 +</tool_call>''', [
463 + ToolCall(function=FunctionCall(name="calculate_area",
464 + arguments=json.dumps({
465 + "shape": "rectangle",
466 + "dimensions": {
467 + "width": 10,
468 + "height": 20
469 + },
470 + "precision": 2
471 + })))
472 + ], None),
473 + ('''<tool_call>
474 <function=get_current_weather>
475 <parameter=city>
476 Dallas
@@ -484,13 +510,36 @@ celsius
510 "state": "FL",
511 "unit": "celsius"
512 })))
487 - ], ""),
513 + ], None),
514 + # Added tool_with_typed_params test case
515 + ('''Let me calculate that area for you.<tool_call>
516 +<function=calculate_area>
517 +<parameter=shape>
518 +circle
519 +</parameter>
520 +<parameter=dimensions>
521 +{"radius": 15.5}
522 +</parameter>
523 +<parameter=precision>
524 +3
525 +</parameter>
526 +</function>
527 +</tool_call>''', [
528 + ToolCall(function=FunctionCall(name="calculate_area",
529 + arguments=json.dumps({
530 + "shape": "circle",
531 + "dimensions": {
532 + "radius": 15.5
533 + },
534 + "precision": 3
535 + })))
536 + ], "Let me calculate that area for you."),
537 ],
538 )
539 def test_extract_tool_calls_streaming(qwen3_tool_parser, qwen3_tokenizer,
540 sample_tools, model_output,
541 expected_tool_calls, expected_content):
493 - """Test incremental streaming behavior"""
542 + """Test incremental streaming behavior including typed parameters"""
543 request = ChatCompletionRequest(model=MODEL,
544 messages=[],
545 tools=sample_tools)
@@ -539,7 +588,7 @@ def test_extract_tool_calls_streaming(qwen3_tool_parser, qwen3_tokenizer,
588 "arguments"] += tool_call.function.arguments
589
590 # Verify final content
542 - assert other_content == expected_content
591 + assert other_content == (expected_content or "") # Handle None case
592
593 # Verify we got all expected tool calls
594 assert len(tool_states) == len(expected_tool_calls)
@@ -559,6 +608,125 @@ def test_extract_tool_calls_streaming(qwen3_tool_parser, qwen3_tokenizer,
608 assert actual_args == expected_args
609
610
611 +def test_extract_tool_calls_missing_closing_parameter_tag(
612 + qwen3_tool_parser, sample_tools):
613 + """Test handling of missing closing </parameter> tag"""
614 + # Using get_current_weather from sample_tools but with malformed XML
615 + model_output = '''Let me check the weather for you:
616 +<tool_call>
617 +<function=get_current_weather>
618 +<parameter=city>
619 +Dallas
620 +<parameter=state>
621 +TX
622 +</parameter>
623 +<parameter=unit>
624 +fahrenheit
625 +</parameter>
626 +</function>
627 +</tool_call>'''
628 +
629 + request = ChatCompletionRequest(model=MODEL,
630 + messages=[],
631 + tools=sample_tools)
632 + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
633 + model_output, request=request)
634 +
635 + # The parser should handle the malformed XML gracefully
636 + assert extracted_tool_calls.tools_called
637 + assert len(extracted_tool_calls.tool_calls) == 1
638 +
639 + # Verify the function name is correct
640 + assert extracted_tool_calls.tool_calls[
641 + 0].function.name == "get_current_weather"
642 +
643 + # Verify the arguments are parsed despite the missing closing tag
644 + args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
645 + assert "city" in args
646 + assert args["city"] == "Dallas"
647 + assert args["state"] == "TX"
648 + assert args["unit"] == "fahrenheit"
649 +
650 + # Check that content before the tool call is preserved
651 + assert "Let me check the weather for you:" in extracted_tool_calls.content
652 +
653 +
654 +def test_extract_tool_calls_streaming_missing_closing_tag(
655 + qwen3_tool_parser, qwen3_tokenizer, sample_tools):
656 + """Test streaming with missing closing </parameter> tag"""
657 + # Using get_current_weather from sample_tools but with malformed XML
658 + model_output = '''Let me check the weather for you:
659 +<tool_call>
660 +<function=get_current_weather>
661 +<parameter=city>
662 +Dallas
663 +<parameter=state>
664 +TX
665 +</parameter>
666 +<parameter=unit>
667 +fahrenheit
668 +</parameter>
669 +</function>
670 +</tool_call>'''
671 +
672 + request = ChatCompletionRequest(model=MODEL,
673 + messages=[],
674 + tools=sample_tools)
675 +
676 + other_content = ''
677 + tool_states = {}
678 +
679 + for delta_message in stream_delta_message_generator(
680 + qwen3_tool_parser, qwen3_tokenizer, model_output, request):
681 +
682 + if delta_message.content:
683 + other_content += delta_message.content
684 +
685 + if delta_message.tool_calls:
686 + for tool_call in delta_message.tool_calls:
687 + idx = tool_call.index
688 +
689 + if idx not in tool_states:
690 + tool_states[idx] = {
691 + "id": None,
692 + "name": None,
693 + "arguments": "",
694 + "type": None
695 + }
696 +
697 + if tool_call.id:
698 + tool_states[idx]["id"] = tool_call.id
699 +
700 + if tool_call.type:
701 + assert tool_call.type == "function"
702 + tool_states[idx]["type"] = tool_call.type
703 +
704 + if tool_call.function:
705 + if tool_call.function.name:
706 + tool_states[idx]["name"] = tool_call.function.name
707 +
708 + if tool_call.function.arguments is not None:
709 + tool_states[idx][
710 + "arguments"] += tool_call.function.arguments
711 +
712 + # Verify content was streamed
713 + assert "Let me check the weather for you:" in other_content
714 +
715 + # Verify we got the tool call
716 + assert len(tool_states) == 1
717 + state = tool_states[0]
718 + assert state["id"] is not None
719 + assert state["type"] == "function"
720 + assert state["name"] == "get_current_weather"
721 +
722 + # Verify arguments were parsed correctly despite missing closing tag
723 + assert state["arguments"] is not None
724 + args = json.loads(state["arguments"])
725 + assert args["city"] == "Dallas"
726 + assert args["state"] == "TX"
727 + assert args["unit"] == "fahrenheit"
728 +
729 +
730 def test_extract_tool_calls_streaming_incremental(qwen3_tool_parser,
731 qwen3_tokenizer,
732 sample_tools):
vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py
+286 -243
@@ -1,6 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
3 +import ast
4 import json
5 import uuid
6 from collections.abc import Sequence
@@ -22,7 +22,7 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
22 logger = init_logger(__name__)
23
24
25 -@ToolParserManager.register_module(["qwen3_coder"])
25 +@ToolParserManager.register_module("qwen3_coder")
26 class Qwen3CoderToolParser(ToolParser):
27
28 def __init__(self, tokenizer: AnyTokenizer):
@@ -30,6 +30,8 @@ class Qwen3CoderToolParser(ToolParser):
30
31 self.current_tool_name_sent: bool = False
32 self.prev_tool_call_arr: list[dict] = []
33 + # Override base class type - we use string IDs for tool calls
34 + self.current_tool_id: Optional[str] = None # type: ignore
35 self.streamed_args_for_tool: list[str] = []
36
37 # Sentinel tokens for streaming mode
@@ -42,20 +44,6 @@ class Qwen3CoderToolParser(ToolParser):
44 self.is_tool_call_started: bool = False
45 self.failed_count: int = 0
46
45 - # Streaming state variables
46 - self.current_tool_index: int = 0
47 - self.header_sent: bool = False
48 - self.current_tool_string_id: Optional[str] = None
49 - self.current_function_name: Optional[str] = None
50 - self.current_param_name: Optional[str] = None
51 - self.current_param_value: str = ""
52 - self.param_count: int = 0
53 - self.in_param: bool = False
54 - self.in_function: bool = False
55 - self.accumulated_text: str = ""
56 - self.json_started: bool = False
57 - self.json_closed: bool = False
58 -
47 # Enhanced streaming state - reset for each new message
48 self._reset_streaming_state()
49
@@ -67,7 +55,8 @@ class Qwen3CoderToolParser(ToolParser):
55 self.tool_call_function_regex = re.compile(
56 r"<function=(.*?)</function>|<function=(.*)$", re.DOTALL)
57 self.tool_call_parameter_regex = re.compile(
70 - r"<parameter=(.*?)</parameter>|<parameter=(.*?)$", re.DOTALL)
58 + r"<parameter=(.*?)(?:</parameter>|(?=<parameter=)|(?=</function>)|$)",
59 + re.DOTALL)
60
61 if not self.model_tokenizer:
62 raise ValueError(
@@ -84,8 +73,8 @@ class Qwen3CoderToolParser(ToolParser):
73 "Qwen3 XML Tool parser could not locate tool call start/end "
74 "tokens in the tokenizer!")
75
87 - logger.debug("vLLM Successfully import tool parser %s !",
88 - self.__class__.__name__)
76 + logger.info("vLLM Successfully import tool parser %s !",
77 + self.__class__.__name__)
78
79 def _generate_tool_call_id(self) -> str:
80 """Generate a unique tool call ID."""
@@ -96,7 +85,7 @@ class Qwen3CoderToolParser(ToolParser):
85 self.current_tool_index = 0
86 self.is_tool_call_started = False
87 self.header_sent = False
99 - self.current_tool_string_id = None
88 + self.current_tool_id = None
89 self.current_function_name = None
90 self.current_param_name = None
91 self.current_param_value = ""
@@ -106,122 +95,122 @@ class Qwen3CoderToolParser(ToolParser):
95 self.accumulated_text = ""
96 self.json_started = False
97 self.json_closed = False
109 -
110 - def _parse_xml_function_call(
111 - self, function_call_str: str,
112 - tools: Optional[list[ChatCompletionToolsParam]]
113 - ) -> Optional[ToolCall]:
114 -
115 - def get_arguments_config(func_name: str) -> dict:
116 - if tools is None:
117 - return {}
118 - for config in tools:
119 - if not hasattr(config, "type") or not (
120 - hasattr(config, "function")
121 - and hasattr(config.function, "name")):
122 - continue
123 - if (config.type == "function"
124 - and config.function.name == func_name):
125 - if not hasattr(config.function, "parameters"):
126 - return {}
127 - params = config.function.parameters
128 - if isinstance(params, dict) and "properties" in params:
129 - return params["properties"]
130 - elif isinstance(params, dict):
131 - return params
132 - else:
133 - return {}
134 - logger.warning("Tool '%s' is not defined in the tools list.",
135 - func_name)
98 + # Store accumulated parameters for type conversion
99 + self.accumulated_params = {}
100 + self.streaming_request = None
101 +
102 + def _get_arguments_config(
103 + self, func_name: str,
104 + tools: Optional[list[ChatCompletionToolsParam]]) -> dict:
105 + """Extract argument configuration for a function."""
106 + if tools is None:
107 return {}
108 + for config in tools:
109 + if not hasattr(config, "type") or not (hasattr(
110 + config, "function") and hasattr(config.function, "name")):
111 + continue
112 + if config.type == "function" and config.function.name == func_name:
113 + if not hasattr(config.function, "parameters"):
114 + return {}
115 + params = config.function.parameters
116 + if isinstance(params, dict) and "properties" in params:
117 + return params["properties"]
118 + elif isinstance(params, dict):
119 + return params
120 + else:
121 + return {}
122 + logger.warning("Tool '%s' is not defined in the tools list.",
123 + func_name)
124 + return {}
125 +
126 + def _convert_param_value(self, param_value: str, param_name: str,
127 + param_config: dict, func_name: str) -> Any:
128 + """Convert parameter value based on its type in the schema."""
129 + # Handle null value for any type
130 + if param_value.lower() == "null":
131 + return None
132
138 - def convert_param_value(param_value: str, param_name: str,
139 - param_config: dict, func_name: str) -> Any:
140 - # Handle null value for any type
141 - if param_value.lower() == "null":
142 - return None
143 -
144 - converted_value: Any
145 -
146 - if param_name not in param_config:
147 - if param_config != {}:
148 - logger.warning(
149 - "Parsed parameter '%s' is not defined in the tool "
150 - "parameters for tool '%s', directly returning the "
151 - "string value.", param_name, func_name)
152 - return param_value
153 -
154 - if (isinstance(param_config[param_name], dict)
155 - and "type" in param_config[param_name]):
156 - param_type = str(
157 - param_config[param_name]["type"]).strip().lower()
158 - else:
159 - param_type = "string"
160 - if param_type in [
161 - "string", "str", "text", "varchar", "char", "enum"
162 - ]:
133 + if param_name not in param_config:
134 + if param_config != {}:
135 + logger.warning(
136 + "Parsed parameter '%s' is not defined in the tool "
137 + "parameters for tool '%s', directly returning the "
138 + "string value.", param_name, func_name)
139 + return param_value
140 +
141 + if isinstance(param_config[param_name],
142 + dict) and "type" in param_config[param_name]:
143 + param_type = str(param_config[param_name]["type"]).strip().lower()
144 + else:
145 + param_type = "string"
146 + if param_type in ["string", "str", "text", "varchar", "char", "enum"]:
147 + return param_value
148 + elif param_type.startswith("int") or param_type.startswith(
149 + "uint") or param_type.startswith(
150 + "long") or param_type.startswith(
151 + "short") or param_type.startswith("unsigned"):
152 + try:
153 + return int(param_value)
154 + except (ValueError, TypeError):
155 + logger.warning(
156 + "Parsed value '%s' of parameter '%s' is not an "
157 + "integer in tool '%s', degenerating to string.",
158 + param_value, param_name, func_name)
159 return param_value
164 - elif (param_type.startswith("int") or param_type.startswith("uint")
165 - or param_type.startswith("long")
166 - or param_type.startswith("short")
167 - or param_type.startswith("unsigned")):
168 - try:
169 - converted_value = int(param_value)
170 - return converted_value
171 - except ValueError:
172 - logger.warning(
173 - "Parsed value '%s' of parameter '%s' is not an "
174 - "integer in tool '%s', degenerating to string.",
175 - param_value, param_name, func_name)
160 + elif param_type.startswith("num") or param_type.startswith("float"):
161 + try:
162 + float_param_value = float(param_value)
163 + return float_param_value if float_param_value - int(
164 + float_param_value) != 0 else int(float_param_value)
165 + except (ValueError, TypeError):
166 + logger.warning(
167 + "Parsed value '%s' of parameter '%s' is not a float "
168 + "in tool '%s', degenerating to string.", param_value,
169 + param_name, func_name)
170 return param_value
177 - elif (param_type.startswith("num")
178 - or param_type.startswith("float")):
171 + elif param_type in ["boolean", "bool", "binary"]:
172 + param_value = param_value.lower()
173 + if param_value not in ["true", "false"]:
174 + logger.warning(
175 + "Parsed value '%s' of parameter '%s' is not a boolean "
176 + "(`true` or `false`) in tool '%s', degenerating to "
177 + "false.", param_value, param_name, func_name)
178 + return param_value == "true"
179 + else:
180 + if param_type in ["object", "array", "arr"
181 + ] or param_type.startswith(
182 + "dict") or param_type.startswith("list"):
183 try:
180 - float_param_value = float(param_value)
181 - converted_value = (float_param_value if float_param_value -
182 - int(float_param_value) != 0 else
183 - int(float_param_value))
184 - return converted_value
185 - except ValueError:
184 + param_value = json.loads(param_value)
185 + return param_value
186 + except (json.JSONDecodeError, TypeError, ValueError):
187 logger.warning(
187 - "Parsed value '%s' of parameter '%s' is not a float "
188 - "in tool '%s', degenerating to string.", param_value,
189 - param_name, func_name)
190 - return param_value
191 - elif param_type in ["boolean", "bool", "binary"]:
192 - param_value = param_value.lower()
193 - if param_value not in ["true", "false"]:
194 - logger.warning(
195 - "Parsed value '%s' of parameter '%s' is not a "
196 - "boolean (`true` of `false`) in tool '%s', "
197 - "degenerating to false.", param_value, param_name,
188 + "Parsed value '%s' of parameter '%s' cannot be "
189 + "parsed with json.loads in tool '%s', will try "
190 + "other methods to parse it.", param_value, param_name,
191 func_name)
199 - return param_value == "true"
200 - else:
201 - if param_type == "object" or param_type.startswith("dict"):
202 - try:
203 - converted_value = json.loads(param_value)
204 - return converted_value
205 - except json.JSONDecodeError:
206 - logger.warning(
207 - "Parsed value '%s' of parameter '%s' is not a "
208 - "valid JSON object in tool '%s', will try other "
209 - "methods to parse it.", param_value, param_name,
210 - func_name)
192 + try:
193 + param_value = ast.literal_eval(param_value) # safer
194 + except (ValueError, SyntaxError, TypeError):
195 logger.warning(
212 - "Parameter '%s' has unknown type '%s'. "
213 - "The value will be treated as a string.", param_name,
214 - param_type)
215 - return param_value
196 + "Parsed value '%s' of parameter '%s' cannot be "
197 + "converted via Python `ast.literal_eval()` in tool "
198 + "'%s', degenerating to string.", param_value, param_name,
199 + func_name)
200 + return param_value
201 +
202 + def _parse_xml_function_call(
203 + self, function_call_str: str,
204 + tools: Optional[list[ChatCompletionToolsParam]]
205 + ) -> Optional[ToolCall]:
206
207 # Extract function name
208 end_index = function_call_str.index(">")
209 function_name = function_call_str[:end_index]
220 - param_config = get_arguments_config(function_name)
210 + param_config = self._get_arguments_config(function_name, tools)
211 parameters = function_call_str[end_index + 1:]
212 param_dict = {}
223 - for match in self.tool_call_parameter_regex.findall(parameters):
224 - match_text = match[0] if match[0] else match[1]
213 + for match_text in self.tool_call_parameter_regex.findall(parameters):
214 idx = match_text.index(">")
215 param_name = match_text[:idx]
216 param_value = str(match_text[idx + 1:])
@@ -231,7 +220,7 @@ class Qwen3CoderToolParser(ToolParser):
220 if param_value.endswith("\n"):
221 param_value = param_value[:-1]
222
234 - param_dict[param_name] = convert_param_value(
223 + param_dict[param_name] = self._convert_param_value(
224 param_value, param_name, param_config, function_name)
225 return ToolCall(
226 type="function",
@@ -284,8 +273,7 @@ class Qwen3CoderToolParser(ToolParser):
273 for function_call_str in function_calls
274 ]
275
287 - # Populate prev_tool_call_arr for serving layer to set
288 - # finish_reason
276 + # Populate prev_tool_call_arr for serving layer to set finish_reason
277 self.prev_tool_call_arr.clear() # Clear previous calls
278 for tool_call in tool_calls:
279 if tool_call:
@@ -298,8 +286,8 @@ class Qwen3CoderToolParser(ToolParser):
286
287 # Extract content before tool calls
288 content_index = model_output.find(self.tool_call_start_token)
301 - content_index = (content_index if content_index >= 0 else
302 - model_output.find(self.tool_call_prefix))
289 + idx = model_output.find(self.tool_call_prefix)
290 + content_index = content_index if content_index >= 0 else idx
291 content = model_output[:content_index] # .rstrip()
292
293 return ExtractedToolCallInformation(
@@ -324,13 +312,16 @@ class Qwen3CoderToolParser(ToolParser):
312 delta_token_ids: Sequence[int],
313 request: ChatCompletionRequest,
314 ) -> Union[DeltaMessage, None]:
327 - # If no delta text, return None unless it's an EOS token after tool
328 - # calls
315 + # Store request for type conversion
316 + if not previous_text:
317 + self._reset_streaming_state()
318 + self.streaming_request = request
319 +
320 + # If no delta text, return None unless it's an EOS token after tools
321 if not delta_text:
322 # Check if this is an EOS token after all tool calls are complete
331 - # We check for tool calls in the text even if is_tool_call_started
332 - # is False because it might have been reset after processing all
333 - # tools
323 + # Check for tool calls in text even if is_tool_call_started
324 + # is False (might have been reset after processing all tools)
325 if (delta_token_ids
326 and self.tool_call_end_token_id not in delta_token_ids):
327 # Count complete tool calls
@@ -339,24 +330,19 @@ class Qwen3CoderToolParser(ToolParser):
330
331 # If we have completed tool calls and populated
332 # prev_tool_call_arr
342 - if (complete_calls > 0 and len(self.prev_tool_call_arr) > 0):
333 + if complete_calls > 0 and len(self.prev_tool_call_arr) > 0:
334 # Check if all tool calls are closed
344 - open_calls = (
345 - current_text.count(self.tool_call_start_token) -
346 - current_text.count(self.tool_call_end_token))
335 + open_calls = current_text.count(
336 + self.tool_call_start_token) - current_text.count(
337 + self.tool_call_end_token)
338 if open_calls == 0:
348 - # Return empty delta message to allow finish_reason
349 - # processing
339 + # Return empty delta for finish_reason processing
340 return DeltaMessage(content="")
341 elif not self.is_tool_call_started and current_text:
342 # This is a regular content response that's now complete
343 return DeltaMessage(content="")
344 return None
345
356 - # Check if this is the first call (reset state if needed)
357 - if not previous_text:
358 - self._reset_streaming_state()
359 -
346 # Update accumulated text
347 self.accumulated_text = current_text
348
@@ -371,11 +357,11 @@ class Qwen3CoderToolParser(ToolParser):
357 self.param_count = 0
358 self.json_started = False
359 self.json_closed = False
360 + self.accumulated_params = {}
361
362 # Check if there are more tool calls
376 - tool_starts_count = current_text.count(
377 - self.tool_call_start_token)
378 - if self.current_tool_index >= tool_starts_count:
363 + tool_starts = current_text.count(self.tool_call_start_token)
364 + if self.current_tool_index >= tool_starts:
365 # No more tool calls
366 self.is_tool_call_started = False
367 # Continue processing next tool
@@ -412,20 +398,20 @@ class Qwen3CoderToolParser(ToolParser):
398
399 # We're in a tool call, find the current tool call portion
400 # Need to find the correct tool call based on current_tool_index
415 - tool_starts: list[int] = []
401 + tool_start_positions: list[int] = []
402 idx = 0
403 while True:
404 idx = current_text.find(self.tool_call_start_token, idx)
405 if idx == -1:
406 break
421 - tool_starts.append(idx)
407 + tool_start_positions.append(idx)
408 idx += len(self.tool_call_start_token)
409
424 - if self.current_tool_index >= len(tool_starts):
410 + if self.current_tool_index >= len(tool_start_positions):
411 # No more tool calls to process yet
412 return None
413
428 - tool_start_idx = tool_starts[self.current_tool_index]
414 + tool_start_idx = tool_start_positions[self.current_tool_index]
415 # Find where this tool call ends (or current position if not ended yet)
416 tool_end_idx = current_text.find(self.tool_call_end_token,
417 tool_start_idx)
@@ -438,19 +424,19 @@ class Qwen3CoderToolParser(ToolParser):
424 # Looking for function header
425 if not self.header_sent:
426 if self.tool_call_prefix in tool_text:
441 - func_start = (tool_text.find(self.tool_call_prefix) +
442 - len(self.tool_call_prefix))
427 + func_start = tool_text.find(self.tool_call_prefix) + len(
428 + self.tool_call_prefix)
429 func_end = tool_text.find(">", func_start)
430
431 if func_end != -1:
432 # Found complete function name
433 self.current_function_name = tool_text[func_start:func_end]
448 - self.current_tool_string_id = self._generate_tool_call_id()
434 + self.current_tool_id = self._generate_tool_call_id()
435 self.header_sent = True
436 self.in_function = True
437
452 - # IMPORTANT: Add to prev_tool_call_arr immediately when we
453 - # detect a tool call. This ensures
438 + # IMPORTANT: Add to prev_tool_call_arr immediately when
439 + # we detect a tool call. This ensures
440 # finish_reason="tool_calls" even if parsing isn't complete
441 already_added = any(
442 tool.get("name") == self.current_function_name
@@ -466,7 +452,7 @@ class Qwen3CoderToolParser(ToolParser):
452 return DeltaMessage(tool_calls=[
453 DeltaToolCall(
454 index=self.current_tool_index,
469 - id=self.current_tool_string_id,
455 + id=self.current_tool_id,
456 function=DeltaFunctionCall(
457 name=self.current_function_name, arguments=""),
458 type="function",
@@ -496,10 +482,11 @@ class Qwen3CoderToolParser(ToolParser):
482 # Close JSON
483 self.json_closed = True
484
499 - # Extract the complete tool call to update prev_tool_call_arr
500 - # with final arguments. Find the function content
501 - func_start = (tool_text.find(self.tool_call_prefix) +
502 - len(self.tool_call_prefix))
485 + # Extract complete tool call to update
486 + # prev_tool_call_arr with final arguments
487 + # Find the function content
488 + func_start = tool_text.find(self.tool_call_prefix) + len(
489 + self.tool_call_prefix)
490 func_content_end = tool_text.find(self.function_end_token,
491 func_start)
492 if func_content_end != -1:
@@ -507,15 +494,17 @@ class Qwen3CoderToolParser(ToolParser):
494 # Parse to get the complete arguments
495 try:
496 parsed_tool = self._parse_xml_function_call(
510 - func_content, request.tools if request else None)
497 + func_content, self.streaming_request.tools
498 + if self.streaming_request else None)
499 if parsed_tool:
512 - # Update existing entry in prev_tool_call_arr with
513 - # complete arguments
500 + # Update existing entry in
501 + # prev_tool_call_arr with complete args
502 for i, tool in enumerate(self.prev_tool_call_arr):
515 - if (tool.get("name") ==
516 - parsed_tool.function.name):
517 - self.prev_tool_call_arr[i]["arguments"] = (
518 - parsed_tool.function.arguments)
503 + if tool.get(
504 + "name") == parsed_tool.function.name:
505 + args = parsed_tool.function.arguments
506 + self.prev_tool_call_arr[i][
507 + "arguments"] = args
508 break
509 except Exception:
510 pass # Ignore parsing errors during streaming
@@ -530,73 +519,110 @@ class Qwen3CoderToolParser(ToolParser):
519 # Reset state for next tool
520 self.in_function = False
521 self.json_closed = True
522 + self.accumulated_params = {}
523
524 return result
525
526 # Look for parameters
537 - # Count how many complete parameters we have processed
538 - complete_params = tool_text.count(self.parameter_end_token)
527 + # Find all parameter starts
528 + param_starts = []
529 + idx = 0
530 + while True:
531 + idx = tool_text.find(self.parameter_prefix, idx)
532 + if idx == -1:
533 + break
534 + param_starts.append(idx)
535 + idx += len(self.parameter_prefix)
536
537 # Check if we should start a new parameter
541 - if not self.in_param and self.param_count < complete_params:
542 - # Find the unprocessed parameter
543 - # Count parameter starts
544 - param_starts = []
545 - idx = 0
546 - while True:
547 - idx = tool_text.find(self.parameter_prefix, idx)
548 - if idx == -1:
549 - break
550 - param_starts.append(idx)
551 - idx += len(self.parameter_prefix)
552 -
553 - if len(param_starts) > self.param_count:
554 - # Process the next parameter
555 - param_idx = param_starts[self.param_count]
556 - param_start = param_idx + len(self.parameter_prefix)
557 - remaining = tool_text[param_start:]
558 -
559 - if ">" in remaining:
560 - # We have the complete parameter name
561 - name_end = remaining.find(">")
562 - self.current_param_name = remaining[:name_end]
563 -
564 - # Find the parameter value
565 - value_start = param_start + name_end + 1
566 - value_text = tool_text[value_start:]
567 - if value_text.startswith("\n"):
568 - value_text = value_text[1:]
569 -
570 - # Find where this parameter ends
571 - param_end_idx = value_text.find(
572 - self.parameter_end_token)
573 - if param_end_idx != -1:
574 - # Complete parameter found
575 - param_value = value_text[:param_end_idx]
576 - if param_value.endswith("\n"):
577 - param_value = param_value[:-1]
578 -
579 - # Build complete JSON fragment for this parameter
580 - if self.param_count == 0:
581 - json_fragment = (
582 - '"' + self.current_param_name + '": "' +
583 - json.dumps(param_value)[1:-1] + '"')
538 + if (not self.in_param and self.param_count < len(param_starts)
539 + and len(param_starts) > self.param_count):
540 + # Process the next parameter
541 + param_idx = param_starts[self.param_count]
542 + param_start = param_idx + len(self.parameter_prefix)
543 + remaining = tool_text[param_start:]
544 +
545 + if ">" in remaining:
546 + # We have the complete parameter name
547 + name_end = remaining.find(">")
548 + self.current_param_name = remaining[:name_end]
549 +
550 + # Find the parameter value
551 + value_start = param_start + name_end + 1
552 + value_text = tool_text[value_start:]
553 + if value_text.startswith("\n"):
554 + value_text = value_text[1:]
555 +
556 + # Find where this parameter ends
557 + param_end_idx = value_text.find(self.parameter_end_token)
558 + if param_end_idx == -1:
559 + # No closing tag, look for next parameter or
560 + # function end
561 + next_param_idx = value_text.find(self.parameter_prefix)
562 + func_end_idx = value_text.find(self.function_end_token)
563 +
564 + if next_param_idx != -1 and (func_end_idx == -1
565 + or next_param_idx
566 + < func_end_idx):
567 + param_end_idx = next_param_idx
568 + elif func_end_idx != -1:
569 + param_end_idx = func_end_idx
570 + else:
571 + # Neither found, check if tool call is complete
572 + if self.tool_call_end_token in tool_text:
573 + # Tool call is complete, so parameter
574 + # must be complete too. Use all
575 + # remaining text before function end
576 + param_end_idx = len(value_text)
577 else:
585 - json_fragment = (
586 - ', "' + self.current_param_name + '": "' +
587 - json.dumps(param_value)[1:-1] + '"')
588 -
589 - self.param_count += 1
590 -
591 - return DeltaMessage(tool_calls=[
592 - DeltaToolCall(
593 - index=self.current_tool_index,
594 - function=DeltaFunctionCall(
595 - arguments=json_fragment),
596 - )
597 - ])
598 -
599 - # Continue parameter value
578 + # Still streaming, wait for more content
579 + return None
580 +
581 + if param_end_idx != -1:
582 + # Complete parameter found
583 + param_value = value_text[:param_end_idx]
584 + if param_value.endswith("\n"):
585 + param_value = param_value[:-1]
586 +
587 + # Store raw value for later processing
588 + self.accumulated_params[
589 + self.current_param_name] = param_value
590 +
591 + # Get parameter configuration for type conversion
592 + param_config = self._get_arguments_config(
593 + self.current_function_name or "",
594 + self.streaming_request.tools
595 + if self.streaming_request else None)
596 +
597 + # Convert param value to appropriate type
598 + converted_value = self._convert_param_value(
599 + param_value, self.current_param_name, param_config,
600 + self.current_function_name or "")
601 +
602 + # Build JSON fragment based on the converted type
603 + # Use json.dumps to properly serialize the value
604 + serialized_value = json.dumps(converted_value,
605 + ensure_ascii=False)
606 +
607 + if self.param_count == 0:
608 + json_fragment = (f'"{self.current_param_name}": '
609 + f'{serialized_value}')
610 + else:
611 + json_fragment = (f', "{self.current_param_name}": '
612 + f'{serialized_value}')
613 +
614 + self.param_count += 1
615 +
616 + return DeltaMessage(tool_calls=[
617 + DeltaToolCall(
618 + index=self.current_tool_index,
619 + function=DeltaFunctionCall(
620 + arguments=json_fragment),
621 + )
622 + ])
623 +
624 + # Continue parameter value - Not used in the current implementation
625 + # since we process complete parameters above
626 if self.in_param:
627 if self.parameter_end_token in delta_text:
628 # End of parameter
@@ -608,25 +634,42 @@ class Qwen3CoderToolParser(ToolParser):
634 gt_idx = value_chunk.find(">")
635 value_chunk = value_chunk[gt_idx + 1:]
636
611 - if (not self.current_param_value
612 - and value_chunk.startswith("\n")):
637 + if not self.current_param_value and value_chunk.startswith(
638 + "\n"):
639 value_chunk = value_chunk[1:]
640
615 - # Calculate incremental JSON
641 + # Store complete value
642 full_value = self.current_param_value + value_chunk
617 - prev_escaped = (json.dumps(self.current_param_value)[1:-1]
618 - if self.current_param_value else "")
619 - full_escaped = json.dumps(full_value)[1:-1]
620 - delta_escaped = full_escaped[len(prev_escaped):]
621 -
643 + self.accumulated_params[
644 + self.current_param_name] = full_value
645 +
646 + # Get parameter configuration for type conversion
647 + param_config = self._get_arguments_config(
648 + self.current_function_name or "",
649 + self.streaming_request.tools
650 + if self.streaming_request else None)
651 +
652 + # Convert the parameter value to the appropriate type
653 + converted_value = self._convert_param_value(
654 + full_value, self.current_param_name or "",
655 + param_config, self.current_function_name or "")
656 +
657 + # Serialize the converted value
658 + serialized_value = json.dumps(converted_value,
659 + ensure_ascii=False)
660 +
661 + # Since we've been streaming the quoted version,
662 + # we need to close it properly
663 + # This is complex - for now just complete the value
664 self.in_param = False
665 self.current_param_value = ""
666
667 + # Just close the current parameter string
668 return DeltaMessage(tool_calls=[
669 DeltaToolCall(
670 index=self.current_tool_index,
671 function=DeltaFunctionCall(
629 - arguments=delta_escaped + '"'),
672 + arguments='"'), # Close the string quote
673 )
674 ])
675 else:
@@ -638,18 +681,18 @@ class Qwen3CoderToolParser(ToolParser):
681 gt_idx = value_chunk.find(">")
682 value_chunk = value_chunk[gt_idx + 1:]
683
641 - if (not self.current_param_value
642 - and value_chunk.startswith("\n")):
684 + if not self.current_param_value and value_chunk.startswith(
685 + "\n"):
686 value_chunk = value_chunk[1:]
687
688 if value_chunk:
689 # Stream the escaped delta
647 - prev_escaped = (json.dumps(
648 - self.current_param_value)[1:-1]
649 - if self.current_param_value else "")
690 + prev_escaped = json.dumps(
691 + self.current_param_value, ensure_ascii=False
692 + )[1:-1] if self.current_param_value else ""
693 self.current_param_value += value_chunk
651 - full_escaped = json.dumps(
652 - self.current_param_value)[1:-1]
694 + full_escaped = json.dumps(self.current_param_value,
695 + ensure_ascii=False)[1:-1]
696 delta_escaped = full_escaped[len(prev_escaped):]
697
698 if delta_escaped:
@@ -661,4 +704,4 @@ class Qwen3CoderToolParser(ToolParser):
704 )
705 ])
706
664 - return None
707 + return None
\ No newline at end of file