@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9383cd6f10

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Frontend] Add a new xml-based tool parser for qwen3-coder (#25028)

Signed-off-by: Zhikaiiii <1658973216@qq.com>

Zhikaiiii committed Sep 23, 2025 at 16:07 UTC 9383cd6f10c55875de7f932e1bf7b0c43f01fc67

4 files changed +1238 -22

docs/features/tool_calling.md

@@ -319,6 +319,15 @@ Supported models:

319

320 Flags: `--tool-call-parser glm45`

321

322 +### Qwen3-Coder Models (`qwen3_xml`)

323 +

324 +Supported models:

325 +

326 +* `Qwen/Qwen3-480B-A35B-Instruct`

327 +* `Qwen/Qwen3-Coder-30B-A3B-Instruct`

328 +

329 +Flags: `--tool-call-parser qwen3_xml`

330 +

331 ### Models with Pythonic Tool Calls (`pythonic`)

332

333 A growing number of models output a python list to represent tool calls instead of using JSON. This has the advantage of inherently supporting parallel tool calls and removing ambiguity around the JSON schema required for tool calls. The `pythonic` tool parser can support such models.

tests/tool_use/test_qwen3coder_tool_parser.py

+90 -22

@@ -13,6 +13,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,

13 ToolCall)

14 from vllm.entrypoints.openai.tool_parsers.qwen3coder_tool_parser import (

15 Qwen3CoderToolParser)

16 +from vllm.entrypoints.openai.tool_parsers.qwen3xml_tool_parser import (

17 + Qwen3XMLToolParser)

18 from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally

19 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer

@@ -29,6 +31,21 @@ def qwen3_tool_parser(qwen3_tokenizer):

31 return Qwen3CoderToolParser(qwen3_tokenizer)

34 +@pytest.fixture

35 +def qwen3_xml_tool_parser(qwen3_tokenizer):

36 + return Qwen3XMLToolParser(qwen3_tokenizer)

37 +

38 +

39 +@pytest.fixture(params=["original", "xml"])

40 +def qwen3_tool_parser_parametrized(qwen3_tool_parser, qwen3_xml_tool_parser,

41 + request):

42 + """Parameterized fixture that provides both parser types for testing"""

43 + if request.param == "original":

44 + return qwen3_tool_parser

45 + else:

46 + return qwen3_xml_tool_parser

47 +

48 +

49 @pytest.fixture

50 def sample_tools():

51 return [

@@ -95,7 +112,7 @@ def assert_tool_calls(actual_tool_calls: list[ToolCall],

112

113

114 def stream_delta_message_generator(

98 - qwen3_tool_parser: Qwen3CoderToolParser,

115 + qwen3_tool_parser,

116 qwen3_tokenizer: AnyTokenizer,

117 model_output: str,

118 request: Optional[ChatCompletionRequest] = None

@@ -144,9 +161,9 @@ def stream_delta_message_generator(

161 read_offset = new_read_offset

162

163

147 -def test_extract_tool_calls_no_tools(qwen3_tool_parser):

164 +def test_extract_tool_calls_no_tools(qwen3_tool_parser_parametrized):

165 model_output = "This is a test response without any tool calls"

149 - extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(

166 + extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(

167 model_output, request=None) # type: ignore[arg-type]

168 assert not extracted_tool_calls.tools_called

169 assert extracted_tool_calls.tool_calls == []

@@ -294,12 +311,13 @@ circle

311 ], "Let me calculate that area for you."),

312 ],

313 )

297 -def test_extract_tool_calls(qwen3_tool_parser, sample_tools, model_output,

298 - expected_tool_calls, expected_content):

314 +def test_extract_tool_calls(qwen3_tool_parser_parametrized, sample_tools,

315 + model_output, expected_tool_calls,

316 + expected_content):

317 request = ChatCompletionRequest(model=MODEL,

318 messages=[],

319 tools=sample_tools)

302 - extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(

320 + extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(

321 model_output, request=request)

322 assert extracted_tool_calls.tools_called

323

@@ -308,7 +326,8 @@ def test_extract_tool_calls(qwen3_tool_parser, sample_tools, model_output,

326 assert extracted_tool_calls.content == expected_content

327

328

311 -def test_extract_tool_calls_fallback_no_tags(qwen3_tool_parser, sample_tools):

329 +def test_extract_tool_calls_fallback_no_tags(qwen3_tool_parser_parametrized,

330 + sample_tools):

331 """Test fallback parsing when XML tags are missing"""

332 model_output = '''<function=get_current_weather>

333 <parameter=city>

@@ -322,7 +341,7 @@ TX

341 request = ChatCompletionRequest(model=MODEL,

342 messages=[],

343 tools=sample_tools)

325 - extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(

344 + extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(

345 model_output, request=request)

346

347 assert extracted_tool_calls.tools_called

@@ -331,7 +350,7 @@ TX

350 "get_current_weather")

351

352

334 -def test_extract_tool_calls_type_conversion(qwen3_tool_parser):

353 +def test_extract_tool_calls_type_conversion(qwen3_tool_parser_parametrized):

354 """Test parameter type conversion based on tool schema"""

355 tools = [

356 ChatCompletionToolsParam(type="function",

@@ -381,7 +400,7 @@ hello world

400 </tool_call>'''

401

402 request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)

384 - extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(

403 + extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(

404 model_output, request=request)

405

406 args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)

@@ -536,9 +555,10 @@ circle

555 ], "Let me calculate that area for you."),

556 ],

557 )

539 -def test_extract_tool_calls_streaming(qwen3_tool_parser, qwen3_tokenizer,

540 - sample_tools, model_output,

541 - expected_tool_calls, expected_content):

558 +def test_extract_tool_calls_streaming(qwen3_tool_parser_parametrized,

559 + qwen3_tokenizer, sample_tools,

560 + model_output, expected_tool_calls,

561 + expected_content):

562 """Test incremental streaming behavior including typed parameters"""

563 request = ChatCompletionRequest(model=MODEL,

564 messages=[],

@@ -548,7 +568,8 @@ def test_extract_tool_calls_streaming(qwen3_tool_parser, qwen3_tokenizer,

568 tool_states = {} # Track state per tool index

569

570 for delta_message in stream_delta_message_generator(

551 - qwen3_tool_parser, qwen3_tokenizer, model_output, request):

571 + qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output,

572 + request):

573 # role should never be streamed from tool parser

574 assert not delta_message.role

575

@@ -609,7 +630,7 @@ def test_extract_tool_calls_streaming(qwen3_tool_parser, qwen3_tokenizer,

630

631

632 def test_extract_tool_calls_missing_closing_parameter_tag(

612 - qwen3_tool_parser, sample_tools):

633 + qwen3_tool_parser_parametrized, sample_tools):

634 """Test handling of missing closing </parameter> tag"""

635 # Using get_current_weather from sample_tools but with malformed XML

636 model_output = '''Let me check the weather for you:

@@ -629,7 +650,7 @@ fahrenheit

650 request = ChatCompletionRequest(model=MODEL,

651 messages=[],

652 tools=sample_tools)

632 - extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(

653 + extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(

654 model_output, request=request)

655

656 # The parser should handle the malformed XML gracefully

@@ -652,7 +673,7 @@ fahrenheit

673

674

675 def test_extract_tool_calls_streaming_missing_closing_tag(

655 - qwen3_tool_parser, qwen3_tokenizer, sample_tools):

676 + qwen3_tool_parser_parametrized, qwen3_tokenizer, sample_tools):

677 """Test streaming with missing closing </parameter> tag"""

678 # Using get_current_weather from sample_tools but with malformed XML

679 model_output = '''Let me check the weather for you:

@@ -677,7 +698,8 @@ fahrenheit

698 tool_states = {}

699

700 for delta_message in stream_delta_message_generator(

680 - qwen3_tool_parser, qwen3_tokenizer, model_output, request):

701 + qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output,

702 + request):

703

704 if delta_message.content:

705 other_content += delta_message.content

@@ -727,9 +749,8 @@ fahrenheit

749 assert args["unit"] == "fahrenheit"

750

751

730 -def test_extract_tool_calls_streaming_incremental(qwen3_tool_parser,

731 - qwen3_tokenizer,

732 - sample_tools):

752 +def test_extract_tool_calls_streaming_incremental(

753 + qwen3_tool_parser_parametrized, qwen3_tokenizer, sample_tools):

754 """Test that streaming is truly incremental"""

755 model_output = '''I'll check the weather.<tool_call>

756 <function=get_current_weather>

@@ -748,7 +769,8 @@ TX

769

770 chunks = []

771 for delta_message in stream_delta_message_generator(

751 - qwen3_tool_parser, qwen3_tokenizer, model_output, request):

772 + qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output,

773 + request):

774 chunks.append(delta_message)

775

776 # Should have multiple chunks

@@ -784,3 +806,49 @@ TX

806 parsed_args = json.loads(full_args)

807 assert parsed_args["city"] == "Dallas"

808 assert parsed_args["state"] == "TX"

809 +

810 +

811 +def test_extract_tool_calls_complex_type_with_single_quote(

812 + qwen3_tool_parser_parametrized):

813 + """Test parameter type conversion based on tool schema"""

814 + tools = [

815 + ChatCompletionToolsParam(type="function",

816 + function={

817 + "name": "test_types",

818 + "parameters": {

819 + "type": "object",

820 + "properties": {

821 + "int_param": {

822 + "type": "integer"

823 + },

824 + "float_param": {

825 + "type": "float"

826 + },

827 + "bool_param": {

828 + "type": "boolean"

829 + },

830 + "str_param": {

831 + "type": "string"

832 + },

833 + "obj_param": {

834 + "type": "object"

835 + }

836 + }

837 + }

838 + })

839 + ]

840 +

841 + model_output = '''<tool_call>

842 +<function=test_types>

843 +<parameter=obj_param>

844 +{'key': 'value'}

845 +</parameter>

846 +</function>

847 +</tool_call>'''

848 +

849 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)

850 + extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(

851 + model_output, request=request)

852 +

853 + args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)

854 + assert args["obj_param"] == {"key": "value"}

vllm/entrypoints/openai/tool_parsers/__init__.py

@@ -20,6 +20,7 @@ from .openai_tool_parser import OpenAIToolParser

20 from .phi4mini_tool_parser import Phi4MiniJsonToolParser

21 from .pythonic_tool_parser import PythonicToolParser

22 from .qwen3coder_tool_parser import Qwen3CoderToolParser

23 +from .qwen3xml_tool_parser import Qwen3XMLToolParser

24 from .seed_oss_tool_parser import SeedOssToolParser

25 from .step3_tool_parser import Step3ToolParser

26 from .xlam_tool_parser import xLAMToolParser

@@ -45,6 +46,7 @@ __all__ = [

46 "HunyuanA13BToolParser",

47 "Glm4MoeModelToolParser",

48 "Qwen3CoderToolParser",

49 + "Qwen3XMLToolParser",

50 "SeedOssToolParser",

51 "Step3ToolParser",

52 "OpenAIToolParser",

vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py new

+1137

@@ -0,0 +1,1137 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import ast

4 +import json

5 +import uuid

6 +from collections.abc import Sequence

7 +from typing import Any, Optional, Union

8 +from xml.parsers.expat import ParserCreate

9 +

10 +import regex as re

11 +

12 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,

13 + ChatCompletionToolsParam,

14 + DeltaFunctionCall, DeltaMessage,

15 + DeltaToolCall,

16 + ExtractedToolCallInformation,

17 + FunctionCall, ToolCall)

18 +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

19 + ToolParser, ToolParserManager)

20 +from vllm.logger import init_logger

21 +from vllm.transformers_utils.tokenizer import AnyTokenizer

22 +

23 +logger = init_logger(__name__)

24 +

25 +

26 +class StreamingXMLToolCallParser:

27 + """

28 + Simplified streaming XML tool call parser

29 + Supports streaming input, parsing, and output

30 + """

31 +

32 + def __init__(self):

33 + self.reset_streaming_state()

34 +

35 + # Tool configuration information

36 + self.tools: Union[list[ChatCompletionToolsParam], None] = None

37 + self.tool_call_start_token: str = '<tool_call>'

38 + self.tool_call_end_token: str = '</tool_call>'

39 + self.function_start_token: str = '<function='

40 + self.function_end_token: str = '</function>'

41 + self.parameter_start_token: str = '<parameter='

42 + self.parameter_end_token: str = '</parameter>'

43 +

44 + def reset_streaming_state(self):

45 + """Reset streaming parsing state"""

46 +

47 + self.deltas = []

48 + # state for streaming

49 + self.tool_call_index = 0

50 + self.current_call_id = None

51 + self.last_completed_call_id = None

52 + self.current_function_name = None

53 + self.current_function_open = False

54 + self.parameters = {}

55 + self.current_param_name = None

56 + self.current_param_value = ''

57 + self.current_param_value_converted = ''

58 + self.current_param_is_first = False

59 + self.should_emit_end_newline = False

60 + self.start_quote_emitted = False

61 +

62 + self.streaming_buffer = ''

63 + self.last_processed_pos = 0

64 +

65 + self.text_content_buffer = ''

66 +

67 + # state for preprocessing and deferred parsing

68 + self._pre_inside_parameter = False

69 + self._pre_param_buffer = ""

70 + self._pre_current_param_name = None

71 + self.defer_current_parameter = False

72 + self.deferred_param_raw_value = ""

73 +

74 + # recreate parser

75 + self.parser = ParserCreate()

76 + self.setup_parser()

77 +

78 + def parse_single_streaming_chunks(self, xml_chunk: str) -> DeltaMessage:

79 + """

80 + Parse single streaming XML chunk and return Delta response

81 + This is the actual streaming interface that receives chunks

82 + one by one and maintains internal state

83 +

84 + Args:

85 + xml_chunk: Single XML chunk string

86 + Returns:

87 + DeltaMessage: Contains delta information generated by this chunk,

88 + returns empty response if no complete elements

89 + """

90 + # Record delta count before processing

91 + initial_delta_count = len(self.deltas)

92 +

93 + self.streaming_buffer += xml_chunk

94 +

95 + found_elements = self._process_complete_xml_elements()

96 +

97 + if found_elements:

98 + # If complete elements found, check if end events were missed

99 + # some tags may not have been triggered

100 + try:

101 + new_deltas = self.deltas[initial_delta_count:]

102 + # If this chunk contains </function>

103 + # but didn't generate '}', then complete it

104 + if (self.current_call_id is not None

105 + and self.function_end_token in xml_chunk):

106 +

107 + # - Added '}' (non-empty parameter ending)

108 + # - Added '{}' (empty parameter function)

109 + has_function_close = any((td.tool_calls and any(

110 + (tc.function and tc.id == self.current_call_id

111 + and isinstance(tc.function.arguments, str) and

112 + (tc.function.arguments in ('}', '{}')))

113 + for tc in td.tool_calls)) for td in new_deltas)

114 + if not has_function_close:

115 + # Close potentially unclosed element

116 + if self.current_param_name:

117 + self._end_element('parameter')

118 + if self.current_function_name:

119 + self._end_element('function')

120 + # If this chunk contains </tool_call>

121 + # but didn't generate final empty delta, then complete it

122 + if (self.current_call_id is not None

123 + and self.tool_call_end_token in xml_chunk):

124 + has_toolcall_close = any((td.tool_calls and any(

125 + (tc.type == 'function' and tc.function and tc.function.

126 + arguments == '' and tc.id == self.current_call_id)

127 + for tc in td.tool_calls)) for td in new_deltas)

128 + if not has_toolcall_close:

129 + # Close potentially unclosed element

130 + if self.current_param_name:

131 + self._end_element('parameter')

132 + if self.current_function_name:

133 + self._end_element('function')

134 + self._end_element('tool_call')

135 + except Exception as e:

136 + logger.warning("Error with fallback parsing: %s", e)

137 + # Merge newly generated deltas into single response

138 + result_delta = self._merge_new_deltas_to_single_response(

139 + initial_delta_count)

140 + return result_delta

141 + else:

142 + # No complete elements, check if there's unoutput text content

143 + if self.text_content_buffer and self.tool_call_index == 0:

144 + # Has text content but no tool_call yet, output text content

145 + text_delta = DeltaMessage(content=self.text_content_buffer)

146 + self._emit_delta(text_delta)

147 + # Clear buffer to avoid duplicate output

148 + self.text_content_buffer = ''

149 + return text_delta

150 +

151 + # If this chunk contains end tags but wasn't triggered by parser,

152 + # manually complete end events

153 + # Only execute when still on the same call as when entered,

154 + # to prevent accidentally closing new calls

155 + # in multi <tool_call> scenarios

156 + if (self.current_call_id is not None

157 + and (self.function_end_token in xml_chunk

158 + or self.tool_call_end_token in xml_chunk)):

159 + # Close potentially unclosed element

160 + if self.current_param_name:

161 + self._end_element('parameter')

162 + if self.function_end_token in xml_chunk and \

163 + self.current_function_name:

164 + self._end_element('function')

165 + if self.tool_call_end_token in xml_chunk:

166 + self._end_element('tool_call')

167 + # Return the merged delta result generated by this fallback

168 + result_delta = self._merge_new_deltas_to_single_response(

169 + initial_delta_count)

170 + return result_delta

171 +

172 + # No complete elements, return empty response

173 + return DeltaMessage(content=None)

174 +

175 + def _escape_xml_special_chars(self, text: str) -> str:

176 + """

177 + Escape XML special characters

178 + Args:

179 + text: Original text

180 + Returns:

181 + Escaped text

182 + """

183 + xml_escapes = {

184 + '&': '&',

185 + '<': '<',

186 + '>': '>',

187 + '"': '"',

188 + "'": '''

189 + }

190 +

191 + for char, escape in xml_escapes.items():

192 + text = text.replace(char, escape)

193 +

194 + return text

195 +

196 + def _process_complete_xml_elements(self) -> bool:

197 + """

198 + Process complete XML elements in buffer

199 +

200 + Returns:

201 + bool: Whether complete elements were found and processed

202 + """

203 + found_any = False

204 +

205 + while self.last_processed_pos < len(self.streaming_buffer):

206 + # Find next complete xml element

207 + element, end_pos = self._find_next_complete_element(

208 + self.last_processed_pos)

209 + if element is None:

210 + # No complete element found, wait for more data

211 + break

212 +

213 + # Check if this element should be skipped

214 + if self._should_skip_element(element):

215 + self.last_processed_pos = end_pos

216 + continue

217 +

218 + # Found complete XML element, process it

219 + try:

220 + preprocessed_element = self._preprocess_xml_chunk(element)

221 + # Check if this is the first tool_call start

222 + if ((preprocessed_element.strip().startswith('<tool_call>') or

223 + preprocessed_element.strip().startswith('<function name=')

224 + ) and self.tool_call_index

225 + == 0) and self.text_content_buffer:

226 + # First tool_call starts,

227 + # output previously collected text content first

228 + text_delta = DeltaMessage(content=self.text_content_buffer)

229 + self._emit_delta(text_delta)

230 + # Clear buffer for potential subsequent text content

231 + self.text_content_buffer = ''

232 +

233 + # If a new tool_call starts and

234 + # there are already completed tool_calls

235 + if (preprocessed_element.strip().startswith('<tool_call>')

236 + and self.tool_call_index > 0 and self.current_call_id):

237 + # Reset parser state but preserve generated deltas

238 + if self.current_param_name:

239 + self._end_element('parameter')

240 + if self.current_function_open or self.current_function_name:

241 + self._end_element('function')

242 + # Output final tool_call tail delta

243 + final_delta = DeltaMessage(

244 + role=None,

245 + content=None,

246 + reasoning_content=None,

247 + tool_calls=[

248 + DeltaToolCall(index=self.tool_call_index - 1,

249 + id=self.current_call_id,

250 + type='function',

251 + function=DeltaFunctionCall(

252 + name=None, arguments=''))

253 + ])

254 + self._emit_delta(final_delta)

255 + # Reset XML parser and current call state

256 + self._reset_xml_parser_after_tool_call()

257 + # Parse preprocessed element

258 + self.parser.Parse(preprocessed_element, False)

259 + found_any = True

260 +

261 + except Exception as e:

262 + logger.warning("Error when parsing XML elements: %s", e)

263 +

264 + # Update processed position

265 + self.last_processed_pos = end_pos

266 +

267 + return found_any

268 +

269 + def _should_skip_element(self, element: str) -> bool:

270 + """

271 + Determine whether an element should be skipped

272 +

273 + Args:

274 + element: Element to evaluate

275 +

276 + Returns:

277 + bool: True means should skip, False means should process

278 + """

279 +

280 + # If it's a tool_call XML tag, don't skip

281 + if element.startswith(

282 + self.tool_call_start_token) or element.startswith(

283 + self.function_start_token) or element.startswith(

284 + self.parameter_start_token):

285 + return False

286 +

287 + # If currently not parsing tool calls and not blank,

288 + # collect this text instead of skipping

289 + # Only process other XML elements after tool_call appears,

290 + # otherwise treat as plain text

291 + if self.current_call_id is None and element:

292 + # Collect text content to buffer

293 + self.text_content_buffer += element

294 + return True # Still skip, but content has been collected

295 +

296 + # If currently parsing tool calls,

297 + # this might be parameter value, don't skip

298 + if self.current_call_id is not None:

299 + return False

300 +

301 + # Skip blank content

302 + return not element

303 +

304 + def _find_next_complete_element(

305 + self, start_pos: int) -> tuple[Optional[str], int]:

306 + """

307 + Find next complete XML element from specified position

308 +

309 + Args:

310 + start_pos: Position to start searching

311 +

312 + Returns:

313 + (Complete element string, element end position),

314 + returns (None, start_pos) if no complete element found

315 + """

316 + buffer = self.streaming_buffer[start_pos:]

317 +

318 + if not buffer:

319 + return None, start_pos

320 +

321 + if buffer.startswith('<'):

322 + # Need to ensure no new < appears,

323 + # find the nearest one between < and >

324 + tag_end = buffer.find('<', 1)

325 + tag_end2 = buffer.find('>', 1)

326 + if tag_end != -1 and tag_end2 != -1:

327 + # Next nearest is <

328 + if tag_end < tag_end2:

329 + return buffer[:tag_end], start_pos + tag_end

330 + # Next nearest is >, means found XML element

331 + else:

332 + return buffer[:tag_end2 + 1], start_pos + tag_end2 + 1

333 + elif tag_end != -1:

334 + return buffer[:tag_end], start_pos + tag_end

335 + elif tag_end2 != -1:

336 + return buffer[:tag_end2 + 1], start_pos + tag_end2 + 1

337 + else:

338 + # If currently not parsing tool calls (entering a tool_call),

339 + # check if starts with <tool_call>

340 + if self.current_call_id is None:

341 + # Check if might be start of <tool_call>

342 + if buffer == '<tool_call>'[:len(buffer)]:

343 + # Might be start of <tool_call>, wait for more data

344 + return None, start_pos

345 + else:

346 + # Not start of <tool_call>, treat as text

347 + return buffer, start_pos + len(buffer)

348 + else:

349 + # When parsing tool calls,

350 + # wait for more data to get complete tag

351 + return None, start_pos

352 + else:

353 + # Find text content (until next < or buffer end)

354 + next_tag_pos = buffer.find('<')

355 + if next_tag_pos != -1:

356 + # Found text content

357 + text_content = buffer[:next_tag_pos]

358 + return text_content, start_pos + next_tag_pos

359 + else:

360 + # Buffer end is all text, process

361 + # (no longer wait for more data)

362 + remaining = buffer

363 + return remaining, start_pos + len(remaining)

364 +

365 + def _merge_new_deltas_to_single_response(

366 + self, initial_count: int) -> DeltaMessage:

367 + """

368 + Merge newly generated deltas from this processing

369 + into a single DeltaMessage

370 +

371 + Args:

372 + initial_count: Delta count before processing

373 +

374 + Returns:

375 + Merged DeltaMessage containing all newly generated delta information

376 + """

377 + if len(self.deltas) <= initial_count:

378 + return DeltaMessage(content=None)

379 +

380 + # Get newly generated deltas

381 + new_deltas = self.deltas[initial_count:]

382 +

383 + if len(new_deltas) == 1:

384 + # Only one new delta, return directly

385 + return new_deltas[0]

386 +

387 + # Merge multiple new deltas

388 + merged_tool_calls: list[DeltaToolCall] = []

389 + merged_content: str = ''

390 +

391 + for delta in new_deltas:

392 + if delta.content:

393 + merged_content += delta.content

394 + if delta.tool_calls:

395 + # For tool_calls, we need to intelligently merge arguments

396 + for tool_call in delta.tool_calls:

397 + # Find if there's already a tool_call with the same call_id

398 + existing_call = None

399 + for existing in merged_tool_calls:

400 + if existing.id == tool_call.id:

401 + existing_call = existing

402 + break

403 +

404 + if existing_call and existing_call.function:

405 + # Merge to existing tool_call

406 + if tool_call.function and tool_call.function.name:

407 + existing_call.function.name = \

408 + tool_call.function.name

409 + if tool_call.function \

410 + and tool_call.function.arguments is not None:

411 + if existing_call.function.arguments is None:

412 + existing_call.function.arguments = ''

413 +

414 + # For streaming JSON parameters,

415 + # simply concatenate in order

416 + new_args = tool_call.function.arguments

417 + existing_call.function.arguments += new_args

418 + if tool_call.type:

419 + existing_call.type = tool_call.type

420 + else:

421 + # Add new tool_call

422 + merged_tool_calls.append(tool_call)

423 +

424 + return DeltaMessage(content=merged_content if merged_content else None,

425 + tool_calls=merged_tool_calls)

426 +

427 + def _preprocess_xml_chunk(self, chunk: str) -> str:

428 + """

429 + Preprocess XML chunk, handle non-standard formats,

430 + and escape special characters

431 +

432 + Args:

433 + chunk: Original XML chunk

434 +

435 + Returns:

436 + Processed XML chunk

437 + """

438 +

439 + # Check if this is a tool_call related element

440 + is_tool_call = False

441 + if chunk.startswith(self.tool_call_start_token) or chunk.startswith(

442 + self.tool_call_end_token):

443 + is_tool_call = True

444 + if chunk.startswith(self.function_start_token) or chunk.startswith(

445 + self.function_end_token):

446 + is_tool_call = True

447 + if chunk.startswith(self.parameter_start_token) or chunk.startswith(

448 + self.parameter_end_token):

449 + is_tool_call = True

450 + # Handle <function=name> format -> <function name="name">

451 + processed = re.sub(r'<function=([^>]+)>', r'<function name="\1">',

452 + chunk)

453 + # Handle <parameter=name> format -> <parameter name="name">

454 + processed = re.sub(r'<parameter=([^>]+)>', r'<parameter name="\1">',

455 + processed)

456 +

457 + original_chunk = chunk

458 + # If in parameter value accumulation mode

459 + if self._pre_inside_parameter:

460 + # Parameter end: output accumulated raw text

461 + # safely then return </parameter>

462 + if processed.startswith('</parameter>'):

463 + body_text = self._pre_param_buffer

464 + # Trigger deferred parsing mode

465 + # literal_eval+json output in end_element

466 + self.defer_current_parameter = True

467 + self.deferred_param_raw_value = body_text

468 + # Clean up state

469 + self._pre_inside_parameter = False

470 + self._pre_param_buffer = ""

471 + self._pre_current_param_name = None

472 + safe_text = self._escape_xml_special_chars(body_text)

473 + return f"{safe_text}</parameter>"

474 + else:

475 + # If this is the first block of content after entering parameter

476 + # evaluate if deferred parsing is needed;

477 + # If not needed, exit accumulation mode

478 + # and pass through directly

479 + if self._pre_param_buffer == "":

480 + # Get current parameter type

481 + param_type = self._get_param_type(

482 + self._pre_current_param_name

483 + ) if self._pre_current_param_name else 'string'

484 + # Only these types need deferred parsing to

485 + # handle Python literals containing single quotes

486 + is_object_type = param_type in ["object"]

487 + is_complex_type = (param_type

488 + in ["array", "arr", "sequence"]

489 + or param_type.startswith("dict")

490 + or param_type.startswith("list"))

491 +

492 + # Only delay when contains container symbols

493 + # and has single quotes and is complex type

494 + has_container_hint = ('[' in original_chunk) or (

495 + '{' in original_chunk) or ('(' in original_chunk)

496 +

497 + # Determine if deferred parsing is needed

498 + need_defer = False

499 + if is_complex_type:

500 + # Complex type, always need deferred parsing

501 + need_defer = True

502 + elif is_object_type and has_container_hint and (

503 + "'" in original_chunk):

504 + # Object type with container symbols

505 + # and single quotes, need deferred parsing

506 + need_defer = True

507 +

508 + if not need_defer:

509 + # No need for deferred parsing,

510 + # exit parameter mode directly

511 + self._pre_inside_parameter = False

512 + return self._escape_xml_special_chars(original_chunk)

513 + self._pre_param_buffer += original_chunk

514 + return ""

515 +

516 + # Parameter start: enable accumulation

517 + if processed.startswith('<parameter name='):

518 + m = re.match(r'<parameter name="([^"]+)">', processed)

519 + if m:

520 + self._pre_current_param_name = m.group(1)

521 + self._pre_inside_parameter = True

522 + self._pre_param_buffer = ""

523 + return processed

524 +

525 + # If processed doesn't contain special_token, escape processed

526 + # This is because XML parsing encounters special characters

527 + # and reports errors, so escaping is needed

528 + if not is_tool_call:

529 + processed = self._escape_xml_special_chars(processed)

530 + return processed

531 +

532 + def _emit_delta(self, delta: DeltaMessage):

533 + """Emit Delta response (streaming output)"""

534 + self.deltas.append(delta)

535 +

536 + def _auto_close_open_parameter_if_needed(self,

537 + incoming_tag: Optional[str] = None

538 + ):

539 + """Before starting to process new elements,

540 + if there are unclosed tags from before,

541 + automatically complete their endings to the parser.

542 + - If there are unclosed parameters,

543 + it's equivalent to feeding `</parameter>`

544 + - When about to start a new function or tool_call,

545 + if there are unclosed functions, complete `</function>`.

546 + - When about to start a new tool_call,

547 + if there are unclosed tool_calls, complete `</tool_call>`.

548 + """

549 + # First close unclosed parameters

550 + if self.current_param_name:

551 + self._end_element('parameter')

552 +

553 + # If about to start new function or tool_call,

554 + # and there are unclosed functions, close function first

555 + if incoming_tag in ('function',

556 + 'tool_call') and self.current_function_name:

557 + self._end_element('function')

558 +

559 + # If about to start new tool_call,

560 + # and there are unclosed tool_calls, close tool_call first

561 + if incoming_tag == 'tool_call' and self.current_call_id:

562 + self._end_element('tool_call')

563 +

564 + def _start_element(self, name: str, attrs: dict[str, str]):

565 + """Handle XML start element events"""

566 +

567 + if name == 'root':

568 + return

569 +

570 + if name == 'tool_call':

571 + # Before opening new tool_call,

572 + # automatically complete previous unclosed tags

573 + self._auto_close_open_parameter_if_needed('tool_call')

574 +

575 + self.parameters = {}

576 + self.current_call_id = self._get_next_call_id()

577 + self.current_param_is_first = True

578 + self.tool_call_index += 1

579 + elif name.startswith('function') or (name == 'function'):

580 + # If missing tool_call, manually complete

581 + if not self.current_call_id:

582 + self._start_element('tool_call', {})

583 + # Before opening new function,

584 + # automatically complete previous unclosed tags (parameter/function)

585 + self._auto_close_open_parameter_if_needed('function')

586 + function_name = self._extract_function_name(name, attrs)

587 + self.current_function_name = function_name

588 + self.current_function_open = True

589 + if function_name:

590 + delta = DeltaMessage(tool_calls=[

591 + DeltaToolCall(index=self.tool_call_index - 1,

592 + id=self.current_call_id,

593 + type='function',

594 + function=DeltaFunctionCall(

595 + name=function_name, arguments=''))

596 + ])

597 + self._emit_delta(delta)

598 + elif name.startswith('parameter') or (name == 'parameter'):

599 + # If previous parameter hasn't ended normally,

600 + # complete its end first, then start new parameter

601 + self._auto_close_open_parameter_if_needed('parameter')

602 + param_name = self._extract_parameter_name(name, attrs)

603 + self.current_param_name = param_name

604 + self.current_param_value = ''

605 + self.current_param_value_converted = ''

606 + self.start_quote_emitted = False # Reset start quote flag

607 +

608 + # Only output parameter name and colon,

609 + # don't output quotes

610 + # decide after parameter value type is determined

611 + if param_name:

612 + if not self.parameters:

613 + # First parameter

614 + # start JSON, only output parameter name and colon

615 + json_start = f'{{"{param_name}": '

616 + delta = DeltaMessage(tool_calls=[

617 + DeltaToolCall(index=self.tool_call_index - 1,

618 + id=self.current_call_id,

619 + type='function',

620 + function=DeltaFunctionCall(

621 + name=None, arguments=json_start))

622 + ])

623 + self._emit_delta(delta)

624 + self.current_param_is_first = True

625 + else:

626 + # Subsequent parameters

627 + # add comma and parameter name, no quotes

628 + json_continue = f', "{param_name}": '

629 + delta = DeltaMessage(tool_calls=[

630 + DeltaToolCall(index=self.tool_call_index - 1,

631 + id=self.current_call_id,

632 + type='function',

633 + function=DeltaFunctionCall(

634 + name=None, arguments=json_continue))

635 + ])

636 + self._emit_delta(delta)

637 + self.current_param_is_first = False

638 +

639 + def _char_data(self, data: str):

640 + """Handle XML character data events"""

641 + if data and self.current_param_name:

642 + # If preprocessing stage determines deferred parsing is needed,

643 + # only cache character data, no streaming output

644 + if self.defer_current_parameter:

645 + original_data = data

646 + if self.should_emit_end_newline:

647 + original_data = '\n' + original_data

648 + self.should_emit_end_newline = False

649 + if original_data.endswith('\n'):

650 + self.should_emit_end_newline = True

651 + original_data = original_data[:-1]

652 + self.current_param_value += original_data

653 + return

654 +

655 + param_type = self._get_param_type(self.current_param_name)

656 +

657 + # Check if this is the first time receiving data for this parameter

658 + # If this is the first packet of data and starts with \n, remove \n

659 + if not self.current_param_value and data.startswith('\n'):

660 + data = data[1:]

661 +

662 + # Output start quote for string type (if not already output)

663 + if (param_type

664 + in ['string', 'str', 'text', 'varchar', 'char', 'enum']

665 + and not self.start_quote_emitted):

666 + quote_delta = DeltaMessage(tool_calls=[

667 + DeltaToolCall(index=self.tool_call_index - 1,

668 + id=self.current_call_id,

669 + type='function',

670 + function=DeltaFunctionCall(name=None,

671 + arguments='"'))

672 + ])

673 + self._emit_delta(quote_delta)

674 + self.start_quote_emitted = True

675 +

676 + if not data:

677 + return

678 +

679 + original_data = data

680 + # Delay output of trailing newline

681 + if self.should_emit_end_newline:

682 + original_data = '\n' + original_data

683 + self.should_emit_end_newline = False

684 + if original_data.endswith('\n'):

685 + self.should_emit_end_newline = True

686 + original_data = original_data[:-1]

687 + self.current_param_value += original_data

688 +

689 + # convert parameter value by param_type

690 + converted_value = self._convert_param_value(

691 + self.current_param_value, param_type)

692 + output_data = self._convert_for_json_streaming(

693 + converted_value, param_type)

694 +

695 + delta_data = output_data[len(self.current_param_value_converted):]

696 + self.current_param_value_converted = output_data

697 +

698 + delta = DeltaMessage(tool_calls=[

699 + DeltaToolCall(index=self.tool_call_index - 1,

700 + id=self.current_call_id,

701 + type='function',

702 + function=DeltaFunctionCall(name=None,

703 + arguments=delta_data))

704 + ])

705 + self._emit_delta(delta)

706 +

707 + def _end_element(self, name: str):

708 + """Handle XML end element events"""

709 +

710 + if name == 'root':

711 + return

712 +

713 + # If function or tool_call ends and there are still unclosed parameters,

714 + # complete parameter end first

715 + if (name.startswith('function') or name == 'function'

716 + or name == 'tool_call') and self.current_param_name:

717 + self._auto_close_open_parameter_if_needed()

718 +

719 + if (name.startswith('parameter')

720 + or name == 'parameter') and self.current_param_name:

721 + # End current parameter

722 + param_name = self.current_param_name

723 + param_value = self.current_param_value

724 +

725 + # If in deferred parsing mode,

726 + # perform overall parsing on raw content

727 + # accumulated in preprocessing stage and output once

728 + if self.defer_current_parameter:

729 + raw_text = self.deferred_param_raw_value \

730 + if self.deferred_param_raw_value else param_value

731 + parsed_value = None

732 + output_arguments = None

733 + try:

734 + # If previously delayed trailing newline,

735 + # add it back before parsing

736 + if self.should_emit_end_newline:

737 + raw_for_parse = raw_text + '\n'

738 + else:

739 + raw_for_parse = raw_text

740 + parsed_value = ast.literal_eval(raw_for_parse)

741 + output_arguments = json.dumps(parsed_value,

742 + ensure_ascii=False)

743 + except Exception:

744 + # Fallback: output as string as-is

745 + output_arguments = json.dumps(raw_text, ensure_ascii=False)

746 + parsed_value = raw_text

747 +

748 + delta = DeltaMessage(tool_calls=[

749 + DeltaToolCall(index=self.tool_call_index - 1,

750 + id=self.current_call_id,

751 + type='function',

752 + function=DeltaFunctionCall(

753 + name=None, arguments=output_arguments))

754 + ])

755 + self._emit_delta(delta)

756 +

757 + # Clean up and store

758 + self.should_emit_end_newline = False

759 + self.parameters[param_name] = parsed_value

760 + self.current_param_name = None

761 + self.current_param_value = ""

762 + self.current_param_value_converted = ""

763 + self.start_quote_emitted = False

764 + self.defer_current_parameter = False

765 + self.deferred_param_raw_value = ""

766 + return

767 +

768 + param_type = self._get_param_type(param_name)

769 +

770 + # convert complete parameter value by param_type

771 + converted_value = self._convert_param_value(

772 + param_value, param_type)

773 +

774 + # Decide whether to add end quote based on parameter type

775 + if param_type in [

776 + 'string', 'str', 'text', 'varchar', 'char', 'enum'

777 + ]:

778 + # For empty string parameters, need special handling

779 + if not param_value and not self.start_quote_emitted:

780 + # No start quote output,

781 + # directly output complete empty string

782 + delta = DeltaMessage(tool_calls=[

783 + DeltaToolCall(index=self.tool_call_index - 1,

784 + id=self.current_call_id,

785 + type='function',

786 + function=DeltaFunctionCall(

787 + name=None, arguments='""'))

788 + ])

789 + self._emit_delta(delta)

790 + else:

791 + # Non-empty parameter value, output end quote

792 + delta = DeltaMessage(tool_calls=[

793 + DeltaToolCall(index=self.tool_call_index - 1,

794 + id=self.current_call_id,

795 + type='function',

796 + function=DeltaFunctionCall(

797 + name=None, arguments='"'))

798 + ])

799 + self._emit_delta(delta)

800 +

801 + self.should_emit_end_newline = False

802 + # Store converted value

803 + self.parameters[param_name] = converted_value

804 + self.current_param_name = None

805 + self.current_param_value = ''

806 + self.current_param_value_converted = ''

807 + self.start_quote_emitted = False

808 +

809 + elif name.startswith('function') or name == 'function':

810 + # if there are parameters, close JSON object

811 + if self.parameters:

812 + delta = DeltaMessage(tool_calls=[

813 + DeltaToolCall(index=self.tool_call_index - 1,

814 + id=self.current_call_id,

815 + type='function',

816 + function=DeltaFunctionCall(name=None,

817 + arguments='}'))

818 + ])

819 + self._emit_delta(delta)

820 + # return empty object

821 + else:

822 + delta = DeltaMessage(tool_calls=[

823 + DeltaToolCall(index=self.tool_call_index - 1,

824 + id=self.current_call_id,

825 + type='function',

826 + function=DeltaFunctionCall(name=None,

827 + arguments='{}'))

828 + ])

829 + self._emit_delta(delta)

830 + self.current_function_open = False

831 +

832 + elif name == 'tool_call':

833 + # Before ending tool_call,

834 + # ensure function is closed to complete missing right brace

835 + if self.current_function_open:

836 + # If there are still unclosed parameters, close them first

837 + if self.current_param_name:

838 + self._end_element('parameter')

839 + # Close function, ensure output '}' or '{}'

840 + self._end_element('function')

841 + # Final Delta

842 + delta = DeltaMessage(tool_calls=[

843 + DeltaToolCall(index=self.tool_call_index - 1,

844 + id=self.current_call_id,

845 + type='function',

846 + function=DeltaFunctionCall(name=None,

847 + arguments=''))

848 + ])

849 + self._emit_delta(delta)

850 +

851 + # Check if there's text content to output (between tool_calls)

852 + if self.text_content_buffer.strip():

853 + text_delta = DeltaMessage(content=self.text_content_buffer)

854 + self._emit_delta(text_delta)

855 +

856 + self._reset_xml_parser_after_tool_call()

857 +

858 + def setup_parser(self):

859 + """Set up XML parser event handlers"""

860 + self.parser.buffer_text = True

861 + self.parser.StartElementHandler = self._start_element

862 + self.parser.EndElementHandler = self._end_element

863 + self.parser.CharacterDataHandler = self._char_data

864 +

865 + def set_tools(self, tools: Union[list[ChatCompletionToolsParam], None]):

866 + """Set tool configuration information"""

867 + self.tools = tools

868 +

869 + def _get_next_call_id(self):

870 + """Generate unique call ID"""

871 + return f'call_{uuid.uuid4().hex[:24]}'

872 +

873 + def _extract_function_name(self, name: str,

874 + attrs: dict[str, str]) -> Optional[str]:

875 + """Extract function name from various formats"""

876 + if attrs and 'name' in attrs:

877 + return attrs['name']

878 +

879 + if '=' in name:

880 + parts = name.split('=', 1)

881 + if len(parts) == 2 and parts[0] == 'function':

882 + return parts[1]

883 +

884 + return None

885 +

886 + def _extract_parameter_name(self, name: str,

887 + attrs: dict[str, str]) -> Optional[str]:

888 + """Extract parameter name from various formats"""

889 + if attrs and 'name' in attrs:

890 + return attrs['name']

891 +

892 + if '=' in name:

893 + parts = name.split('=', 1)

894 + if len(parts) == 2 and parts[0] == 'parameter':

895 + return parts[1]

896 +

897 + return None

898 +

899 + def _get_param_type(self, param_name: str) -> str:

900 + """Get parameter type based on tool configuration, defaults to string

901 + Args:

902 + param_name: Parameter name

903 +

904 + Returns:

905 + Parameter type

906 + """

907 + if not self.tools or not self.current_function_name:

908 + return 'string'

909 +

910 + for tool in self.tools:

911 + if not hasattr(tool, 'type') or not (hasattr(

912 + tool, 'function') and hasattr(tool.function, 'name')):

913 + continue

914 + if tool.type == 'function' and \

915 + tool.function.name == self.current_function_name:

916 + if not hasattr(tool.function, 'parameters'):

917 + return 'string'

918 + params = tool.function.parameters

919 + if isinstance(params, dict) and 'properties' in params:

920 + properties = params['properties']

921 + if param_name in properties and isinstance(

922 + properties[param_name], dict):

923 + return self.repair_param_type(

924 + str(properties[param_name].get('type', 'string')))

925 + elif isinstance(params, dict) and param_name in params:

926 + param_config = params[param_name]

927 + if isinstance(param_config, dict):

928 + return self.repair_param_type(

929 + str(param_config.get('type', 'string')))

930 + break

931 + return 'string'

932 +

933 + def repair_param_type(self, param_type: str) -> str:

934 + """Repair unknown parameter types by treating them as string

935 + Args:

936 + param_type: Parameter type

937 +

938 + Returns:

939 + Repaired parameter type

940 + """

941 + if param_type in [

942 + 'string', 'str', 'text', 'varchar', 'char', 'enum'

943 + ] or param_type.startswith('int') or param_type.startswith(

944 + 'uint'

945 + ) or param_type.startswith('long') or param_type.startswith(

946 + 'short'

947 + ) or param_type.startswith('unsigned') or param_type.startswith(

948 + 'num') or param_type.startswith('float') or param_type in [

949 + 'boolean', 'bool', 'binary'

950 + ] or (param_type in ["object", "array", "arr", "sequence"]

951 + or param_type.startswith("dict")

952 + or param_type.startswith("list")):

953 + return param_type

954 + else:

955 + return 'string'

956 +

957 + def _convert_param_value(self, param_value: str, param_type: str) -> Any:

958 + """Convert value based on parameter type

959 + Args:

960 + param_value: Parameter value

961 + param_type: Parameter type

962 +

963 + Returns:

964 + Converted value

965 + """

966 + if param_value.lower() == 'null':

967 + return None

968 +

969 + param_type = param_type.strip().lower()

970 + if param_type in ['string', 'str', 'text', 'varchar', 'char', 'enum']:

971 + return param_value

972 + elif (param_type.startswith('int') or param_type.startswith('uint')

973 + or param_type.startswith('long')

974 + or param_type.startswith('short')

975 + or param_type.startswith('unsigned')):

976 + try:

977 + return int(param_value)

978 + except (ValueError, TypeError):

979 + logger.warning(

980 + "Parsed value '%s' of parameter '%s' is not an integer "

981 + "in tool '%s', degenerating to string.", param_value)

982 + return param_value

983 + elif param_type.startswith('num') or param_type.startswith('float'):

984 + try:

985 + float_param_value: float = float(param_value)

986 + return float_param_value if float_param_value - int(

987 + float_param_value) != 0 else int(float_param_value)

988 + except (ValueError, TypeError):

989 + logger.warning(

990 + "Parsed value '%s' of parameter '%s' is not a float "

991 + "in tool '%s', degenerating to string.", param_value)

992 + return param_value

993 + elif param_type in ['boolean', 'bool', 'binary']:

994 + param_value = param_value.lower()

995 + return param_value == 'true'

996 + else:

997 + return param_value

998 +

999 + def _convert_for_json_streaming(self, converted_value: Any,

1000 + param_type: str) -> str:

1001 + """Convert converted_value based on

1002 + whether it's empty and if type is string

1003 + Args:

1004 + converted_value: Converted value

1005 + param_type: Parameter type

1006 +

1007 + Returns:

1008 + Converted string for streaming output

1009 + """

1010 + # Check if value is empty, but exclude numeric 0

1011 + if converted_value is None or converted_value == '':

1012 + return ''

1013 +

1014 + if param_type in ['string', 'str', 'text', 'varchar', 'char', 'enum']:

1015 + # String type, remove double quotes

1016 + return json.dumps(converted_value, ensure_ascii=False)[1:-1]

1017 + else:

1018 + # Non-string type, return complete JSON string

1019 + if not isinstance(converted_value, str):

1020 + return json.dumps(converted_value, ensure_ascii=False)

1021 + else:

1022 + return converted_value

1023 +

1024 + def _reset_xml_parser_after_tool_call(self):

1025 + """

1026 + Each tool_call is treated as a separate XML document,

1027 + so we need to reset the parser after each tool_call.

1028 + """

1029 +

1030 + # recreate XML parser

1031 + self.parser = ParserCreate()

1032 + self.setup_parser()

1033 +

1034 + # Reset current tool_call state

1035 + if self.current_call_id:

1036 + self.last_completed_call_id = self.current_call_id

1037 + self.current_call_id = None

1038 + self.current_function_name = None

1039 + self.current_function_open = False

1040 + self.parameters = {}

1041 + self.current_param_name = None

1042 + self.current_param_value = ''

1043 + self.current_param_value_converted = ''

1044 + self.current_param_is_first = False

1045 + self.should_emit_end_newline = False

1046 + self.start_quote_emitted = False

1047 + self.text_content_buffer = ''

1048 +

1049 + # Reset preprocessing and deferred parsing state

1050 + self._pre_inside_parameter = False

1051 + self._pre_param_buffer = ""

1052 + self._pre_current_param_name = None

1053 + self.defer_current_parameter = False

1054 + self.deferred_param_raw_value = ""

1055 +

1056 +

1057 +@ToolParserManager.register_module("qwen3_xml")

1058 +class Qwen3XMLToolParser(ToolParser):

1059 +

1060 + def __init__(self, tokenizer: AnyTokenizer):

1061 + super().__init__(tokenizer)

1062 + self.parser = StreamingXMLToolCallParser()

1063 +

1064 + logger.info("vLLM Successfully import tool parser %s !",

1065 + self.__class__.__name__)

1066 +

1067 + def extract_tool_calls(

1068 + self,

1069 + model_output: str,

1070 + request: ChatCompletionRequest,

1071 + ) -> ExtractedToolCallInformation:

1072 + self.parser.reset_streaming_state()

1073 + if request:

1074 + self.parser.set_tools(request.tools)

1075 + result = self.parser.parse_single_streaming_chunks(model_output)

1076 + if not result.tool_calls:

1077 + return ExtractedToolCallInformation(

1078 + tool_calls=[],

1079 + tools_called=False,

1080 + content=result.content,

1081 + )

1082 + else:

1083 + tool_calls = []

1084 + for tool_call in result.tool_calls:

1085 + if tool_call.function and tool_call.function.name:

1086 + tool_calls.append(

1087 + ToolCall(

1088 + id=tool_call.id,

1089 + type=tool_call.type,

1090 + function=FunctionCall(

1091 + name=tool_call.function.name,

1092 + arguments=tool_call.function.arguments,

1093 + ),

1094 + ))

1095 + return ExtractedToolCallInformation(

1096 + tool_calls=tool_calls,

1097 + tools_called=len(tool_calls) > 0,

1098 + content=result.content,

1099 + )

1100 +

1101 + def extract_tool_calls_streaming(

1102 + self,

1103 + previous_text: str,

1104 + current_text: str,

1105 + delta_text: str,

1106 + previous_token_ids: Sequence[int],

1107 + current_token_ids: Sequence[int],

1108 + delta_token_ids: Sequence[int],

1109 + request: ChatCompletionRequest,

1110 + ) -> Union[DeltaMessage, None]:

1111 + if not previous_text:

1112 + self.parser.reset_streaming_state()

1113 + if request:

1114 + self.parser.set_tools(request.tools)

1115 +

1116 + # Model sometimes outputs separately causing delta_text to be empty.

1117 + # If there were tool_calls before and all current tool_calls have ended,

1118 + # return an empty tool_call for outer streaming output

1119 + # to correctly output tool_call field

1120 + if not delta_text and delta_token_ids:

1121 + open_calls = current_text.count(

1122 + self.parser.tool_call_start_token) - current_text.count(

1123 + self.parser.tool_call_end_token)

1124 + if open_calls == 0 and self.parser.tool_call_index > 0:

1125 + # If current_call_id is None, use last_completed_call_id

1126 + call_id = self.parser.current_call_id or \

1127 + self.parser.last_completed_call_id

1128 + return DeltaMessage(tool_calls=[

1129 + DeltaToolCall(

1130 + index=self.parser.tool_call_index - 1,

1131 + id=call_id,

1132 + function=DeltaFunctionCall(arguments=''),

1133 + type='function',

1134 + )

1135 + ])

1136 +

1137 + return self.parser.parse_single_streaming_chunks(delta_text)