@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4594fc3b28
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 01:00 UTC

[Model] Add Qwen3CoderToolParser (#21396)

Signed-off-by: simon-mo <xmo@berkeley.edu> Co-authored-by: simon-mo <xmo@berkeley.edu>

Yiheng Xu committed Jul 22, 2025 at 15:05 UTC 4594fc3b281713bd3d7634405b4a1393af40d294
3 files changed +1289
tests/tool_use/test_qwen3coder_tool_parser.py new
+618
@@ -0,0 +1,618 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +from collections.abc import Generator
6 +from typing import Optional
7 +
8 +import pytest
9 +
10 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
11 + ChatCompletionToolsParam,
12 + DeltaMessage, FunctionCall,
13 + ToolCall)
14 +from vllm.entrypoints.openai.tool_parsers.qwen3coder_tool_parser import (
15 + Qwen3CoderToolParser)
16 +from vllm.transformers_utils.detokenizer import detokenize_incrementally
17 +from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
18 +
19 +MODEL = "Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8"
20 +
21 +
22 +@pytest.fixture(scope="module")
23 +def qwen3_tokenizer():
24 + return get_tokenizer(tokenizer_name=MODEL)
25 +
26 +
27 +@pytest.fixture
28 +def qwen3_tool_parser(qwen3_tokenizer):
29 + return Qwen3CoderToolParser(qwen3_tokenizer)
30 +
31 +
32 +@pytest.fixture
33 +def sample_tools():
34 + return [
35 + ChatCompletionToolsParam(type="function",
36 + function={
37 + "name": "get_current_weather",
38 + "description": "Get the current weather",
39 + "parameters": {
40 + "type": "object",
41 + "properties": {
42 + "city": {
43 + "type": "string",
44 + "description": "The city name"
45 + },
46 + "state": {
47 + "type": "string",
48 + "description":
49 + "The state code"
50 + },
51 + "unit": {
52 + "type": "string",
53 + "enum":
54 + ["fahrenheit", "celsius"]
55 + }
56 + },
57 + "required": ["city", "state"]
58 + }
59 + }),
60 + ChatCompletionToolsParam(type="function",
61 + function={
62 + "name": "calculate_area",
63 + "description":
64 + "Calculate area of a shape",
65 + "parameters": {
66 + "type": "object",
67 + "properties": {
68 + "shape": {
69 + "type": "string"
70 + },
71 + "dimensions": {
72 + "type": "object"
73 + },
74 + "precision": {
75 + "type": "integer"
76 + }
77 + }
78 + }
79 + })
80 + ]
81 +
82 +
83 +def assert_tool_calls(actual_tool_calls: list[ToolCall],
84 + expected_tool_calls: list[ToolCall]):
85 + assert len(actual_tool_calls) == len(expected_tool_calls)
86 +
87 + for actual_tool_call, expected_tool_call in zip(actual_tool_calls,
88 + expected_tool_calls):
89 + # Qwen3 parser doesn't generate IDs during extraction
90 + assert actual_tool_call.type == "function"
91 + assert (
92 + actual_tool_call.function.name == expected_tool_call.function.name)
93 + assert (json.loads(actual_tool_call.function.arguments) == json.loads(
94 + expected_tool_call.function.arguments))
95 +
96 +
97 +def stream_delta_message_generator(
98 + qwen3_tool_parser: Qwen3CoderToolParser,
99 + qwen3_tokenizer: AnyTokenizer,
100 + model_output: str,
101 + request: Optional[ChatCompletionRequest] = None
102 +) -> Generator[DeltaMessage, None, None]:
103 + all_token_ids = qwen3_tokenizer.encode(model_output,
104 + add_special_tokens=False)
105 +
106 + previous_text = ""
107 + previous_tokens = None
108 + prefix_offset = 0
109 + read_offset = 0
110 + for i, delta_token in enumerate(all_token_ids):
111 + delta_token_ids = [delta_token]
112 + previous_token_ids = all_token_ids[:i]
113 + current_token_ids = all_token_ids[:i + 1]
114 +
115 + (new_tokens, delta_text, new_prefix_offset,
116 + new_read_offset) = detokenize_incrementally(
117 + tokenizer=qwen3_tokenizer,
118 + all_input_ids=current_token_ids,
119 + prev_tokens=previous_tokens,
120 + prefix_offset=prefix_offset,
121 + read_offset=read_offset,
122 + skip_special_tokens=False,
123 + spaces_between_special_tokens=True,
124 + )
125 +
126 + current_text = previous_text + delta_text
127 +
128 + delta_message = qwen3_tool_parser.extract_tool_calls_streaming(
129 + previous_text,
130 + current_text,
131 + delta_text,
132 + previous_token_ids,
133 + current_token_ids,
134 + delta_token_ids,
135 + request=request,
136 + )
137 + if delta_message:
138 + yield delta_message
139 +
140 + previous_text = current_text
141 + previous_tokens = (previous_tokens +
142 + new_tokens if previous_tokens else new_tokens)
143 + prefix_offset = new_prefix_offset
144 + read_offset = new_read_offset
145 +
146 +
147 +def test_extract_tool_calls_no_tools(qwen3_tool_parser):
148 + model_output = "This is a test response without any tool calls"
149 + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
150 + model_output, request=None) # type: ignore[arg-type]
151 + assert not extracted_tool_calls.tools_called
152 + assert extracted_tool_calls.tool_calls == []
153 + assert extracted_tool_calls.content == model_output
154 +
155 +
156 +@pytest.mark.parametrize(
157 + ids=[
158 + "single_tool",
159 + "single_tool_with_content",
160 + "single_tool_multiline_param",
161 + "parallel_tools",
162 + "tool_with_typed_params",
163 + ],
164 + argnames=["model_output", "expected_tool_calls", "expected_content"],
165 + argvalues=[
166 + ('''<tool_call>
167 +<function=get_current_weather>
168 +<parameter=city>
169 +Dallas
170 +</parameter>
171 +<parameter=state>
172 +TX
173 +</parameter>
174 +<parameter=unit>
175 +fahrenheit
176 +</parameter>
177 +</function>
178 +</tool_call>''', [
179 + ToolCall(
180 + function=FunctionCall(name="get_current_weather",
181 + arguments=json.dumps({
182 + "city": "Dallas",
183 + "state": "TX",
184 + "unit": "fahrenheit"
185 + })))
186 + ], None),
187 + ('''Sure! Let me check the weather for you.<tool_call>
188 +<function=get_current_weather>
189 +<parameter=city>
190 +Dallas
191 +</parameter>
192 +<parameter=state>
193 +TX
194 +</parameter>
195 +<parameter=unit>
196 +fahrenheit
197 +</parameter>
198 +</function>
199 +</tool_call>''', [
200 + ToolCall(
201 + function=FunctionCall(name="get_current_weather",
202 + arguments=json.dumps({
203 + "city": "Dallas",
204 + "state": "TX",
205 + "unit": "fahrenheit"
206 + })))
207 + ], "Sure! Let me check the weather for you."),
208 + ('''<tool_call>
209 +<function=calculate_area>
210 +<parameter=shape>
211 +rectangle
212 +</parameter>
213 +<parameter=dimensions>
214 +{"width": 10,
215 + "height": 20}
216 +</parameter>
217 +<parameter=precision>
218 +2
219 +</parameter>
220 +</function>
221 +</tool_call>''', [
222 + ToolCall(function=FunctionCall(name="calculate_area",
223 + arguments=json.dumps({
224 + "shape": "rectangle",
225 + "dimensions": {
226 + "width": 10,
227 + "height": 20
228 + },
229 + "precision": 2
230 + })))
231 + ], None),
232 + ('''<tool_call>
233 +<function=get_current_weather>
234 +<parameter=city>
235 +Dallas
236 +</parameter>
237 +<parameter=state>
238 +TX
239 +</parameter>
240 +<parameter=unit>
241 +fahrenheit
242 +</parameter>
243 +</function>
244 +</tool_call>
245 +<tool_call>
246 +<function=get_current_weather>
247 +<parameter=city>
248 +Orlando
249 +</parameter>
250 +<parameter=state>
251 +FL
252 +</parameter>
253 +<parameter=unit>
254 +fahrenheit
255 +</parameter>
256 +</function>
257 +</tool_call>''', [
258 + ToolCall(
259 + function=FunctionCall(name="get_current_weather",
260 + arguments=json.dumps({
261 + "city": "Dallas",
262 + "state": "TX",
263 + "unit": "fahrenheit"
264 + }))),
265 + ToolCall(
266 + function=FunctionCall(name="get_current_weather",
267 + arguments=json.dumps({
268 + "city": "Orlando",
269 + "state": "FL",
270 + "unit": "fahrenheit"
271 + })))
272 + ], None),
273 + ('''Let me calculate that area for you.<tool_call>
274 +<function=calculate_area>
275 +<parameter=shape>
276 +circle
277 +</parameter>
278 +<parameter=dimensions>
279 +{"radius": 15.5}
280 +</parameter>
281 +<parameter=precision>
282 +3
283 +</parameter>
284 +</function>
285 +</tool_call>''', [
286 + ToolCall(function=FunctionCall(name="calculate_area",
287 + arguments=json.dumps({
288 + "shape": "circle",
289 + "dimensions": {
290 + "radius": 15.5
291 + },
292 + "precision": 3
293 + })))
294 + ], "Let me calculate that area for you."),
295 + ],
296 +)
297 +def test_extract_tool_calls(qwen3_tool_parser, sample_tools, model_output,
298 + expected_tool_calls, expected_content):
299 + request = ChatCompletionRequest(model=MODEL,
300 + messages=[],
301 + tools=sample_tools)
302 + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
303 + model_output, request=request)
304 + assert extracted_tool_calls.tools_called
305 +
306 + assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
307 +
308 + assert extracted_tool_calls.content == expected_content
309 +
310 +
311 +def test_extract_tool_calls_fallback_no_tags(qwen3_tool_parser, sample_tools):
312 + """Test fallback parsing when XML tags are missing"""
313 + model_output = '''<function=get_current_weather>
314 +<parameter=city>
315 +Dallas
316 +</parameter>
317 +<parameter=state>
318 +TX
319 +</parameter>
320 +</function>'''
321 +
322 + request = ChatCompletionRequest(model=MODEL,
323 + messages=[],
324 + tools=sample_tools)
325 + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
326 + model_output, request=request)
327 +
328 + assert extracted_tool_calls.tools_called
329 + assert len(extracted_tool_calls.tool_calls) == 1
330 + assert (extracted_tool_calls.tool_calls[0].function.name ==
331 + "get_current_weather")
332 +
333 +
334 +def test_extract_tool_calls_type_conversion(qwen3_tool_parser):
335 + """Test parameter type conversion based on tool schema"""
336 + tools = [
337 + ChatCompletionToolsParam(type="function",
338 + function={
339 + "name": "test_types",
340 + "parameters": {
341 + "type": "object",
342 + "properties": {
343 + "int_param": {
344 + "type": "integer"
345 + },
346 + "float_param": {
347 + "type": "float"
348 + },
349 + "bool_param": {
350 + "type": "boolean"
351 + },
352 + "str_param": {
353 + "type": "string"
354 + },
355 + "obj_param": {
356 + "type": "object"
357 + }
358 + }
359 + }
360 + })
361 + ]
362 +
363 + model_output = '''<tool_call>
364 +<function=test_types>
365 +<parameter=int_param>
366 +42
367 +</parameter>
368 +<parameter=float_param>
369 +3.14
370 +</parameter>
371 +<parameter=bool_param>
372 +true
373 +</parameter>
374 +<parameter=str_param>
375 +hello world
376 +</parameter>
377 +<parameter=obj_param>
378 +{"key": "value"}
379 +</parameter>
380 +</function>
381 +</tool_call>'''
382 +
383 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
384 + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
385 + model_output, request=request)
386 +
387 + args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
388 + assert args["int_param"] == 42
389 + assert args["float_param"] == 3.14
390 + assert args["bool_param"] is True
391 + assert args["str_param"] == "hello world"
392 + assert args["obj_param"] == {"key": "value"}
393 +
394 +
395 +@pytest.mark.parametrize(
396 + ids=[
397 + "no_tools",
398 + "single_tool",
399 + "single_tool_with_content",
400 + "parallel_tools",
401 + ],
402 + argnames=["model_output", "expected_tool_calls", "expected_content"],
403 + argvalues=[
404 + ("This is a test without tools", [], "This is a test without tools"),
405 + ('''<tool_call>
406 +<function=get_current_weather>
407 +<parameter=city>
408 +Dallas
409 +</parameter>
410 +<parameter=state>
411 +TX
412 +</parameter>
413 +<parameter=unit>
414 +fahrenheit
415 +</parameter>
416 +</function>
417 +</tool_call>''', [
418 + ToolCall(
419 + function=FunctionCall(name="get_current_weather",
420 + arguments=json.dumps({
421 + "city": "Dallas",
422 + "state": "TX",
423 + "unit": "fahrenheit"
424 + })))
425 + ], ""),
426 + ('''Sure! Let me check the weather for you.<tool_call>
427 +<function=get_current_weather>
428 +<parameter=city>
429 +Dallas
430 +</parameter>
431 +<parameter=state>
432 +TX
433 +</parameter>
434 +<parameter=unit>
435 +fahrenheit
436 +</parameter>
437 +</function>
438 +</tool_call>''', [
439 + ToolCall(
440 + function=FunctionCall(name="get_current_weather",
441 + arguments=json.dumps({
442 + "city": "Dallas",
443 + "state": "TX",
444 + "unit": "fahrenheit"
445 + })))
446 + ], "Sure! Let me check the weather for you."),
447 + ('''<tool_call>
448 +<function=get_current_weather>
449 +<parameter=city>
450 +Dallas
451 +</parameter>
452 +<parameter=state>
453 +TX
454 +</parameter>
455 +<parameter=unit>
456 +fahrenheit
457 +</parameter>
458 +</function>
459 +</tool_call>
460 +<tool_call>
461 +<function=get_current_weather>
462 +<parameter=city>
463 +Orlando
464 +</parameter>
465 +<parameter=state>
466 +FL
467 +</parameter>
468 +<parameter=unit>
469 +celsius
470 +</parameter>
471 +</function>
472 +</tool_call>''', [
473 + ToolCall(
474 + function=FunctionCall(name="get_current_weather",
475 + arguments=json.dumps({
476 + "city": "Dallas",
477 + "state": "TX",
478 + "unit": "fahrenheit"
479 + }))),
480 + ToolCall(
481 + function=FunctionCall(name="get_current_weather",
482 + arguments=json.dumps({
483 + "city": "Orlando",
484 + "state": "FL",
485 + "unit": "celsius"
486 + })))
487 + ], ""),
488 + ],
489 +)
490 +def test_extract_tool_calls_streaming(qwen3_tool_parser, qwen3_tokenizer,
491 + sample_tools, model_output,
492 + expected_tool_calls, expected_content):
493 + """Test incremental streaming behavior"""
494 + request = ChatCompletionRequest(model=MODEL,
495 + messages=[],
496 + tools=sample_tools)
497 +
498 + other_content = ''
499 + tool_states = {} # Track state per tool index
500 +
501 + for delta_message in stream_delta_message_generator(
502 + qwen3_tool_parser, qwen3_tokenizer, model_output, request):
503 + # role should never be streamed from tool parser
504 + assert not delta_message.role
505 +
506 + if delta_message.content:
507 + other_content += delta_message.content
508 +
509 + if delta_message.tool_calls:
510 + for tool_call in delta_message.tool_calls:
511 + idx = tool_call.index
512 +
513 + # Initialize state for new tool
514 + if idx not in tool_states:
515 + tool_states[idx] = {
516 + "id": None,
517 + "name": None,
518 + "arguments": "",
519 + "type": None
520 + }
521 +
522 + # First chunk should have id, name, and type
523 + if tool_call.id:
524 + tool_states[idx]["id"] = tool_call.id
525 +
526 + if tool_call.type:
527 + assert tool_call.type == "function"
528 + tool_states[idx]["type"] = tool_call.type
529 +
530 + if tool_call.function:
531 + if tool_call.function.name:
532 + # Should only be set once
533 + assert tool_states[idx]["name"] is None
534 + tool_states[idx]["name"] = tool_call.function.name
535 +
536 + if tool_call.function.arguments is not None:
537 + # Accumulate arguments incrementally
538 + tool_states[idx][
539 + "arguments"] += tool_call.function.arguments
540 +
541 + # Verify final content
542 + assert other_content == expected_content
543 +
544 + # Verify we got all expected tool calls
545 + assert len(tool_states) == len(expected_tool_calls)
546 +
547 + # Verify each tool call
548 + for idx, expected_tool in enumerate(expected_tool_calls):
549 + state = tool_states[idx]
550 + assert state["id"] is not None
551 + assert state["type"] == "function"
552 + assert state["name"] == expected_tool.function.name
553 +
554 + # Parse accumulated arguments
555 + arguments_str = state["arguments"]
556 + assert arguments_str is not None
557 + actual_args = json.loads(arguments_str)
558 + expected_args = json.loads(expected_tool.function.arguments)
559 + assert actual_args == expected_args
560 +
561 +
562 +def test_extract_tool_calls_streaming_incremental(qwen3_tool_parser,
563 + qwen3_tokenizer,
564 + sample_tools):
565 + """Test that streaming is truly incremental"""
566 + model_output = '''I'll check the weather.<tool_call>
567 +<function=get_current_weather>
568 +<parameter=city>
569 +Dallas
570 +</parameter>
571 +<parameter=state>
572 +TX
573 +</parameter>
574 +</function>
575 +</tool_call>'''
576 +
577 + request = ChatCompletionRequest(model=MODEL,
578 + messages=[],
579 + tools=sample_tools)
580 +
581 + chunks = []
582 + for delta_message in stream_delta_message_generator(
583 + qwen3_tool_parser, qwen3_tokenizer, model_output, request):
584 + chunks.append(delta_message)
585 +
586 + # Should have multiple chunks
587 + assert len(chunks) > 3
588 +
589 + # First chunk(s) should be content
590 + assert chunks[0].content is not None
591 + assert chunks[0].tool_calls is None or chunks[0].tool_calls == []
592 +
593 + # Should have a chunk with tool header (id, name, type)
594 + header_found = False
595 + for chunk in chunks:
596 + if chunk.tool_calls and chunk.tool_calls[0].id:
597 + header_found = True
598 + assert (chunk.tool_calls[0].function.name == "get_current_weather")
599 + assert chunk.tool_calls[0].type == "function"
600 + # Empty initially
601 + assert chunk.tool_calls[0].function.arguments == ""
602 + break
603 + assert header_found
604 +
605 + # Should have chunks with incremental arguments
606 + arg_chunks = []
607 + for chunk in chunks:
608 + if chunk.tool_calls and chunk.tool_calls[0].function.arguments:
609 + arg_chunks.append(chunk.tool_calls[0].function.arguments)
610 +
611 + # Arguments should be streamed incrementally
612 + assert len(arg_chunks) > 1
613 +
614 + # Concatenated arguments should form valid JSON
615 + full_args = "".join(arg_chunks)
616 + parsed_args = json.loads(full_args)
617 + assert parsed_args["city"] == "Dallas"
618 + assert parsed_args["state"] == "TX"
vllm/entrypoints/openai/tool_parsers/__init__.py
+2
@@ -17,6 +17,7 @@ from .minimax_tool_parser import MinimaxToolParser
17 from .mistral_tool_parser import MistralToolParser
18 from .phi4mini_tool_parser import Phi4MiniJsonToolParser
19 from .pythonic_tool_parser import PythonicToolParser
20 +from .qwen3coder_tool_parser import Qwen3CoderToolParser
21 from .xlam_tool_parser import xLAMToolParser
22
23 __all__ = [
@@ -38,4 +39,5 @@ __all__ = [
39 "KimiK2ToolParser",
40 "HunyuanA13BToolParser",
41 "Glm4MoeModelToolParser",
42 + "Qwen3CoderToolParser",
43 ]
vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py new
+669
@@ -0,0 +1,669 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +import uuid
6 +from collections.abc import Sequence
7 +from typing import Any, Optional, Union
8 +
9 +import regex as re
10 +
11 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12 + ChatCompletionToolsParam,
13 + DeltaFunctionCall, DeltaMessage,
14 + DeltaToolCall,
15 + ExtractedToolCallInformation,
16 + FunctionCall, ToolCall)
17 +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
18 + ToolParser, ToolParserManager)
19 +from vllm.logger import init_logger
20 +from vllm.transformers_utils.tokenizer import AnyTokenizer
21 +
22 +logger = init_logger(__name__)
23 +
24 +
25 +@ToolParserManager.register_module(["qwen3_coder"])
26 +class Qwen3CoderToolParser(ToolParser):
27 +
28 + def __init__(self, tokenizer: AnyTokenizer):
29 + super().__init__(tokenizer)
30 +
31 + self.current_tool_name_sent: bool = False
32 + self.prev_tool_call_arr: list[dict] = []
33 + self.streamed_args_for_tool: list[str] = []
34 +
35 + # Sentinel tokens for streaming mode
36 + self.tool_call_start_token: str = "<tool_call>"
37 + self.tool_call_end_token: str = "</tool_call>"
38 + self.tool_call_prefix: str = "<function="
39 + self.function_end_token: str = "</function>"
40 + self.parameter_prefix: str = "<parameter="
41 + self.parameter_end_token: str = "</parameter>"
42 + self.is_tool_call_started: bool = False
43 + self.failed_count: int = 0
44 +
45 + # Streaming state variables
46 + self.current_tool_index: int = 0
47 + self.header_sent: bool = False
48 + self.current_tool_string_id: Optional[str] = None
49 + self.current_function_name: Optional[str] = None
50 + self.current_param_name: Optional[str] = None
51 + self.current_param_value: str = ""
52 + self.param_count: int = 0
53 + self.in_param: bool = False
54 + self.in_function: bool = False
55 + self.accumulated_text: str = ""
56 + self.json_started: bool = False
57 + self.json_closed: bool = False
58 +
59 + # Enhanced streaming state - reset for each new message
60 + self._reset_streaming_state()
61 +
62 + # Regex patterns
63 + self.tool_call_complete_regex = re.compile(
64 + r"<tool_call>(.*?)</tool_call>", re.DOTALL)
65 + self.tool_call_regex = re.compile(
66 + r"<tool_call>(.*?)</tool_call>|<tool_call>(.*?)$", re.DOTALL)
67 + self.tool_call_function_regex = re.compile(
68 + r"<function=(.*?)</function>|<function=(.*)$", re.DOTALL)
69 + self.tool_call_parameter_regex = re.compile(
70 + r"<parameter=(.*?)</parameter>|<parameter=(.*?)$", re.DOTALL)
71 +
72 + if not self.model_tokenizer:
73 + raise ValueError(
74 + "The model tokenizer must be passed to the ToolParser "
75 + "constructor during construction.")
76 +
77 + self.tool_call_start_token_id = self.vocab.get(
78 + self.tool_call_start_token)
79 + self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
80 +
81 + if (self.tool_call_start_token_id is None
82 + or self.tool_call_end_token_id is None):
83 + raise RuntimeError(
84 + "Qwen3 XML Tool parser could not locate tool call start/end "
85 + "tokens in the tokenizer!")
86 +
87 + logger.debug("vLLM Successfully import tool parser %s !",
88 + self.__class__.__name__)
89 +
90 + def _generate_tool_call_id(self) -> str:
91 + """Generate a unique tool call ID."""
92 + return f"call_{uuid.uuid4().hex[:24]}"
93 +
94 + def _reset_streaming_state(self):
95 + """Reset all streaming state."""
96 + self.current_tool_index = 0
97 + self.is_tool_call_started = False
98 + self.header_sent = False
99 + self.current_tool_string_id = None
100 + self.current_function_name = None
101 + self.current_param_name = None
102 + self.current_param_value = ""
103 + self.param_count = 0
104 + self.in_param = False
105 + self.in_function = False
106 + self.accumulated_text = ""
107 + self.json_started = False
108 + self.json_closed = False
109 +
110 + def _parse_xml_function_call(
111 + self, function_call_str: str,
112 + tools: Optional[list[ChatCompletionToolsParam]]
113 + ) -> Optional[ToolCall]:
114 +
115 + def get_arguments_config(func_name: str) -> dict:
116 + if tools is None:
117 + return {}
118 + for config in tools:
119 + if not hasattr(config, "type") or not (
120 + hasattr(config, "function")
121 + and hasattr(config.function, "name")):
122 + continue
123 + if (config.type == "function"
124 + and config.function.name == func_name):
125 + if not hasattr(config.function, "parameters"):
126 + return {}
127 + params = config.function.parameters
128 + if isinstance(params, dict) and "properties" in params:
129 + return params["properties"]
130 + elif isinstance(params, dict):
131 + return params
132 + else:
133 + return {}
134 + logger.warning("Tool '%s' is not defined in the tools list.",
135 + func_name)
136 + return {}
137 +
138 + def convert_param_value(param_value: str, param_name: str,
139 + param_config: dict, func_name: str) -> Any:
140 + # Handle null value for any type
141 + if param_value.lower() == "null":
142 + return None
143 +
144 + converted_value: Any
145 +
146 + if param_name not in param_config:
147 + if param_config != {}:
148 + logger.warning(
149 + "Parsed parameter '%s' is not defined in the tool "
150 + "parameters for tool '%s', directly returning the "
151 + "string value.", param_name, func_name)
152 + return param_value
153 +
154 + if (isinstance(param_config[param_name], dict)
155 + and "type" in param_config[param_name]):
156 + param_type = str(
157 + param_config[param_name]["type"]).strip().lower()
158 + else:
159 + param_type = "string"
160 + if param_type in [
161 + "string", "str", "text", "varchar", "char", "enum"
162 + ]:
163 + return param_value
164 + elif (param_type.startswith("int") or param_type.startswith("uint")
165 + or param_type.startswith("long")
166 + or param_type.startswith("short")
167 + or param_type.startswith("unsigned")):
168 + try:
169 + converted_value = int(param_value)
170 + return converted_value
171 + except ValueError:
172 + logger.warning(
173 + "Parsed value '%s' of parameter '%s' is not an "
174 + "integer in tool '%s', degenerating to string.",
175 + param_value, param_name, func_name)
176 + return param_value
177 + elif (param_type.startswith("num")
178 + or param_type.startswith("float")):
179 + try:
180 + float_param_value = float(param_value)
181 + converted_value = (float_param_value if float_param_value -
182 + int(float_param_value) != 0 else
183 + int(float_param_value))
184 + return converted_value
185 + except ValueError:
186 + logger.warning(
187 + "Parsed value '%s' of parameter '%s' is not a float "
188 + "in tool '%s', degenerating to string.", param_value,
189 + param_name, func_name)
190 + return param_value
191 + elif param_type in ["boolean", "bool", "binary"]:
192 + param_value = param_value.lower()
193 + if param_value not in ["true", "false"]:
194 + logger.warning(
195 + "Parsed value '%s' of parameter '%s' is not a "
196 + "boolean (`true` of `false`) in tool '%s', "
197 + "degenerating to false.", param_value, param_name,
198 + func_name)
199 + return param_value == "true"
200 + else:
201 + if param_type == "object" or param_type.startswith("dict"):
202 + try:
203 + converted_value = json.loads(param_value)
204 + return converted_value
205 + except json.JSONDecodeError:
206 + logger.warning(
207 + "Parsed value '%s' of parameter '%s' is not a "
208 + "valid JSON object in tool '%s', will try other "
209 + "methods to parse it.", param_value, param_name,
210 + func_name)
211 + try:
212 + converted_value = eval(param_value)
213 + return converted_value
214 + except Exception:
215 + logger.warning(
216 + "Parsed value '%s' of parameter '%s' cannot be "
217 + "converted via Python `eval()` in tool '%s', "
218 + "degenerating to string.", param_value, param_name,
219 + func_name)
220 + return param_value
221 +
222 + # Extract function name
223 + end_index = function_call_str.index(">")
224 + function_name = function_call_str[:end_index]
225 + param_config = get_arguments_config(function_name)
226 + parameters = function_call_str[end_index + 1:]
227 + param_dict = {}
228 + for match in self.tool_call_parameter_regex.findall(parameters):
229 + match_text = match[0] if match[0] else match[1]
230 + idx = match_text.index(">")
231 + param_name = match_text[:idx]
232 + param_value = str(match_text[idx + 1:])
233 + # Remove prefix and trailing \n
234 + if param_value.startswith("\n"):
235 + param_value = param_value[1:]
236 + if param_value.endswith("\n"):
237 + param_value = param_value[:-1]
238 +
239 + param_dict[param_name] = convert_param_value(
240 + param_value, param_name, param_config, function_name)
241 + return ToolCall(
242 + type="function",
243 + function=FunctionCall(name=function_name,
244 + arguments=json.dumps(param_dict,
245 + ensure_ascii=False)),
246 + )
247 +
248 + def _get_function_calls(self, model_output: str) -> list[str]:
249 + # Find all tool calls
250 + matched_ranges = self.tool_call_regex.findall(model_output)
251 + raw_tool_calls = [
252 + match[0] if match[0] else match[1] for match in matched_ranges
253 + ]
254 +
255 + # Back-off strategy if no tool_call tags found
256 + if len(raw_tool_calls) == 0:
257 + raw_tool_calls = [model_output]
258 +
259 + raw_function_calls = []
260 + for tool_call in raw_tool_calls:
261 + raw_function_calls.extend(
262 + self.tool_call_function_regex.findall(tool_call))
263 +
264 + function_calls = [
265 + match[0] if match[0] else match[1] for match in raw_function_calls
266 + ]
267 + return function_calls
268 +
269 + def extract_tool_calls(
270 + self,
271 + model_output: str,
272 + request: ChatCompletionRequest,
273 + ) -> ExtractedToolCallInformation:
274 + # Quick check to avoid unnecessary processing
275 + if self.tool_call_prefix not in model_output:
276 + return ExtractedToolCallInformation(tools_called=False,
277 + tool_calls=[],
278 + content=model_output)
279 +
280 + try:
281 + function_calls = self._get_function_calls(model_output)
282 + if len(function_calls) == 0:
283 + return ExtractedToolCallInformation(tools_called=False,
284 + tool_calls=[],
285 + content=model_output)
286 +
287 + tool_calls = [
288 + self._parse_xml_function_call(function_call_str, request.tools)
289 + for function_call_str in function_calls
290 + ]
291 +
292 + # Populate prev_tool_call_arr for serving layer to set
293 + # finish_reason
294 + self.prev_tool_call_arr.clear() # Clear previous calls
295 + for tool_call in tool_calls:
296 + if tool_call:
297 + self.prev_tool_call_arr.append({
298 + "name":
299 + tool_call.function.name,
300 + "arguments":
301 + tool_call.function.arguments,
302 + })
303 +
304 + # Extract content before tool calls
305 + content_index = model_output.find(self.tool_call_start_token)
306 + content_index = (content_index if content_index >= 0 else
307 + model_output.find(self.tool_call_prefix))
308 + content = model_output[:content_index] # .rstrip()
309 +
310 + return ExtractedToolCallInformation(
311 + tools_called=(len(tool_calls) > 0),
312 + tool_calls=tool_calls,
313 + content=content if content else None,
314 + )
315 +
316 + except Exception:
317 + logger.exception("Error in extracting tool call from response.")
318 + return ExtractedToolCallInformation(tools_called=False,
319 + tool_calls=[],
320 + content=model_output)
321 +
322 + def extract_tool_calls_streaming(
323 + self,
324 + previous_text: str,
325 + current_text: str,
326 + delta_text: str,
327 + previous_token_ids: Sequence[int],
328 + current_token_ids: Sequence[int],
329 + delta_token_ids: Sequence[int],
330 + request: ChatCompletionRequest,
331 + ) -> Union[DeltaMessage, None]:
332 + # If no delta text, return None unless it's an EOS token after tool
333 + # calls
334 + if not delta_text:
335 + # Check if this is an EOS token after all tool calls are complete
336 + # We check for tool calls in the text even if is_tool_call_started
337 + # is False because it might have been reset after processing all
338 + # tools
339 + if (delta_token_ids
340 + and self.tool_call_end_token_id not in delta_token_ids):
341 + # Count complete tool calls
342 + complete_calls = len(
343 + self.tool_call_complete_regex.findall(current_text))
344 +
345 + # If we have completed tool calls and populated
346 + # prev_tool_call_arr
347 + if (complete_calls > 0 and len(self.prev_tool_call_arr) > 0):
348 + # Check if all tool calls are closed
349 + open_calls = (
350 + current_text.count(self.tool_call_start_token) -
351 + current_text.count(self.tool_call_end_token))
352 + if open_calls == 0:
353 + # Return empty delta message to allow finish_reason
354 + # processing
355 + return DeltaMessage(content="")
356 + elif not self.is_tool_call_started and current_text:
357 + # This is a regular content response that's now complete
358 + return DeltaMessage(content="")
359 + return None
360 +
361 + # Check if this is the first call (reset state if needed)
362 + if not previous_text:
363 + self._reset_streaming_state()
364 +
365 + # Update accumulated text
366 + self.accumulated_text = current_text
367 +
368 + # Check if we need to advance to next tool
369 + if self.json_closed and not self.in_function:
370 + # Check if this tool call has ended
371 + tool_ends = current_text.count(self.tool_call_end_token)
372 + if tool_ends > self.current_tool_index:
373 + # This tool has ended, advance to next
374 + self.current_tool_index += 1
375 + self.header_sent = False
376 + self.param_count = 0
377 + self.json_started = False
378 + self.json_closed = False
379 +
380 + # Check if there are more tool calls
381 + tool_starts_count = current_text.count(
382 + self.tool_call_start_token)
383 + if self.current_tool_index >= tool_starts_count:
384 + # No more tool calls
385 + self.is_tool_call_started = False
386 + # Continue processing next tool
387 + return None
388 +
389 + # Handle normal content before tool calls
390 + if not self.is_tool_call_started:
391 + # Check if tool call is starting
392 + if (self.tool_call_start_token_id in delta_token_ids
393 + or self.tool_call_start_token in delta_text):
394 + self.is_tool_call_started = True
395 + # Return any content before the tool call
396 + if self.tool_call_start_token in delta_text:
397 + content_before = delta_text[:delta_text.index(
398 + self.tool_call_start_token)]
399 + if content_before:
400 + return DeltaMessage(content=content_before)
401 + return None
402 + else:
403 + # Check if we're between tool calls - skip whitespace
404 + if (current_text.rstrip().endswith(self.tool_call_end_token)
405 + and delta_text.strip() == ""):
406 + # We just ended a tool call, skip whitespace
407 + return None
408 + # Normal content, no tool call
409 + return DeltaMessage(content=delta_text)
410 +
411 + # Check if we're between tool calls (waiting for next one)
412 + # Count tool calls we've seen vs processed
413 + tool_starts_count = current_text.count(self.tool_call_start_token)
414 + if self.current_tool_index >= tool_starts_count:
415 + # We're past all tool calls, shouldn't be here
416 + return None
417 +
418 + # We're in a tool call, find the current tool call portion
419 + # Need to find the correct tool call based on current_tool_index
420 + tool_starts: list[int] = []
421 + idx = 0
422 + while True:
423 + idx = current_text.find(self.tool_call_start_token, idx)
424 + if idx == -1:
425 + break
426 + tool_starts.append(idx)
427 + idx += len(self.tool_call_start_token)
428 +
429 + if self.current_tool_index >= len(tool_starts):
430 + # No more tool calls to process yet
431 + return None
432 +
433 + tool_start_idx = tool_starts[self.current_tool_index]
434 + # Find where this tool call ends (or current position if not ended yet)
435 + tool_end_idx = current_text.find(self.tool_call_end_token,
436 + tool_start_idx)
437 + if tool_end_idx == -1:
438 + tool_text = current_text[tool_start_idx:]
439 + else:
440 + tool_text = current_text[tool_start_idx:tool_end_idx +
441 + len(self.tool_call_end_token)]
442 +
443 + # Looking for function header
444 + if not self.header_sent:
445 + if self.tool_call_prefix in tool_text:
446 + func_start = (tool_text.find(self.tool_call_prefix) +
447 + len(self.tool_call_prefix))
448 + func_end = tool_text.find(">", func_start)
449 +
450 + if func_end != -1:
451 + # Found complete function name
452 + self.current_function_name = tool_text[func_start:func_end]
453 + self.current_tool_string_id = self._generate_tool_call_id()
454 + self.header_sent = True
455 + self.in_function = True
456 +
457 + # IMPORTANT: Add to prev_tool_call_arr immediately when we
458 + # detect a tool call. This ensures
459 + # finish_reason="tool_calls" even if parsing isn't complete
460 + already_added = any(
461 + tool.get("name") == self.current_function_name
462 + for tool in self.prev_tool_call_arr)
463 + if not already_added:
464 + self.prev_tool_call_arr.append({
465 + "name": self.current_function_name,
466 + "arguments":
467 + "{}", # Placeholder, will be updated later
468 + })
469 +
470 + # Send header with function info
471 + return DeltaMessage(tool_calls=[
472 + DeltaToolCall(
473 + index=self.current_tool_index,
474 + id=self.current_tool_string_id,
475 + function=DeltaFunctionCall(
476 + name=self.current_function_name, arguments=""),
477 + type="function",
478 + )
479 + ])
480 + return None
481 +
482 + # We've sent header, now handle function body
483 + if self.in_function:
484 + # Send opening brace if not sent yet
485 + if (not self.json_started
486 + and self.parameter_prefix not in delta_text):
487 + self.json_started = True
488 + return DeltaMessage(tool_calls=[
489 + DeltaToolCall(
490 + index=self.current_tool_index,
491 + function=DeltaFunctionCall(arguments="{"),
492 + )
493 + ])
494 +
495 + # Make sure json_started is set if we're processing parameters
496 + if not self.json_started:
497 + self.json_started = True
498 +
499 + # Check for function end in accumulated text
500 + if not self.json_closed and self.function_end_token in tool_text:
501 + # Close JSON
502 + self.json_closed = True
503 +
504 + # Extract the complete tool call to update prev_tool_call_arr
505 + # with final arguments. Find the function content
506 + func_start = (tool_text.find(self.tool_call_prefix) +
507 + len(self.tool_call_prefix))
508 + func_content_end = tool_text.find(self.function_end_token,
509 + func_start)
510 + if func_content_end != -1:
511 + func_content = tool_text[func_start:func_content_end]
512 + # Parse to get the complete arguments
513 + try:
514 + parsed_tool = self._parse_xml_function_call(
515 + func_content, request.tools if request else None)
516 + if parsed_tool:
517 + # Update existing entry in prev_tool_call_arr with
518 + # complete arguments
519 + for i, tool in enumerate(self.prev_tool_call_arr):
520 + if (tool.get("name") ==
521 + parsed_tool.function.name):
522 + self.prev_tool_call_arr[i]["arguments"] = (
523 + parsed_tool.function.arguments)
524 + break
525 + except Exception:
526 + pass # Ignore parsing errors during streaming
527 +
528 + result = DeltaMessage(tool_calls=[
529 + DeltaToolCall(
530 + index=self.current_tool_index,
531 + function=DeltaFunctionCall(arguments="}"),
532 + )
533 + ])
534 +
535 + # Reset state for next tool
536 + self.in_function = False
537 + self.json_closed = True
538 +
539 + return result
540 +
541 + # Look for parameters
542 + # Count how many complete parameters we have processed
543 + complete_params = tool_text.count(self.parameter_end_token)
544 +
545 + # Check if we should start a new parameter
546 + if not self.in_param and self.param_count < complete_params:
547 + # Find the unprocessed parameter
548 + # Count parameter starts
549 + param_starts = []
550 + idx = 0
551 + while True:
552 + idx = tool_text.find(self.parameter_prefix, idx)
553 + if idx == -1:
554 + break
555 + param_starts.append(idx)
556 + idx += len(self.parameter_prefix)
557 +
558 + if len(param_starts) > self.param_count:
559 + # Process the next parameter
560 + param_idx = param_starts[self.param_count]
561 + param_start = param_idx + len(self.parameter_prefix)
562 + remaining = tool_text[param_start:]
563 +
564 + if ">" in remaining:
565 + # We have the complete parameter name
566 + name_end = remaining.find(">")
567 + self.current_param_name = remaining[:name_end]
568 +
569 + # Find the parameter value
570 + value_start = param_start + name_end + 1
571 + value_text = tool_text[value_start:]
572 + if value_text.startswith("\n"):
573 + value_text = value_text[1:]
574 +
575 + # Find where this parameter ends
576 + param_end_idx = value_text.find(
577 + self.parameter_end_token)
578 + if param_end_idx != -1:
579 + # Complete parameter found
580 + param_value = value_text[:param_end_idx]
581 + if param_value.endswith("\n"):
582 + param_value = param_value[:-1]
583 +
584 + # Build complete JSON fragment for this parameter
585 + if self.param_count == 0:
586 + json_fragment = (
587 + '"' + self.current_param_name + '": "' +
588 + json.dumps(param_value)[1:-1] + '"')
589 + else:
590 + json_fragment = (
591 + ', "' + self.current_param_name + '": "' +
592 + json.dumps(param_value)[1:-1] + '"')
593 +
594 + self.param_count += 1
595 +
596 + return DeltaMessage(tool_calls=[
597 + DeltaToolCall(
598 + index=self.current_tool_index,
599 + function=DeltaFunctionCall(
600 + arguments=json_fragment),
601 + )
602 + ])
603 +
604 + # Continue parameter value
605 + if self.in_param:
606 + if self.parameter_end_token in delta_text:
607 + # End of parameter
608 + end_idx = delta_text.find(self.parameter_end_token)
609 + value_chunk = delta_text[:end_idx]
610 +
611 + # Skip past > if at start
612 + if not self.current_param_value and ">" in value_chunk:
613 + gt_idx = value_chunk.find(">")
614 + value_chunk = value_chunk[gt_idx + 1:]
615 +
616 + if (not self.current_param_value
617 + and value_chunk.startswith("\n")):
618 + value_chunk = value_chunk[1:]
619 +
620 + # Calculate incremental JSON
621 + full_value = self.current_param_value + value_chunk
622 + prev_escaped = (json.dumps(self.current_param_value)[1:-1]
623 + if self.current_param_value else "")
624 + full_escaped = json.dumps(full_value)[1:-1]
625 + delta_escaped = full_escaped[len(prev_escaped):]
626 +
627 + self.in_param = False
628 + self.current_param_value = ""
629 +
630 + return DeltaMessage(tool_calls=[
631 + DeltaToolCall(
632 + index=self.current_tool_index,
633 + function=DeltaFunctionCall(
634 + arguments=delta_escaped + '"'),
635 + )
636 + ])
637 + else:
638 + # Continue accumulating value
639 + value_chunk = delta_text
640 +
641 + # Handle first chunk after param name
642 + if not self.current_param_value and ">" in value_chunk:
643 + gt_idx = value_chunk.find(">")
644 + value_chunk = value_chunk[gt_idx + 1:]
645 +
646 + if (not self.current_param_value
647 + and value_chunk.startswith("\n")):
648 + value_chunk = value_chunk[1:]
649 +
650 + if value_chunk:
651 + # Stream the escaped delta
652 + prev_escaped = (json.dumps(
653 + self.current_param_value)[1:-1]
654 + if self.current_param_value else "")
655 + self.current_param_value += value_chunk
656 + full_escaped = json.dumps(
657 + self.current_param_value)[1:-1]
658 + delta_escaped = full_escaped[len(prev_escaped):]
659 +
660 + if delta_escaped:
661 + return DeltaMessage(tool_calls=[
662 + DeltaToolCall(
663 + index=self.current_tool_index,
664 + function=DeltaFunctionCall(
665 + arguments=delta_escaped),
666 + )
667 + ])
668 +
669 + return None