@tduncan / Nvidia-Optimization-Stack-VLLM / commits / a5105dba05

[Frontend] Port Step-3.5 parsers to the streaming parser engine (#59321)

Signed-off-by: sfeng33 <4florafeng@gmail.com>

Flora Feng committed Oct 1, 2026 at 18:18 UTC a5105dba05616d74dde5d40267fc500a4422b3db

12 files changed +236 -1647

docs/features/reasoning_outputs.md

@@ -29,6 +29,7 @@ vLLM currently supports the following reasoning models:

34 !!! note

35 IBM Granite 3.2 and DeepSeek-V3.1 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`.

docs/features/tool_calling.md

+12

@@ -469,6 +469,18 @@ Use the checkpoint's chat template with `--tool-call-parser mimo --reasoning-par

469 For schema-constrained tool arguments, set `strict: true` inside each tool's `function`, or use `--tool-strict-level parameter`.

470 The parser supports `auto`, `required`, named tools, and `parallel_tool_calls=False`, preserving leading/trailing whitespace in string arguments.

471

472 +### Step-3.5 and Step-3.7 Models (`step3p5`)

473 +

474 +Supported models:

475 +

476 +* `stepfun-ai/Step-3.5-Flash`

477 +* `stepfun-ai/Step-3.5-Flash-FP8`

478 +* `stepfun-ai/Step-3.7-Flash`

479 +

480 +Flags: `--tool-call-parser step3p5 --reasoning-parser step3p5 --enable-auto-tool-choice`

481 +

482 +The parser supports `auto`, `required`, and named tools.

483 +

484 ### Olmo 3 Models (`olmo3`)

485

486 Olmo 3 models output tool calls in a format that is very similar to the one expected by the `pythonic` parser (see below), with a few differences. Each tool call is a pythonic string, but the parallel tool calls are newline-delimited, and the calls are wrapped within XML tags as `<function_calls>..</function_calls>`. In addition, the parser also allows JSON boolean and null literals (`true`, `false`, and `null`) in addition to the pythonic ones (`True`, `False`, and `None`).

tests/parser/engine/test_step3p5.py new

+122

@@ -0,0 +1,122 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Tests for the engine-based Step-3.5 parser.

4 +

5 +Step-3.5 reuses the Qwen3 grammar, which ``test_qwen3.py`` covers. These tests

6 +cover what differs: thinking is always on, the newline the model emits before

7 +``</think>`` is dropped, prior-turn reasoning does not end the current turn,

8 +and the ``step3p5`` names resolve to the engine adapters.

9 +"""

10 +

11 +import json

12 +

13 +import pytest

14 +

15 +from tests.parser.engine.conftest import make_mock_tokenizer

16 +from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming

17 +from vllm.parser.engine.registered_adapters import (

18 + Step3p5ParserReasoningAdapter,

19 + Step3p5ParserToolAdapter,

20 +)

21 +from vllm.parser.step3p5 import Step3p5Parser

22 +from vllm.reasoning import ReasoningParserManager

23 +from vllm.tool_parsers import ToolParserManager

24 +

25 +_THINK_START_ID = 50

26 +_THINK_END_ID = 51

27 +_IM_START_ID = 70

28 +_IM_END_ID = 71

29 +_TEXT_ID = 100

30 +

31 +_STEP3P5_VOCAB = {

32 + "<think>": _THINK_START_ID,

33 + "</think>": _THINK_END_ID,

34 + "<tool_call>": 60,

35 + "</tool_call>": 61,

36 + "<|im_start|>": _IM_START_ID,

37 + "<|im_end|>": _IM_END_ID,

38 +}

39 +

40 +

41 +@pytest.fixture

42 +def mock_tokenizer():

43 + return make_mock_tokenizer(_STEP3P5_VOCAB)

44 +

45 +

46 +@pytest.fixture

47 +def parser(mock_tokenizer):

48 + return Step3p5Parser(mock_tokenizer)

49 +

50 +

51 +def test_parser_names_resolve_to_engine_adapters():

52 + tool_parser_cls = ToolParserManager.get_tool_parser("step3p5")

53 + assert issubclass(tool_parser_cls, Step3p5ParserToolAdapter)

54 + assert tool_parser_cls.structural_tag_model == "qwen_3_coder"

55 + reasoning_parser_cls = ReasoningParserManager.get_reasoning_parser("step3p5")

56 + assert reasoning_parser_cls is Step3p5ParserReasoningAdapter

57 +

58 +

59 +def test_enable_thinking_false_is_ignored(mock_tokenizer):

60 + # The chat template always prefills "<think>\n" and has no thinking switch.

61 + parser = Step3p5Parser(

62 + mock_tokenizer, chat_template_kwargs={"enable_thinking": False}

63 + )

64 + assert parser.extract_reasoning("plan</think>answer", None) == ("plan", "answer")

65 + assert not parser.is_reasoning_end([_IM_START_ID, _TEXT_ID, _THINK_START_ID])

66 +

67 +

68 +def test_newline_before_think_end_dropped(parser):

69 + text = "Line one.\nLine two.\n</think>\nThe answer."

70 + assert parser.extract_reasoning(text, None) == (

71 + "Line one.\nLine two.",

72 + "\nThe answer.",

73 + )

74 +

75 +

76 +def test_newline_before_think_end_dropped_streaming(parser):

77 + reasoning, content = simulate_reasoning_streaming(

78 + parser,

79 + ["Line one.\n", "Line two.\n", "</think>", "\nThe answer."],

80 + [(_TEXT_ID,), (_TEXT_ID,), (_THINK_END_ID,), (_TEXT_ID,)],

81 + )

82 + assert reasoning == "Line one.\nLine two."

83 + assert content == "\nThe answer."

84 +

85 +

86 +class TestPriorTurnReasoning:

87 + """The template replays earlier assistant turns as

88 + ``<think>\\n...\\n</think>\\n``, so a ``</think>`` from history must not end

89 + reasoning for the new turn (#34211)."""

90 +

91 + _HISTORY = [

92 + _IM_START_ID,

93 + _TEXT_ID,

94 + _THINK_START_ID,

95 + _TEXT_ID,

96 + _THINK_END_ID,

97 + _TEXT_ID,

98 + _IM_END_ID,

99 + ]

100 +

101 + def test_prior_turn_think_end_not_end(self, parser):

102 + assert not parser.is_reasoning_end([*self._HISTORY, _IM_START_ID, _TEXT_ID])

103 +

104 + def test_think_end_in_current_turn_is_end(self, parser):

105 + assert parser.is_reasoning_end(

106 + [*self._HISTORY, _IM_START_ID, _THINK_START_ID, _TEXT_ID, _THINK_END_ID]

107 + )

108 +

109 +

110 +def test_reasoning_then_tool_call(parser, mock_request):

111 + text = (

112 + "Need the weather.\n</think>\n<tool_call>\n<function=get_weather>\n"

113 + "<parameter=city>\nTokyo\n</parameter>\n</function>\n</tool_call>"

114 + )

115 + reasoning, content, tool_calls = parser.parse(

116 + text, mock_request, enable_auto_tools=True

117 + )

118 + assert reasoning == "Need the weather."

119 + assert content is None

120 + assert tool_calls is not None

121 + assert tool_calls[0].name == "get_weather"

122 + assert json.loads(tool_calls[0].arguments) == {"city": "Tokyo"}

tests/parser/engine/trace_builder.py

+15

@@ -46,6 +46,7 @@ from vllm.parser.engine.registered_adapters import (

46 Plamo3Parser,

47 Qwen3Parser,

48 SeedOssParser,

49 + Step3p5Parser,

50 )

52 # ── Data structures ──────────────────────────────────────────────────

@@ -449,6 +450,19 @@ def _build_mimo(scenario: Scenario, validate: bool = True) -> Sample:

450 )

451

452

453 +# ── Step-3.5 (Qwen3 XML, trailing reasoning whitespace stripped) ────

454 +

455 +

456 +def _build_step3p5(scenario: Scenario, validate: bool = True) -> Sample:

457 + return _build_qwen3(

458 + scenario,

459 + name="step3p5",

460 + parser_cls=Step3p5Parser,

461 + strip_trailing_ws=True,

462 + validate=validate,

463 + )

464 +

465 +

466 # ── MiniMax M2 (XML invoke format, starts in REASONING) ──────────────

467

468 _MINIMAX_M2_VOCAB: dict[str, int] = {

@@ -1268,6 +1282,7 @@ _BUILDERS: dict[str, Any] = {

1282 "kimi_k2": _build_kimi_k2,

1283 "qwen3": _build_qwen3,

1284 "mimo": _build_mimo,

1285 + "step3p5": _build_step3p5,

1286 "inkling": _build_inkling,

1287 "plamo3": _build_plamo3,

1288 }

tests/reasoning/test_step3p5_reasoning_parser.py

+9 -9

@@ -4,8 +4,8 @@

4 import pytest

6 from tests.reasoning.utils import run_reasoning_extraction

7 +from vllm.parser.engine.registered_adapters import Step3p5ParserReasoningAdapter

8 from vllm.reasoning import ReasoningParserManager

8 -from vllm.reasoning.step3p5_reasoning_parser import Step3p5ReasoningParser

9 from vllm.tokenizers import get_tokenizer

11 parser_name = "step3p5"

@@ -112,7 +112,7 @@ EMPTY_STREAMING = {

112 }

113 NEW_LINE = {

114 "output": "\n<think>This is a reasoning section</think>\nThis is the rest",

115 - "reasoning": "This is a reasoning section",

115 + "reasoning": "\nThis is a reasoning section",

116 "content": "\nThis is the rest",

117 "is_reasoning_end": True,

118 }

@@ -126,7 +126,7 @@ NEW_LINE_STREAMING = {

126

127 NEW_LINE_STREAMING_COMPLEX_CONTENT = {

128 "output": "\n This is a \n reasoning section\n\n\n</think>\n\nThis is the rest",

129 - "reasoning": "\n This is a \n reasoning section\n\n",

129 + "reasoning": "\n This is a \n reasoning section",

130 "content": "\n\nThis is the rest",

131 "is_reasoning_end": True,

132 }

@@ -285,7 +285,7 @@ def test_reasoning(

285 step3p5_tokenizer.convert_tokens_to_string([token]) for token in output

286 ]

287 parser = ReasoningParserManager.get_reasoning_parser(parser_name)(step3p5_tokenizer)

288 - assert isinstance(parser, Step3p5ReasoningParser)

288 + assert isinstance(parser, Step3p5ParserReasoningAdapter)

289

290 reasoning, content = run_reasoning_extraction(

291 parser, output_tokens, streaming=streaming

@@ -307,9 +307,10 @@ def test_reasoning(

307 current_ids: list[int] = []

308 for token_id in output_ids:

309 current_ids.append(token_id)

310 - if token_id == parser.end_token_id:

310 + if token_id in parser.reasoning_end_token_ids:

311 assert parser.is_reasoning_end_streaming(current_ids, [token_id])

312 - assert parser.is_reasoning_end_streaming(output_ids, output_ids)

312 + if param_dict["is_reasoning_end"]:

313 + assert parser.is_reasoning_end_streaming(output_ids, output_ids)

314 break

315 assert not parser.is_reasoning_end_streaming(current_ids, [token_id])

316 else:

@@ -333,6 +334,5 @@ def test_reasoning(

334 step3p5_tokenizer.tokenize(param_dict["content"])

335 )

336 assert content_ids == expected_content_ids

336 - else:

337 - content_ids = parser.extract_content_ids(output_ids)

338 - assert content_ids == []

337 + elif param_dict["is_reasoning_end"]:

338 + assert parser.extract_content_ids(output_ids) == []

tests/tool_parsers/test_step3p5_tool_parser.py

+12 -57

@@ -847,7 +847,7 @@ TX

847 assert chunk.tool_calls[0].function.name == "get_current_weather"

848 assert chunk.tool_calls[0].type == "function"

849 # Empty initially

850 - assert chunk.tool_calls[0].function.arguments == ""

850 + assert not chunk.tool_calls[0].function.arguments

851 break

852 assert header_found

853

@@ -867,43 +867,6 @@ TX

867 assert parsed_args["state"] == "TX"

868

869

870 -def test_extract_tool_calls_complex_type_with_single_quote(step3p5_tokenizer):

871 - """Test parameter type conversion based on tool schema."""

872 - tools = [

873 - ChatCompletionToolsParam(

874 - type="function",

875 - function={

876 - "name": "test_types",

877 - "parameters": {

878 - "type": "object",

879 - "properties": {

880 - "int_param": {"type": "integer"},

881 - "float_param": {"type": "float"},

882 - "bool_param": {"type": "boolean"},

883 - "str_param": {"type": "string"},

884 - "obj_param": {"type": "object"},

885 - },

886 - },

887 - },

888 - )

889 - ]

890 -

891 - model_output = """<tool_call>

892 -<function=test_types>

893 -<parameter=obj_param>

894 -{'key': 'value'}

895 -</parameter>

896 -</function>

897 -</tool_call>"""

898 -

899 - parser = Step3p5ToolParser(step3p5_tokenizer, tools=tools)

900 - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)

901 - extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)

902 -

903 - args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)

904 - assert args["obj_param"] == {"key": "value"}

905 -

906 -

870 def test_extract_tool_calls_streaming_mixed_content_and_multiple_tool_calls(

871 step3p5_tool_parser, step3p5_tokenizer, sample_tools

872 ):

@@ -1016,8 +979,8 @@ def test_extract_tool_calls_non_streaming_mixed_content_and_multiple_tool_calls(

979 """Test non-streaming extraction with mixed content and multiple tool calls.

980

981 Scenario: Model outputs "hello" + complete tool call + "hi" + complete tool call.

1019 - Expected: "hello" as content, first tool call parsed (index=0), "hi" as content,

1020 - second tool call parsed (index=1)

982 + Expected: "hello" as content, first tool call parsed (index=0), second tool call

983 + parsed (index=1). Non-streaming content stops at the first tool call.

984 """

985 # Model output: hello + complete tool call + hi + complete tool call

986 model_output = """hello<tool_call>

@@ -1066,17 +1029,7 @@ rectangle

1029 assert args_dict_1["dimensions"]["width"] == 10

1030 assert args_dict_1["dimensions"]["height"] == 5

1031

1069 - # Verify content: should contain "hello", "hi"

1070 - assert extracted_tool_calls.content is not None

1071 - assert "hello" in extracted_tool_calls.content, "Should contain 'hello' as content"

1072 - assert "hi" in extracted_tool_calls.content, "Should contain 'hi' as content"

1073 -

1074 - # Verify the order: hello should come first, then hi

1075 - hello_index = extracted_tool_calls.content.find("hello")

1076 - hi_index = extracted_tool_calls.content.find("hi")

1077 -

1078 - assert hello_index >= 0, "'hello' should be in content"

1079 - assert hi_index > hello_index, "'hi' should come after 'hello'"

1032 + assert extracted_tool_calls.content == "hello"

1033

1034 # Verify that tool call tags are NOT in the content

1035 assert "<function=get_current_weather>" not in extracted_tool_calls.content, (

@@ -1155,12 +1108,14 @@ rectangle

1108 request=request,

1109 )

1110

1158 - # Process delta result

1159 - if delta_result:

1160 - if delta_result.content:

1161 - other_content += delta_result.content

1162 - if delta_result.tool_calls:

1163 - for tool_call in delta_result.tool_calls:

1111 + # Serving flushes the parser at the end of the stream.

1112 + for delta in (delta_result, step3p5_tool_parser.finish_streaming()):

1113 + if not delta:

1114 + continue

1115 + if delta.content:

1116 + other_content += delta.content

1117 + if delta.tool_calls:

1118 + for tool_call in delta.tool_calls:

1119 idx = tool_call.index

1120 if idx not in tool_states:

1121 tool_states[idx] = {

vllm/parser/engine/registered_adapters.py

@@ -24,6 +24,7 @@ from vllm.parser.nemotron_v3 import NemotronV3Parser

24 from vllm.parser.plamo3 import Plamo3Parser

25 from vllm.parser.qwen3 import Qwen3Parser

26 from vllm.parser.seed_oss import SeedOssParser

27 +from vllm.parser.step3p5 import Step3p5Parser

29 (

30 DeepSeekV32ParserReasoningAdapter,

@@ -80,6 +81,11 @@ from vllm.parser.seed_oss import SeedOssParser

81 SeedOssParserToolAdapter,

82 ) = make_adapters(SeedOssParser)

84 +(

85 + Step3p5ParserReasoningAdapter,

86 + Step3p5ParserToolAdapter,

87 +) = make_adapters(Step3p5Parser)

88 +

89 (

90 Glm47MoeParserReasoningAdapter,

91 Glm47MoeParserToolAdapter,

vllm/parser/step3p5.py new

+49

@@ -0,0 +1,49 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Step-3.5 / Step-3.7 parser for tool calls and reasoning.

4 +

5 +Both models use the Qwen3 XML tool-call grammar and ``<think>`` reasoning

6 +unchanged. Two things differ from :class:`Qwen3Parser`:

7 +

8 +* The chat template always prefills ``<think>\\n`` and has no switch to turn

9 + thinking off, so ``enable_thinking`` is ignored.

10 +* The model closes reasoning with ``\\n</think>\\n``, so trailing reasoning

11 + whitespace is stripped.

12 +"""

13 +

14 +from __future__ import annotations

15 +

16 +import functools

17 +from dataclasses import replace

18 +from typing import TYPE_CHECKING

19 +

20 +from vllm.parser.engine.parser_engine_config import ParserEngineConfig

21 +from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config

22 +

23 +if TYPE_CHECKING:

24 + from vllm.tokenizers import TokenizerLike

25 + from vllm.tool_parsers.abstract_tool_parser import Tool

26 +

27 +

28 +@functools.cache

29 +def step3p5_config() -> ParserEngineConfig:

30 + return replace(

31 + qwen3_config(

32 + thinking=True,

33 + name="step3p5",

34 + turn_boundary_tokens=CHATML_TURN_BOUNDARIES,

35 + ),

36 + strip_trailing_reasoning_whitespace=True,

37 + )

38 +

39 +

40 +class Step3p5Parser(Qwen3Parser):

41 + def __init__(

42 + self,

43 + tokenizer: TokenizerLike,

44 + tools: list[Tool] | None = None,

45 + **kwargs,

46 + ) -> None:

47 + kwargs.setdefault("parser_engine_config", step3p5_config())

48 + super().__init__(tokenizer, tools, **kwargs)

49 + self.thinking_enabled = True

vllm/reasoning/__init__.py

+2 -2

@@ -157,8 +157,8 @@ _REASONING_PARSERS_TO_REGISTER = {

157 "Step3ReasoningParser",

158 ),

159 "step3p5": (

160 - "step3p5_reasoning_parser",

161 - "Step3p5ReasoningParser",

160 + "step3p5_engine_reasoning_parser",

161 + "Step3p5ParserReasoningAdapter",

162 ),

163 "inkling": (

164 "inkling_reasoning_parser",

vllm/reasoning/step3p5_engine_reasoning_parser.py new

@@ -0,0 +1,6 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +from vllm.parser.engine.registered_adapters import Step3p5ParserReasoningAdapter

5 +

6 +__all__ = ["Step3p5ParserReasoningAdapter"]

vllm/reasoning/step3p5_reasoning_parser.py deleted

-121

@@ -1,121 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -from collections.abc import Sequence

5 -from typing import TYPE_CHECKING

6 -

7 -from vllm.entrypoints.generate.base.protocol import DeltaMessage

8 -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser

9 -from vllm.tokenizers import TokenizerLike

10 -

11 -if TYPE_CHECKING:

12 - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest

13 - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest

14 -

15 -

16 -class Step3p5ReasoningParser(BaseThinkingReasoningParser):

17 - """Reasoning parser for Step3p5 model.

18 -

19 - Step3p5 uses the <think>...</think> format, but it tends to emit an extra

20 - newline immediately before and/or after the </think> token. This parser

21 - trims the newline before </think> in extract_reasoning*.

22 -

23 - Reasoning end detection doesn't wait for an extra token after </think> so

24 - structured output can constrain the first post-think token.

25 - """

26 -

27 - @property

28 - def start_token(self) -> str:

29 - return "<think>"

30 -

31 - @property

32 - def end_token(self) -> str:

33 - return "</think>"

34 -

35 - def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):

36 - super().__init__(tokenizer, *args, **kwargs)

37 -

38 - # Used to hold a trailing "\n" from reasoning content so we can decide

39 - # whether it is immediately before </think>.

40 - self._pending_reasoning_newline = False

41 -

42 - def extract_reasoning(

43 - self,

44 - model_output: str,

45 - request: "ChatCompletionRequest | ResponsesRequest",

46 - ) -> tuple[str | None, str | None]:

47 - reasoning, content = super().extract_reasoning(model_output, request)

48 - if reasoning is not None:

49 - reasoning = reasoning.removesuffix("\n")

50 - return reasoning or None, content or None

51 -

52 - def extract_reasoning_streaming(

53 - self,

54 - previous_text: str,

55 - current_text: str,

56 - delta_text: str,

57 - previous_token_ids: Sequence[int],

58 - current_token_ids: Sequence[int],

59 - delta_token_ids: Sequence[int],

60 - ) -> DeltaMessage | None:

61 - ret = super().extract_reasoning_streaming(

62 - previous_text,

63 - current_text,

64 - delta_text,

65 - previous_token_ids,

66 - current_token_ids,

67 - delta_token_ids,

68 - )

69 -

70 - if ret is None:

71 - return None

72 -

73 - # Compatibility path for models that don't generate the start token:

74 - # treat everything before </think> as reasoning and everything after

75 - # as content.

76 - if (

77 - self.start_token_id not in previous_token_ids

78 - and self.start_token_id not in delta_token_ids

79 - ):

80 - if self.end_token_id in delta_token_ids:

81 - end_index = delta_text.find(self.end_token)

82 - reasoning = delta_text[:end_index]

83 - content = delta_text[end_index + len(self.end_token) :]

84 - ret = DeltaMessage(reasoning=reasoning, content=content or None)

85 - elif self.end_token_id in previous_token_ids:

86 - ret = DeltaMessage(content=delta_text)

87 - else:

88 - ret = DeltaMessage(reasoning=delta_text)

89 -

90 - reasoning_to_output = ret.reasoning

91 - content_to_output = ret.content

92 -

93 - # Reasoning: handle the newline immediately before </think>.

94 - if reasoning_to_output is not None:

95 - if self._pending_reasoning_newline:

96 - reasoning_to_output = "\n" + reasoning_to_output

97 - self._pending_reasoning_newline = False

98 -

99 - if reasoning_to_output.endswith("\n"):

100 - reasoning_to_output = reasoning_to_output.removesuffix("\n")

101 - if self.end_token in delta_text:

102 - # Trailing "\n" is right before </think>, drop it.

103 - self._pending_reasoning_newline = False

104 - else:

105 - # Hold the trailing "\n" until we know whether </think> follows.

106 - self._pending_reasoning_newline = True

107 -

108 - # Content: handle the newline immediately after </think>.

109 - if content_to_output is not None:

110 - # If we have content, reasoning must have ended.

111 - self._pending_reasoning_newline = False

112 -

113 - if self.end_token in delta_text and content_to_output.startswith("\n"):

114 - content_to_output = content_to_output.removeprefix("\n")

115 -

116 - reasoning_to_output = reasoning_to_output or None

117 - content_to_output = content_to_output or None

118 - if reasoning_to_output is None and content_to_output is None:

119 - return None

120 -

121 - return DeltaMessage(reasoning=reasoning_to_output, content=content_to_output)

vllm/tool_parsers/step3p5_tool_parser.py

+2 -1458

@@ -1,1464 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -import json

4 -from collections.abc import Sequence

5 -from typing import Any

6 -from xml.parsers.expat import ParserCreate

8 -import regex as re

4 +from vllm.parser.engine.registered_adapters import Step3p5ParserToolAdapter

10 -from vllm.entrypoints.chat_utils import make_tool_call_id

11 -from vllm.entrypoints.generate.base.protocol import (

12 - DeltaFunctionCall,

13 - DeltaMessage,

14 - DeltaToolCall,

15 - ExtractedToolCallInformation,

16 - FunctionCall,

17 - ToolCall,

18 -)

19 -from vllm.entrypoints.openai.chat_completion.protocol import (

20 - ChatCompletionRequest,

21 -)

22 -from vllm.logger import init_logger

23 -from vllm.tokenizers import TokenizerLike

24 -from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser

25 -from vllm.tool_parsers.utils import (

26 - find_tool_name,

27 - find_tool_properties,

28 - safe_literal_eval,

29 -)

31 -logger = init_logger(__name__)

32 -

33 -

34 -class StreamingXMLToolCallParser:

35 - """Simplified streaming XML tool call parser

36 - Supports streaming input, parsing, and output

37 - """

38 -

39 - def __init__(self):

40 - self.reset_streaming_state()

41 -

42 - # Tool configuration information

43 - self.tools: list[Tool] | None = None

44 - self.tool_call_start_token: str = "<tool_call>"

45 - self.tool_call_end_token: str = "</tool_call>"

46 - self.function_start_token: str = "<function="

47 - self.function_end_token: str = "</function>"

48 - self.parameter_start_token: str = "<parameter="

49 - self.parameter_end_token: str = "</parameter>"

50 -

51 - def reset_streaming_state(self):

52 - """Reset streaming parsing state."""

53 - self.deltas = []

54 - # state for streaming

55 - self.tool_call_index = 0

56 - self.current_call_id = None

57 - self.last_completed_call_id = None

58 - self.current_function_name = None

59 - self.current_function_open = False

60 - self.parameters = {}

61 - self.current_param_name = None

62 - self.current_param_value = ""

63 - self.current_param_value_converted = ""

64 - self.current_param_is_first = False

65 - self.should_emit_end_newline = False

66 - self.start_quote_emitted = False

67 -

68 - self.streaming_buffer = ""

69 - self.last_processed_pos = 0

70 -

71 - self.text_content_buffer = ""

72 -

73 - # state for preprocessing and deferred parsing

74 - self._pre_inside_parameter = False

75 - self._pre_param_buffer = ""

76 - self._pre_current_param_name = None

77 - self.defer_current_parameter = False

78 - self.deferred_param_raw_value = ""

79 -

80 - # recreate parser

81 - self.parser = ParserCreate()

82 - self.setup_parser()

83 -

84 - def parse_single_streaming_chunks(self, xml_chunk: str) -> DeltaMessage:

85 - """Parse single streaming XML chunk and return Delta response

86 - This is the actual streaming interface that receives chunks

87 - one by one and maintains internal state

88 -

89 - Args:

90 - xml_chunk: Single XML chunk string

91 - Returns:

92 - DeltaMessage: Contains delta information generated by this chunk,

93 - returns empty response if no complete elements

94 -

95 - """

96 - # Record delta count before processing

97 - initial_delta_count = len(self.deltas)

98 - entry_call_id = self.current_call_id

99 - entry_tool_call_index = self.tool_call_index

100 -

101 - self.streaming_buffer += xml_chunk

102 -

103 - found_elements = self._process_complete_xml_elements()

104 -

105 - fallback_call_id = None

106 - if entry_call_id is not None:

107 - if (

108 - self.current_call_id == entry_call_id

109 - and self.tool_call_index == entry_tool_call_index

110 - ):

111 - fallback_call_id = entry_call_id

112 - elif (

113 - self.current_call_id is not None

114 - and self.tool_call_index == entry_tool_call_index + 1

115 - ):

116 - fallback_call_id = self.current_call_id

117 -

118 - if found_elements:

119 - # If complete elements found, check if end events were missed

120 - # some tags may not have been triggered

121 - try:

122 - new_deltas = self.deltas[initial_delta_count:]

123 - # If this chunk contains </function>

124 - # but didn't generate '}', then complete it

125 - if (

126 - fallback_call_id is not None

127 - and self.function_end_token in xml_chunk

128 - ):

129 - # - Added '}' (non-empty parameter ending)

130 - # - Added '{}' (empty parameter function)

131 - has_function_close = any(

132 - (

133 - td.tool_calls

134 - and any(

135 - (

136 - tc.function

137 - and tc.id == fallback_call_id

138 - and isinstance(tc.function.arguments, str)

139 - and (tc.function.arguments in ("}", "{}"))

140 - )

141 - for tc in td.tool_calls

142 - )

143 - )

144 - for td in new_deltas

145 - )

146 - if not has_function_close:

147 - # Close potentially unclosed element

148 - if self.current_param_name:

149 - self._end_element("parameter")

150 - if self.current_function_name:

151 - self._end_element("function")

152 - # If this chunk contains </tool_call>

153 - # but didn't generate final empty delta, then complete it

154 - if (

155 - fallback_call_id is not None

156 - and self.tool_call_end_token in xml_chunk

157 - ):

158 - has_toolcall_close = any(

159 - (

160 - td.tool_calls

161 - and any(

162 - (

163 - tc.type == "function"

164 - and tc.function

165 - and tc.function.arguments == ""

166 - and tc.id == fallback_call_id

167 - )

168 - for tc in td.tool_calls

169 - )

170 - )

171 - for td in new_deltas

172 - )

173 - if not has_toolcall_close:

174 - # Close potentially unclosed element

175 - if self.current_param_name:

176 - self._end_element("parameter")

177 - if self.current_function_name:

178 - self._end_element("function")

179 - self._end_element("tool_call")

180 - except Exception as e:

181 - logger.warning("Error with fallback parsing: %s", e)

182 - # Merge newly generated deltas into single response

183 - result_delta = self._merge_new_deltas_to_single_response(

184 - initial_delta_count

185 - )

186 - return result_delta

187 - else:

188 - # No complete elements, check if there's unoutput text content

189 - if self.text_content_buffer and self.tool_call_index == 0:

190 - # Has text content but no tool_call yet, output text content

191 - text_delta = DeltaMessage(content=self.text_content_buffer)

192 - self._emit_delta(text_delta)

193 - # Clear buffer to avoid duplicate output

194 - self.text_content_buffer = ""

195 - return text_delta

196 -

197 - # If this chunk contains end tags but wasn't triggered by parser,

198 - # manually complete end events

199 - # Only execute when still on the same call as when entered,

200 - # to prevent accidentally closing new calls

201 - # in multi <tool_call> scenarios

202 - if fallback_call_id is not None and (

203 - self.function_end_token in xml_chunk

204 - or self.tool_call_end_token in xml_chunk

205 - ):

206 - # Close potentially unclosed element

207 - if self.current_param_name:

208 - self._end_element("parameter")

209 - if self.function_end_token in xml_chunk and self.current_function_name:

210 - self._end_element("function")

211 - if self.tool_call_end_token in xml_chunk:

212 - self._end_element("tool_call")

213 - # Return the merged delta result generated by this fallback

214 - result_delta = self._merge_new_deltas_to_single_response(

215 - initial_delta_count

216 - )

217 - return result_delta

218 -

219 - # No complete elements, return empty response

220 - return DeltaMessage(content=None)

221 -

222 - def _escape_xml_special_chars(self, text: str) -> str:

223 - """Escape XML special characters

224 - Args:

225 - text: Original text

226 - Returns:

227 - Escaped text

228 - """

229 - xml_escapes = {

230 - "&": "&",

231 - "<": "<",

232 - ">": ">",

233 - '"': """,

234 - "'": "'",

235 - }

236 -

237 - for char, escape in xml_escapes.items():

238 - text = text.replace(char, escape)

239 -

240 - return text

241 -

242 - def _process_complete_xml_elements(self) -> bool:

243 - """Process complete XML elements in buffer.

244 -

245 - Returns:

246 - bool: Whether complete elements were found and processed

247 -

248 - """

249 - found_any = False

250 -

251 - while self.last_processed_pos < len(self.streaming_buffer):

252 - # Find next complete xml element

253 - element, end_pos = self._find_next_complete_element(self.last_processed_pos)

254 - if element is None:

255 - # No complete element found, wait for more data

256 - break

257 -

258 - # Check if this element should be skipped

259 - if self._should_skip_element(element):

260 - self.last_processed_pos = end_pos

261 - continue

262 -

263 - # Found complete XML element, process it

264 - try:

265 - preprocessed_element = self._preprocess_xml_chunk(element)

266 - # Check if this is the first tool_call start

267 - if (

268 - (

269 - preprocessed_element.strip().startswith("<tool_call>")

270 - or preprocessed_element.strip().startswith("<function name=")

271 - )

272 - and self.tool_call_index == 0

273 - ) and self.text_content_buffer:

274 - # First tool_call starts,

275 - # output previously collected text content first

276 - text_delta = DeltaMessage(content=self.text_content_buffer)

277 - self._emit_delta(text_delta)

278 - # Clear buffer for potential subsequent text content

279 - self.text_content_buffer = ""

280 -

281 - # If a new tool_call starts and

282 - # there are already completed tool_calls with function name

283 - if (

284 - preprocessed_element.strip().startswith("<tool_call>")

285 - and self.tool_call_index > 0

286 - and self.current_call_id

287 - and self.current_function_name

288 - ):

289 - # Reset parser state but preserve generated deltas

290 - if self.current_param_name:

291 - self._end_element("parameter")

292 - if self.current_function_open:

293 - self._end_element("function")

294 - # Output final tool_call tail delta

295 - final_delta = DeltaMessage(

296 - role=None,

297 - content=None,

298 - reasoning=None,

299 - tool_calls=[

300 - DeltaToolCall(

301 - index=self.tool_call_index - 1,

302 - id=self.current_call_id,

303 - type="function",

304 - function=DeltaFunctionCall(name=None, arguments=""),

305 - )

306 - ],

307 - )

308 - self._emit_delta(final_delta)

309 - # Reset XML parser and current call state

310 - self._reset_xml_parser_after_tool_call()

311 - # Parse preprocessed element

312 - self.parser.Parse(preprocessed_element, False)

313 - found_any = True

314 -

315 - except Exception as e:

316 - logger.warning("Error when parsing XML elements: %s", e)

317 -

318 - # Update processed position

319 - self.last_processed_pos = end_pos

320 -

321 - return found_any

322 -

323 - def _fix_incomplete_tag_in_chunk(self, chunk: str) -> str:

324 - """Fallback: fix incomplete <parameter=xxx or <function=xxx tags

325 - (missing >)

326 - Examples: <parameter=-C: -> <parameter=-C>, <parameter=parameter=-n:

327 - -> <parameter=-n>

328 - Also handles missing = cases: <function xxx> -> <function=xxx>,

329 - <functionxxx> -> <function=xxx>

330 - Only fixes tags that pass validation (parameter exists in tool definition)

331 - """

332 - # First, handle missing = cases for function tags

333 - chunk = self._fix_missing_equals_in_function_tag(chunk)

334 -

335 - for tag_type in ["parameter", "function"]:

336 - pattern = f"<{tag_type}="

337 - if pattern not in chunk:

338 - continue

339 -

340 - start_idx = chunk.find(pattern)

341 - after_tag = chunk[start_idx:]

342 - gt_pos = after_tag.find(">")

343 - lt_pos = after_tag.find("<", len(pattern))

344 -

345 - # Skip if already well-formed

346 - if (

347 - gt_pos != -1

348 - and (lt_pos == -1 or gt_pos < lt_pos)

349 - and pattern in after_tag[:gt_pos]

350 - ):

351 - continue

352 -

353 - # Extract tag name (stop at space, newline, or <)

354 - content = chunk[start_idx + len(pattern) :]

355 - end_pos = next(

356 - (i for i, ch in enumerate(content) if ch in (" ", "\n", "<")),

357 - len(content),

358 - )

359 - tag_name = content[:end_pos]

360 -

361 - if not tag_name:

362 - continue

363 -

364 - # Remove duplicate prefix: <parameter=parameter=xxx -> <parameter=xxx

365 - if tag_name.startswith(f"{tag_type}="):

366 - tag_name = tag_name[len(tag_type) + 1 :]

367 -

368 - # Remove trailing non-alphanumeric chars (keep - and _)

369 - while tag_name and not (

370 - tag_name[-1].isalnum() or tag_name[-1] in ("-", "_")

371 - ):

372 - tag_name = tag_name[:-1]

373 -

374 - if not tag_name:

375 - continue

376 -

377 - # Validate parameter exists in tool definition

378 - if tag_type == "parameter" and not self._validate_parameter_name(tag_name):

379 - continue

380 -

381 - # Apply fix

382 - chunk = chunk.replace(

383 - f"<{tag_type}={content[:end_pos]}", f"<{tag_type}={tag_name}>", 1

384 - )

385 -

386 - return chunk

387 -

388 - def _fix_missing_equals_in_function_tag(self, chunk: str) -> str:

389 - """Fix missing = in function tags: <function xxx> or <functionxxx>

390 - Examples:

391 - <function execute_bash> -> <function=execute_bash>

392 - <functionexecute_bash> -> <function=execute_bash>

393 - Only fixes if function name exists in tool definition

394 - """

395 - # already correct

396 - if "<function=" in chunk:

397 - return chunk

398 -

399 - # Pattern 1: <function xxx> (with space/newline but no =)

400 - pattern1 = r"<function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*>"

401 - match1 = re.search(pattern1, chunk)

402 - if match1:

403 - func_name = match1.group(1).strip()

404 - # must validate function name exists before fixing

405 - if func_name and self._validate_function_name(func_name):

406 - original = match1.group(0)

407 - fixed = f"<function={func_name}>"

408 - chunk = chunk.replace(original, fixed, 1)

409 - return chunk

410 -

411 - # Pattern 2: <functionxxx> (no space, no =)

412 - # only match <function followed by letters

413 - pattern2 = r"<function([a-zA-Z_][a-zA-Z0-9_]*)\s*>"

414 - match2 = re.search(pattern2, chunk)

415 - if match2:

416 - func_name = match2.group(1).strip()

417 - # must validate function name exists before fixing

418 - if func_name and self._validate_function_name(func_name):

419 - original = match2.group(0)

420 - fixed = f"<function={func_name}>"

421 - chunk = chunk.replace(original, fixed, 1)

422 - return chunk

423 -

424 - return chunk

425 -

426 - def _validate_function_name(self, func_name: str) -> bool:

427 - """Check if function name exists in tool definitions."""

428 - return find_tool_name(self.tools, func_name)

429 -

430 - def _validate_parameter_name(self, param_name: str) -> bool:

431 - """Check if parameter exists in current function's tool definition."""

432 - if not self.tools or not self.current_function_name:

433 - return True

434 - properties = find_tool_properties(self.tools, self.current_function_name)

435 - return not properties or param_name in properties

436 -

437 - def _should_skip_element(self, element: str) -> bool:

438 - """Determine whether an element should be skipped.

439 -

440 - Args:

441 - element: Element to evaluate

442 -

443 - Returns:

444 - bool: True means should skip, False means should process

445 -

446 - """

447 - # If it's a tool_call XML tag, don't skip

448 - if (

449 - element.startswith(self.tool_call_start_token)

450 - or element.startswith(self.function_start_token)

451 - or element.startswith(self.parameter_start_token)

452 - ):

453 - return False

454 -

455 - # If currently not parsing tool calls and not blank,

456 - # collect this text instead of skipping

457 - # Only process other XML elements after tool_call appears,

458 - # otherwise treat as plain text

459 - if self.current_call_id is None and element:

460 - # Collect text content to buffer

461 - self.text_content_buffer += element

462 - return True # Still skip, but content has been collected

463 -

464 - # If currently parsing tool calls,

465 - # this might be parameter value, don't skip

466 - if self.current_call_id is not None:

467 - return False

468 -

469 - # Skip blank content

470 - return not element

471 -

472 - def _find_next_complete_element(self, start_pos: int) -> tuple[str | None, int]:

473 - """Find next complete XML element from specified position.

474 -

475 - Args:

476 - start_pos: Position to start searching

477 -

478 - Returns:

479 - (Complete element string, element end position),

480 - returns (None, start_pos) if no complete element found

481 -

482 - """

483 - buffer = self.streaming_buffer[start_pos:]

484 -

485 - if not buffer:

486 - return None, start_pos

487 -

488 - if buffer.startswith("<"):

489 - # Check if this is an incomplete parameter/function tag

490 - # e.g., <parameter=-C: or <function=xxx

491 - is_incomplete_param = (

492 - buffer.startswith("<parameter=") and ">" not in buffer.split("\n")[0]

493 - )

494 - is_incomplete_func = (

495 - buffer.startswith("<function=") and ">" not in buffer.split("\n")[0]

496 - )

497 -

498 - if is_incomplete_param or is_incomplete_func:

499 - # Find the corresponding closing tag

500 - tag_type = "parameter" if is_incomplete_param else "function"

501 - closing_tag = f"</{tag_type}>"

502 - closing_pos = buffer.find(closing_tag)

503 -

504 - if closing_pos != -1:

505 - # Found closing tag, return complete element including closing tag

506 - complete_element = buffer[: closing_pos + len(closing_tag)]

507 - return complete_element, start_pos + closing_pos + len(closing_tag)

508 -

509 - # Need to ensure no new < appears,

510 - # find the nearest one between < and >

511 - tag_end = buffer.find("<", 1)

512 - tag_end2 = buffer.find(">", 1)

513 - if tag_end != -1 and tag_end2 != -1:

514 - # Next nearest is <

515 - if tag_end < tag_end2:

516 - return buffer[:tag_end], start_pos + tag_end

517 - # Next nearest is >, means found XML element

518 - else:

519 - return buffer[: tag_end2 + 1], start_pos + tag_end2 + 1

520 - elif tag_end != -1:

521 - return buffer[:tag_end], start_pos + tag_end

522 - elif tag_end2 != -1:

523 - return buffer[: tag_end2 + 1], start_pos + tag_end2 + 1

524 - else:

525 - # If currently not parsing tool calls (entering a tool_call),

526 - # check if starts with <tool_call> or <function=

527 - if self.current_call_id is None:

528 - # Check if might be start of <tool_call>

529 - if buffer == "<tool_call>"[: len(buffer)]:

530 - # Might be start of <tool_call>, wait for more data

531 - return None, start_pos

532 - elif (

533 - buffer.startswith("<function=")

534 - or buffer == "<function="[: len(buffer)]

535 - ):

536 - # Might be start of <function=, wait for more data

537 - # to get the complete function tag

538 - return None, start_pos

539 - else:

540 - # Not start of <tool_call> or <function=, treat as text

541 - return buffer, start_pos + len(buffer)

542 - else:

543 - # When parsing tool calls,

544 - # wait for more data to get complete tag

545 - return None, start_pos

546 - else:

547 - # Find text content (until next < or buffer end)

548 - next_tag_pos = buffer.find("<")

549 - if next_tag_pos != -1:

550 - # Found text content

551 - text_content = buffer[:next_tag_pos]

552 - return text_content, start_pos + next_tag_pos

553 - else:

554 - # Buffer end is all text, process

555 - # (no longer wait for more data)

556 - remaining = buffer

557 - return remaining, start_pos + len(remaining)

558 -

559 - def _merge_new_deltas_to_single_response(self, initial_count: int) -> DeltaMessage:

560 - """Merge newly generated deltas from this processing

561 - into a single DeltaMessage

562 -

563 - Args:

564 - initial_count: Delta count before processing

565 -

566 - Returns:

567 - Merged DeltaMessage containing all newly generated delta information

568 -

569 - """

570 - if len(self.deltas) <= initial_count:

571 - return DeltaMessage(content=None)

572 -

573 - # Get newly generated deltas

574 - new_deltas = self.deltas[initial_count:]

575 -

576 - if len(new_deltas) == 1:

577 - # Only one new delta, return directly

578 - return new_deltas[0]

579 -

580 - # Merge multiple new deltas

581 - merged_tool_calls: list[DeltaToolCall] = []

582 - merged_content: str = ""

583 -

584 - for delta in new_deltas:

585 - if delta.content:

586 - merged_content += delta.content

587 - if delta.tool_calls:

588 - # For tool_calls, we need to intelligently merge arguments

589 - for tool_call in delta.tool_calls:

590 - # Find if there's already a tool_call with the same call_id

591 - existing_call = None

592 - for existing in merged_tool_calls:

593 - if existing.id == tool_call.id:

594 - existing_call = existing

595 - break

596 -

597 - if existing_call and existing_call.function:

598 - # Merge to existing tool_call

599 - if tool_call.function and tool_call.function.name:

600 - existing_call.function.name = tool_call.function.name

601 - if (

602 - tool_call.function

603 - and tool_call.function.arguments is not None

604 - ):

605 - if existing_call.function.arguments is None:

606 - existing_call.function.arguments = ""

607 -

608 - # For streaming JSON parameters,

609 - # simply concatenate in order

610 - new_args = tool_call.function.arguments

611 - existing_call.function.arguments += new_args

612 - if tool_call.type:

613 - existing_call.type = tool_call.type

614 - else:

615 - # Add new tool_call

616 - merged_tool_calls.append(tool_call)

617 -

618 - return DeltaMessage(

619 - content=merged_content if merged_content else None,

620 - tool_calls=merged_tool_calls,

621 - )

622 -

623 - def _preprocess_xml_chunk(self, chunk: str) -> str:

624 - """Preprocess XML chunk, handle non-standard formats,

625 - and escape special characters

626 -

627 - Args:

628 - chunk: Original XML chunk

629 -

630 - Returns:

631 - Processed XML chunk

632 -

633 - """

634 - # Check if this is a tool_call related element

635 - is_tool_call = False

636 - if chunk.startswith(self.tool_call_start_token) or chunk.startswith(

637 - self.tool_call_end_token

638 - ):

639 - is_tool_call = True

640 - # Check for function tags (including malformed ones without =)

641 - # <function=xxx>, </function>, <function xxx>, <functionxxx>

642 - if (

643 - chunk.startswith(self.function_start_token)

644 - or chunk.startswith(self.function_end_token)

645 - or chunk.startswith("<function ")

646 - or re.match(r"^<function[a-zA-Z_]", chunk)

647 - ): # <functionXXX without space or =

648 - is_tool_call = True

649 - if chunk.startswith(self.parameter_start_token) or chunk.startswith(

650 - self.parameter_end_token

651 - ):

652 - is_tool_call = True

653 -

654 - # Fallback: fix incomplete <parameter= or <function= tags without

655 - # closing >

656 - # This handles cases like: <parameter=-C:\n or <parameter=-B 5\n

657 - # Apply when parsing tool calls OR when chunk looks like a function/

658 - # parameter tag

659 - if (

660 - self.current_call_id is not None

661 - or chunk.startswith("<function")

662 - or chunk.startswith("<parameter")

663 - ):

664 - chunk = self._fix_incomplete_tag_in_chunk(chunk)

665 -

666 - # Handle <function=name> format -> <function name="name">

667 - processed = re.sub(r"<function=([^>]+)>", r'<function name="\1">', chunk)

668 - # Handle <parameter=name> format -> <parameter name="name">

669 - processed = re.sub(r"<parameter=([^>]+)>", r'<parameter name="\1">', processed)

670 -

671 - original_chunk = chunk

672 - # If in parameter value accumulation mode

673 - if self._pre_inside_parameter:

674 - # Parameter end: output accumulated raw text

675 - # safely then return </parameter>

676 - if processed.startswith("</parameter>"):

677 - body_text = self._pre_param_buffer

678 - # Trigger deferred parsing mode

679 - # literal_eval+json output in end_element

680 - self.defer_current_parameter = True

681 - self.deferred_param_raw_value = body_text

682 - # Clean up state

683 - self._pre_inside_parameter = False

684 - self._pre_param_buffer = ""

685 - self._pre_current_param_name = None

686 - safe_text = self._escape_xml_special_chars(body_text)

687 - return f"{safe_text}</parameter>"

688 - else:

689 - # If this is the first block of content after entering parameter

690 - # evaluate if deferred parsing is needed;

691 - # If not needed, exit accumulation mode

692 - # and pass through directly

693 - if self._pre_param_buffer == "":

694 - # Get current parameter type

695 - param_type = (

696 - self._get_param_type(self._pre_current_param_name)

697 - if self._pre_current_param_name

698 - else "string"

699 - )

700 - # Only these types need deferred parsing to

701 - # handle Python literals containing single quotes

702 - is_object_type = param_type in ["object"]

703 - is_complex_type = (

704 - param_type in ["array", "arr", "sequence"]

705 - or param_type.startswith("dict")

706 - or param_type.startswith("list")

707 - )

708 -

709 - # Only delay when contains container symbols

710 - # and has single quotes and is complex type

711 - has_container_hint = (

712 - ("[" in original_chunk)

713 - or ("{" in original_chunk)

714 - or ("(" in original_chunk)

715 - )

716 -

717 - # Determine if deferred parsing is needed

718 - need_defer = False

719 - if is_complex_type:

720 - # Complex type, always need deferred parsing

721 - need_defer = True

722 - elif (

723 - is_object_type

724 - and has_container_hint

725 - and ("'" in original_chunk)

726 - ):

727 - # Object type with container symbols

728 - # and single quotes, need deferred parsing

729 - need_defer = True

730 -

731 - if not need_defer:

732 - # No need for deferred parsing,

733 - # exit parameter mode directly

734 - self._pre_inside_parameter = False

735 - return self._escape_xml_special_chars(original_chunk)

736 - self._pre_param_buffer += original_chunk

737 - return ""

738 -

739 - # Parameter start: enable accumulation

740 - if processed.startswith("<parameter name="):

741 - m = re.match(r'<parameter name="([^"]+)">', processed)

742 - if m:

743 - self._pre_current_param_name = m.group(1)

744 - self._pre_inside_parameter = True

745 - self._pre_param_buffer = ""

746 - return processed

747 -

748 - # If processed doesn't contain special_token, escape processed

749 - # This is because XML parsing encounters special characters

750 - # and reports errors, so escaping is needed

751 - if not is_tool_call:

752 - processed = self._escape_xml_special_chars(processed)

753 - return processed

754 -

755 - def _emit_delta(self, delta: DeltaMessage):

756 - """Emit Delta response (streaming output)."""

757 - self.deltas.append(delta)

758 -

759 - def _auto_close_open_parameter_if_needed(self, incoming_tag: str | None = None):

760 - """Before starting to process new elements,

761 - if there are unclosed tags from before,

762 - automatically complete their endings to the parser.

763 - - If there are unclosed parameters,

764 - it's equivalent to feeding `</parameter>`

765 - - When about to start a new function or tool_call,

766 - if there are unclosed functions, complete `</function>`.

767 - - When about to start a new tool_call,

768 - if there are unclosed tool_calls, complete `</tool_call>`.

769 - """

770 - # First close unclosed parameters

771 - if self.current_param_name:

772 - self._end_element("parameter")

773 -

774 - # If about to start new function or tool_call,

775 - # and there are unclosed functions, close function first

776 - if incoming_tag in ("function", "tool_call") and self.current_function_name:

777 - self._end_element("function")

778 -

779 - # If about to start new tool_call,

780 - # and there are unclosed tool_calls, close tool_call first

781 - if incoming_tag == "tool_call" and self.current_call_id:

782 - self._end_element("tool_call")

783 -

784 - def _start_element(self, name: str, attrs: dict[str, str]):

785 - """Handle XML start element events."""

786 - if name == "root":

787 - return

788 -

789 - if name == "tool_call":

790 - # Before opening new tool_call,

791 - # automatically complete previous unclosed tags

792 - self._auto_close_open_parameter_if_needed("tool_call")

793 -

794 - self.parameters = {}

795 - self.current_call_id = make_tool_call_id()

796 - self.current_param_is_first = True

797 - self.tool_call_index += 1

798 - elif name.startswith("function") or (name == "function"):

799 - # If missing tool_call, manually complete

800 - if not self.current_call_id:

801 - self._start_element("tool_call", {})

802 - # Before opening new function,

803 - # automatically complete previous unclosed tags (parameter/function)

804 - self._auto_close_open_parameter_if_needed("function")

805 - function_name = self._extract_function_name(name, attrs)

806 - self.current_function_name = function_name

807 - self.current_function_open = True

808 - if function_name:

809 - delta = DeltaMessage(

810 - tool_calls=[

811 - DeltaToolCall(

812 - index=self.tool_call_index - 1,

813 - id=self.current_call_id,

814 - type="function",

815 - function=DeltaFunctionCall(

816 - name=function_name, arguments=""

817 - ),

818 - )

819 - ]

820 - )

821 - self._emit_delta(delta)

822 - elif name.startswith("parameter") or (name == "parameter"):

823 - # If previous parameter hasn't ended normally,

824 - # complete its end first, then start new parameter

825 - self._auto_close_open_parameter_if_needed("parameter")

826 - param_name = self._extract_parameter_name(name, attrs)

827 - self.current_param_name = param_name

828 - self.current_param_value = ""

829 - self.current_param_value_converted = ""

830 - self.start_quote_emitted = False # Reset start quote flag

831 -

832 - # Only output parameter name and colon,

833 - # don't output quotes

834 - # decide after parameter value type is determined

835 - if param_name:

836 - if not self.parameters:

837 - # First parameter

838 - # start JSON, only output parameter name and colon

839 - json_start = f'{{"{param_name}": '

840 - delta = DeltaMessage(

841 - tool_calls=[

842 - DeltaToolCall(

843 - index=self.tool_call_index - 1,

844 - id=self.current_call_id,

845 - type="function",

846 - function=DeltaFunctionCall(

847 - name=None, arguments=json_start

848 - ),

849 - )

850 - ]

851 - )

852 - self._emit_delta(delta)

853 - self.current_param_is_first = True

854 - else:

855 - # Subsequent parameters

856 - # add comma and parameter name, no quotes

857 - json_continue = f', "{param_name}": '

858 - delta = DeltaMessage(

859 - tool_calls=[

860 - DeltaToolCall(

861 - index=self.tool_call_index - 1,

862 - id=self.current_call_id,

863 - type="function",

864 - function=DeltaFunctionCall(

865 - name=None, arguments=json_continue

866 - ),

867 - )

868 - ]

869 - )

870 - self._emit_delta(delta)

871 - self.current_param_is_first = False

872 -

873 - def _char_data(self, data: str):

874 - """Handle XML character data events."""

875 - if data and self.current_param_name:

876 - # If preprocessing stage determines deferred parsing is needed,

877 - # only cache character data, no streaming output

878 - if self.defer_current_parameter:

879 - original_data = data

880 - if self.should_emit_end_newline:

881 - original_data = "\n" + original_data

882 - self.should_emit_end_newline = False

883 - if original_data.endswith("\n"):

884 - self.should_emit_end_newline = True

885 - original_data = original_data[:-1]

886 - self.current_param_value += original_data

887 - return

888 -

889 - param_type = self._get_param_type(self.current_param_name)

890 -

891 - # Check if this is the first time receiving data for this parameter

892 - # If this is the first packet of data and starts with \n, remove \n

893 - if not self.current_param_value and data.startswith("\n"):

894 - data = data[1:]

895 -

896 - # Output start quote for string type (if not already output)

897 - if (

898 - param_type in ["string", "str", "text", "varchar", "char", "enum"]

899 - and not self.start_quote_emitted

900 - ):

901 - quote_delta = DeltaMessage(

902 - tool_calls=[

903 - DeltaToolCall(

904 - index=self.tool_call_index - 1,

905 - id=self.current_call_id,

906 - type="function",

907 - function=DeltaFunctionCall(name=None, arguments='"'),

908 - )

909 - ]

910 - )

911 - self._emit_delta(quote_delta)

912 - self.start_quote_emitted = True

913 -

914 - if not data:

915 - return

916 -

917 - original_data = data

918 - # Delay output of trailing newline

919 - if self.should_emit_end_newline:

920 - original_data = "\n" + original_data

921 - self.should_emit_end_newline = False

922 - if original_data.endswith("\n"):

923 - self.should_emit_end_newline = True

924 - original_data = original_data[:-1]

925 - self.current_param_value += original_data

926 -

927 - # convert parameter value by param_type

928 - converted_value = self._convert_param_value(

929 - self.current_param_value, param_type

930 - )

931 - output_data = self._convert_for_json_streaming(converted_value, param_type)

932 -

933 - delta_data = output_data[len(self.current_param_value_converted) :]

934 - self.current_param_value_converted = output_data

935 -

936 - delta = DeltaMessage(

937 - tool_calls=[

938 - DeltaToolCall(

939 - index=self.tool_call_index - 1,

940 - id=self.current_call_id,

941 - type="function",

942 - function=DeltaFunctionCall(name=None, arguments=delta_data),

943 - )

944 - ]

945 - )

946 - self._emit_delta(delta)

947 -

948 - def _end_element(self, name: str):

949 - """Handle XML end element events."""

950 - if name == "root":

951 - return

952 -

953 - # If function or tool_call ends and there are still unclosed parameters,

954 - # complete parameter end first

955 - if (

956 - name.startswith("function") or name == "function" or name == "tool_call"

957 - ) and self.current_param_name:

958 - self._auto_close_open_parameter_if_needed()

959 -

960 - if (

961 - name.startswith("parameter") or name == "parameter"

962 - ) and self.current_param_name:

963 - # End current parameter

964 - param_name = self.current_param_name

965 - param_value = self.current_param_value

966 -

967 - # If in deferred parsing mode,

968 - # perform overall parsing on raw content

969 - # accumulated in preprocessing stage and output once

970 - if self.defer_current_parameter:

971 - raw_text = (

972 - self.deferred_param_raw_value

973 - if self.deferred_param_raw_value

974 - else param_value

975 - )

976 - parsed_value = None

977 - output_arguments = None

978 - try:

979 - # If previously delayed trailing newline,

980 - # add it back before parsing

981 - if self.should_emit_end_newline:

982 - raw_for_parse = raw_text + "\n"

983 - else:

984 - raw_for_parse = raw_text

985 - parsed_value = safe_literal_eval(raw_for_parse)

986 - output_arguments = json.dumps(parsed_value, ensure_ascii=False)

987 - except Exception:

988 - # Fallback: output as string as-is

989 - output_arguments = json.dumps(raw_text, ensure_ascii=False)

990 - parsed_value = raw_text

991 -

992 - delta = DeltaMessage(

993 - tool_calls=[

994 - DeltaToolCall(

995 - index=self.tool_call_index - 1,

996 - id=self.current_call_id,

997 - type="function",

998 - function=DeltaFunctionCall(

999 - name=None, arguments=output_arguments

1000 - ),

1001 - )

1002 - ]

1003 - )

1004 - self._emit_delta(delta)

1005 -

1006 - # Clean up and store

1007 - self.should_emit_end_newline = False

1008 - self.parameters[param_name] = parsed_value

1009 - self.current_param_name = None

1010 - self.current_param_value = ""

1011 - self.current_param_value_converted = ""

1012 - self.start_quote_emitted = False

1013 - self.defer_current_parameter = False

1014 - self.deferred_param_raw_value = ""

1015 - return

1016 -

1017 - param_type = self._get_param_type(param_name)

1018 -

1019 - # convert complete parameter value by param_type

1020 - converted_value = self._convert_param_value(param_value, param_type)

1021 -

1022 - # Decide whether to add end quote based on parameter type

1023 - if param_type in ["string", "str", "text", "varchar", "char", "enum"]:

1024 - # For empty string parameters, need special handling

1025 - if not param_value and not self.start_quote_emitted:

1026 - # No start quote output,

1027 - # directly output complete empty string

1028 - delta = DeltaMessage(

1029 - tool_calls=[

1030 - DeltaToolCall(

1031 - index=self.tool_call_index - 1,

1032 - id=self.current_call_id,

1033 - type="function",

1034 - function=DeltaFunctionCall(name=None, arguments='""'),

1035 - )

1036 - ]

1037 - )

1038 - self._emit_delta(delta)

1039 - else:

1040 - # Non-empty parameter value, output end quote

1041 - delta = DeltaMessage(

1042 - tool_calls=[

1043 - DeltaToolCall(

1044 - index=self.tool_call_index - 1,

1045 - id=self.current_call_id,

1046 - type="function",

1047 - function=DeltaFunctionCall(name=None, arguments='"'),

1048 - )

1049 - ]

1050 - )

1051 - self._emit_delta(delta)

1052 -

1053 - self.should_emit_end_newline = False

1054 - # Store converted value

1055 - self.parameters[param_name] = converted_value

1056 - self.current_param_name = None

1057 - self.current_param_value = ""

1058 - self.current_param_value_converted = ""

1059 - self.start_quote_emitted = False

1060 -

1061 - elif name.startswith("function") or name == "function":

1062 - # if there are parameters, close JSON object

1063 - if self.parameters:

1064 - delta = DeltaMessage(

1065 - tool_calls=[

1066 - DeltaToolCall(

1067 - index=self.tool_call_index - 1,

1068 - id=self.current_call_id,

1069 - type="function",

1070 - function=DeltaFunctionCall(name=None, arguments="}"),

1071 - )

1072 - ]

1073 - )

1074 - self._emit_delta(delta)

1075 - # return empty object

1076 - else:

1077 - delta = DeltaMessage(

1078 - tool_calls=[

1079 - DeltaToolCall(

1080 - index=self.tool_call_index - 1,

1081 - id=self.current_call_id,

1082 - type="function",

1083 - function=DeltaFunctionCall(name=None, arguments="{}"),

1084 - )

1085 - ]

1086 - )

1087 - self._emit_delta(delta)

1088 - self.current_function_open = False

1089 - self.current_function_name = (

1090 - None # Clear function name to prevent duplicate closing

1091 - )

1092 -

1093 - elif name == "tool_call":

1094 - # Before ending tool_call,

1095 - # ensure function is closed to complete missing right brace

1096 - if self.current_function_open:

1097 - # If there are still unclosed parameters, close them first

1098 - if self.current_param_name:

1099 - self._end_element("parameter")

1100 - # Close function, ensure output '}' or '{}'

1101 - self._end_element("function")

1102 - # Final Delta

1103 - delta = DeltaMessage(

1104 - tool_calls=[

1105 - DeltaToolCall(

1106 - index=self.tool_call_index - 1,

1107 - id=self.current_call_id,

1108 - type="function",

1109 - function=DeltaFunctionCall(name=None, arguments=""),

1110 - )

1111 - ]

1112 - )

1113 - self._emit_delta(delta)

1114 -

1115 - # Check if there's text content to output (between tool_calls)

1116 - if self.text_content_buffer.strip():

1117 - text_delta = DeltaMessage(content=self.text_content_buffer)

1118 - self._emit_delta(text_delta)

1119 -

1120 - self._reset_xml_parser_after_tool_call()

1121 -

1122 - def setup_parser(self):

1123 - """Set up XML parser event handlers."""

1124 - self.parser.buffer_text = True

1125 - self.parser.StartElementHandler = self._start_element

1126 - self.parser.EndElementHandler = self._end_element

1127 - self.parser.CharacterDataHandler = self._char_data

1128 -

1129 - def set_tools(self, tools: list[Tool] | None):

1130 - """Set tool configuration information."""

1131 - self.tools = tools

1132 -

1133 - def _extract_function_name(self, name: str, attrs: dict[str, str]) -> str | None:

1134 - """Extract function name from various formats."""

1135 - if attrs and "name" in attrs:

1136 - return attrs["name"]

1137 -

1138 - if "=" in name:

1139 - parts = name.split("=", 1)

1140 - if len(parts) == 2 and parts[0] == "function":

1141 - return parts[1]

1142 -

1143 - return None

1144 -

1145 - def _extract_parameter_name(self, name: str, attrs: dict[str, str]) -> str | None:

1146 - """Extract parameter name from various formats."""

1147 - if attrs and "name" in attrs:

1148 - return attrs["name"]

1149 -

1150 - if "=" in name:

1151 - parts = name.split("=", 1)

1152 - if len(parts) == 2 and parts[0] == "parameter":

1153 - return parts[1]

1154 -

1155 - return None

1156 -

1157 - def _get_param_type(self, param_name: str) -> str:

1158 - """Get parameter type based on tool configuration, defaults to string

1159 - Args:

1160 - param_name: Parameter name

1161 -

1162 - Returns:

1163 - Parameter type

1164 -

1165 - """

1166 - if not self.tools or not self.current_function_name:

1167 - return "string"

1168 - properties = find_tool_properties(self.tools, self.current_function_name)

1169 - param_config = properties.get(param_name)

1170 - if isinstance(param_config, dict):

1171 - return self.repair_param_type(str(param_config.get("type", "string")))

1172 - return "string"

1173 -

1174 - def repair_param_type(self, param_type: str) -> str:

1175 - """Repair unknown parameter types by treating them as string

1176 - Args:

1177 - param_type: Parameter type

1178 -

1179 - Returns:

1180 - Repaired parameter type

1181 -

1182 - """

1183 - if (

1184 - param_type in ["string", "str", "text", "varchar", "char", "enum"]

1185 - or param_type.startswith("int")

1186 - or param_type.startswith("uint")

1187 - or param_type.startswith("long")

1188 - or param_type.startswith("short")

1189 - or param_type.startswith("unsigned")

1190 - or param_type.startswith("num")

1191 - or param_type.startswith("float")

1192 - or param_type in ["boolean", "bool", "binary"]

1193 - or (

1194 - param_type in ["object", "array", "arr", "sequence"]

1195 - or param_type.startswith("dict")

1196 - or param_type.startswith("list")

1197 - )

1198 - ):

1199 - return param_type

1200 - else:

1201 - return "string"

1202 -

1203 - def _convert_param_value(self, param_value: str, param_type: str) -> Any:

1204 - """Convert value based on parameter type

1205 - Args:

1206 - param_value: Parameter value

1207 - param_type: Parameter type

1208 -

1209 - Returns:

1210 - Converted value

1211 -

1212 - """

1213 - if param_value.lower() == "null":

1214 - return None

1215 -

1216 - param_type = param_type.strip().lower()

1217 - if param_type in ["string", "str", "text", "varchar", "char", "enum"]:

1218 - return param_value

1219 - elif (

1220 - param_type.startswith("int")

1221 - or param_type.startswith("uint")

1222 - or param_type.startswith("long")

1223 - or param_type.startswith("short")

1224 - or param_type.startswith("unsigned")

1225 - ):

1226 - try:

1227 - return int(param_value)

1228 - except (ValueError, TypeError):

1229 - logger.warning(

1230 - "Parsed value '%s' is not an integer, degenerating to string.",

1231 - param_value,

1232 - )

1233 - return param_value

1234 - elif param_type.startswith("num") or param_type.startswith("float"):

1235 - try:

1236 - float_param_value: float = float(param_value)

1237 - return (

1238 - float_param_value

1239 - if float_param_value - int(float_param_value) != 0

1240 - else int(float_param_value)

1241 - )

1242 - except (ValueError, TypeError):

1243 - logger.warning(

1244 - "Parsed value '%s' is not a float, degenerating to string.",

1245 - param_value,

1246 - )

1247 - return param_value

1248 - elif param_type in ["boolean", "bool", "binary"]:

1249 - param_value = param_value.lower()

1250 - return param_value == "true"

1251 - else:

1252 - return param_value

1253 -

1254 - def _convert_for_json_streaming(self, converted_value: Any, param_type: str) -> str:

1255 - """Convert converted_value based on

1256 - whether it's empty and if type is string

1257 - Args:

1258 - converted_value: Converted value

1259 - param_type: Parameter type

1260 -

1261 - Returns:

1262 - Converted string for streaming output

1263 -

1264 - """

1265 - # Check if value is empty, but exclude numeric 0

1266 - if converted_value is None or converted_value == "":

1267 - return ""

1268 -

1269 - if param_type in ["string", "str", "text", "varchar", "char", "enum"]:

1270 - # String type, remove double quotes

1271 - return json.dumps(converted_value, ensure_ascii=False)[1:-1]

1272 - else:

1273 - # Non-string type, return complete JSON string

1274 - if not isinstance(converted_value, str):

1275 - return json.dumps(converted_value, ensure_ascii=False)

1276 - else:

1277 - return converted_value

1278 -

1279 - def _reset_xml_parser_after_tool_call(self):

1280 - """Each tool_call is treated as a separate XML document,

1281 - so we need to reset the parser after each tool_call.

1282 - """

1283 - # recreate XML parser

1284 - self.parser = ParserCreate()

1285 - self.setup_parser()

1286 -

1287 - # Reset current tool_call state

1288 - if self.current_call_id:

1289 - self.last_completed_call_id = self.current_call_id

1290 - self.current_call_id = None

1291 - self.current_function_name = None

1292 - self.current_function_open = False

1293 - self.parameters = {}

1294 - self.current_param_name = None

1295 - self.current_param_value = ""

1296 - self.current_param_value_converted = ""

1297 - self.current_param_is_first = False

1298 - self.should_emit_end_newline = False

1299 - self.start_quote_emitted = False

1300 - self.text_content_buffer = ""

1301 -

1302 - # Reset preprocessing and deferred parsing state

1303 - self._pre_inside_parameter = False

1304 - self._pre_param_buffer = ""

1305 - self._pre_current_param_name = None

1306 - self.defer_current_parameter = False

1307 - self.deferred_param_raw_value = ""

1308 -

1309 -

1310 -class Step3p5ToolParser(ToolParser):

1311 - # Step-3.5/3.7 chat templates use the Qwen3-Coder XML tool-call format.

7 +class Step3p5ToolParser(Step3p5ParserToolAdapter): # type: ignore[valid-type, misc]

8 structural_tag_model = "qwen_3_coder"

1313 -

1314 - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):

1315 - super().__init__(tokenizer, tools)

1316 - self.parser = StreamingXMLToolCallParser()

1317 -

1318 - # Add missing attributes for compatibility with serving_chat.py

1319 - self.prev_tool_call_arr: list[dict] = []

1320 - self.streamed_args_for_tool: list[str] = []

1321 -

1322 - logger.info(

1323 - "vLLM Successfully import tool parser %s !", self.__class__.__name__

1324 - )

1325 -

1326 - def extract_tool_calls(

1327 - self,

1328 - model_output: str,

1329 - request: ChatCompletionRequest,

1330 - ) -> ExtractedToolCallInformation:

1331 - self.parser.reset_streaming_state()

1332 - # Reset tool call tracking arrays for new extraction

1333 - self.prev_tool_call_arr = []

1334 - self.streamed_args_for_tool = []

1335 - self.parser.set_tools(self.tools)

1336 - result = self.parser.parse_single_streaming_chunks(model_output)

1337 - if not result.tool_calls:

1338 - return ExtractedToolCallInformation(

1339 - tool_calls=[],

1340 - tools_called=False,

1341 - content=result.content,

1342 - )

1343 - else:

1344 - tool_calls = []

1345 - for tool_call in result.tool_calls:

1346 - if tool_call.function and tool_call.function.name:

1347 - tool_calls.append(

1348 - ToolCall(

1349 - id=tool_call.id,

1350 - type=tool_call.type,

1351 - function=FunctionCall(

1352 - name=tool_call.function.name,

1353 - arguments=tool_call.function.arguments,

1354 - ),

1355 - )

1356 - )

1357 -

1358 - # Update tool call tracking arrays for compatibility

1359 - tool_index = (

1360 - tool_call.index

1361 - if tool_call.index is not None

1362 - else len(self.prev_tool_call_arr) - 1

1363 - )

1364 -

1365 - # Ensure we have enough entries in our tracking arrays

1366 - while len(self.prev_tool_call_arr) <= tool_index:

1367 - self.prev_tool_call_arr.append({"name": "", "arguments": ""})

1368 - while len(self.streamed_args_for_tool) <= tool_index:

1369 - self.streamed_args_for_tool.append("")

1370 -

1371 - # Update tool call information

1372 - self.prev_tool_call_arr[tool_index]["name"] = (

1373 - tool_call.function.name

1374 - )

1375 - self.prev_tool_call_arr[tool_index]["arguments"] = (

1376 - tool_call.function.arguments

1377 - )

1378 -

1379 - # Update streamed arguments

1380 - if tool_call.function.arguments:

1381 - self.streamed_args_for_tool[tool_index] = (

1382 - tool_call.function.arguments

1383 - )

1384 -

1385 - return ExtractedToolCallInformation(

1386 - tool_calls=tool_calls,

1387 - tools_called=len(tool_calls) > 0,

1388 - content=result.content,

1389 - )

1390 -

1391 - def extract_tool_calls_streaming(

1392 - self,

1393 - previous_text: str,

1394 - current_text: str,

1395 - delta_text: str,

1396 - previous_token_ids: Sequence[int],

1397 - current_token_ids: Sequence[int],

1398 - delta_token_ids: Sequence[int],

1399 - request: ChatCompletionRequest,

1400 - ) -> DeltaMessage | None:

1401 - if not previous_text:

1402 - self.parser.reset_streaming_state()

1403 - # Reset tool call tracking arrays for new streaming session

1404 - self.prev_tool_call_arr = []

1405 - self.streamed_args_for_tool = []

1406 - self.parser.set_tools(self.tools)

1407 -

1408 - # Model sometimes outputs separately causing delta_text to be empty.

1409 - # If there were tool_calls before and all current tool_calls have ended,

1410 - # return an empty tool_call for outer streaming output

1411 - # to correctly output tool_call field

1412 - if not delta_text and delta_token_ids:

1413 - open_calls = current_text.count(

1414 - self.parser.tool_call_start_token

1415 - ) - current_text.count(self.parser.tool_call_end_token)

1416 - if (

1417 - open_calls == 0

1418 - and self.parser.tool_call_index > 0

1419 - or not self.parser.tool_call_index

1420 - and current_text

1421 - ):

1422 - return DeltaMessage(content="")

1423 - return None

1424 -

1425 - # Parse the delta text and get the result

1426 - result = self.parser.parse_single_streaming_chunks(delta_text)

1427 -

1428 - # Update tool call tracking arrays based on incremental parsing results

1429 - if result and result.tool_calls:

1430 - for tool_call in result.tool_calls:

1431 - if tool_call.function:

1432 - tool_index = (

1433 - tool_call.index

1434 - if tool_call.index is not None

1435 - else len(self.prev_tool_call_arr) - 1

1436 - )

1437 -

1438 - # Ensure we have enough entries in our tracking arrays

1439 - while len(self.prev_tool_call_arr) <= tool_index:

1440 - self.prev_tool_call_arr.append({"name": "", "arguments": ""})

1441 - while len(self.streamed_args_for_tool) <= tool_index:

1442 - self.streamed_args_for_tool.append("")

1443 -

1444 - # Update tool name if provided

1445 - if tool_call.function.name:

1446 - self.prev_tool_call_arr[tool_index]["name"] = (

1447 - tool_call.function.name

1448 - )

1449 -

1450 - # Update arguments incrementally

1451 - if tool_call.function.arguments is not None:

1452 - # Concatenate the incremental arguments

1453 - # to the existing streamed arguments

1454 - self.prev_tool_call_arr[tool_index]["arguments"] += (

1455 - tool_call.function.arguments

1456 - )

1457 - self.streamed_args_for_tool[tool_index] += (

1458 - tool_call.function.arguments

1459 - )

1460 - return result

1461 -

1462 - def parser_should_check_for_unstreamed_tool_arg_tokens(self) -> bool:

1463 - """Skip the remaining_call calculation in serving_chat."""

1464 - return False