@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 6c379b9e54

[Frontend] Add Streaming Parser Engine and new GLM4.7/GLM5.1/GLM5.2 Parser (#45915)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Chauncey committed Jun 19, 2026 at 00:42 UTC 6c379b9e5439ae305913e4a87ebf2b2e816072b4
11 files changed +534 -1948
tests/parser/engine/trace_builder.py
+76
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
30 )
31 from vllm.parser.engine.registered_adapters import (
32 Gemma4Parser,
33 + Glm47MoeParser,
34 MinimaxM2Parser,
35 NemotronV3Parser,
36 Qwen3Parser,
@@ -571,6 +572,80 @@ def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample:
572 )
573
574
575 +# ── GLM-4.7 MoE (XML tool format, starts in REASONING) ──────────────
576 +
577 +_GLM47_MOE_VOCAB: dict[str, int] = {
578 + "<think>": 50,
579 + "</think>": 51,
580 + "<tool_call>": 60,
581 + "</tool_call>": 61,
582 + "<arg_key>": 62,
583 + "</arg_key>": 63,
584 + "<arg_value>": 64,
585 + "</arg_value>": 65,
586 +}
587 +
588 +
589 +def _glm47_moe_arg_value(value: Any) -> str:
590 + if isinstance(value, bool):
591 + return "true" if value else "false"
592 + if isinstance(value, (int, float)):
593 + return str(value)
594 + if isinstance(value, str):
595 + return value
596 + return json.dumps(value, ensure_ascii=False)
597 +
598 +
599 +def _glm47_moe_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
600 + segs: list[tuple[str, bool]] = [
601 + ("<tool_call>", True),
602 + (tc.name, False),
603 + ]
604 + for key, value in tc.arguments.items():
605 + segs.extend(
606 + [
607 + ("<arg_key>", True),
608 + (key, False),
609 + ("</arg_key>", True),
610 + ("<arg_value>", True),
611 + (_glm47_moe_arg_value(value), False),
612 + ("</arg_value>", True),
613 + ]
614 + )
615 + segs.append(("</tool_call>", True))
616 + return segs
617 +
618 +
619 +def _glm47_moe_segments(scenario: Scenario) -> list[tuple[str, bool]]:
620 + segs: list[tuple[str, bool]] = []
621 + if scenario.reasoning is not None:
622 + segs.append((scenario.reasoning, False))
623 + if scenario.content is not None or scenario.tool_calls:
624 + segs.append(("</think>", True))
625 + if scenario.content is not None:
626 + segs.append((scenario.content, False))
627 + if scenario.tool_calls:
628 + for tc in scenario.tool_calls:
629 + segs.extend(_glm47_moe_tool_segments(tc))
630 + return segs
631 +
632 +
633 +def _build_glm47_moe(scenario: Scenario, validate: bool = True) -> Sample:
634 + sample = _make_sample(
635 + sample_id=f"glm47_moe-{scenario.id}",
636 + description=scenario.description,
637 + vocab=_GLM47_MOE_VOCAB,
638 + segments=_glm47_moe_segments(scenario),
639 + expected_reasoning=scenario.reasoning if scenario.reasoning is not None else "",
640 + expected_content=_qwen3_expected_content(scenario),
641 + expected_tool_calls=_expected_tc(scenario),
642 + tools=_expected_tools(scenario),
643 + )
644 + if validate:
645 + _validate_sample(sample, Glm47MoeParser)
646 + return sample
647 +
648 +
649 # ── Registry and public API ──────────────────────────────────────────
650
651 _BUILDERS: dict[str, Any] = {
@@ -578,6 +653,7 @@ _BUILDERS: dict[str, Any] = {
653 "gemma4": _build_gemma4,
654 "minimax_m2": _build_minimax_m2,
655 "nemotron_v3": _build_nemotron_v3,
656 + "glm47_moe": _build_glm47_moe,
657 }
658
659
tests/reasoning/test_glm4_moe_reasoning_parser.py
+31 -7
@@ -11,7 +11,7 @@ parser_name = "glm45"
11 start_token = "<think>"
12 end_token = "</think>"
13
14 -REASONING_MODEL_NAME = "zai-org/GLM-4.5"
14 +REASONING_MODEL_NAME = "zai-org/GLM-4.7"
15
16
17 @pytest.fixture(scope="module")
@@ -35,18 +35,32 @@ WITH_THINK_STREAM = {
35
36 WITHOUT_THINK = {
37 "output": "This is the rest",
38 - "reasoning": None,
39 - "content": "This is the rest",
38 + "reasoning": "This is the rest",
39 + "content": None,
40 "is_reasoning_end": False,
41 }
42
43 WITHOUT_THINK_STREAM = {
44 "output": "This is the rest",
45 - "reasoning": None,
46 - "content": "This is the rest",
45 + "reasoning": "This is the rest",
46 + "content": None,
47 "is_reasoning_end": False,
48 }
49
50 +WITHOUT_OPEN_THINK = {
51 + "output": "This is a reasoning section</think>This is the rest",
52 + "reasoning": "This is a reasoning section",
53 + "content": "This is the rest",
54 + "is_reasoning_end": True,
55 +}
56 +
57 +WITHOUT_OPEN_THINK_STREAM = {
58 + "output": "This is a reasoning section</think>This is the rest",
59 + "reasoning": "This is a reasoning section",
60 + "content": "This is the rest",
61 + "is_reasoning_end": True,
62 +}
63 +
64 COMPLETE_REASONING = {
65 "output": "<think>This is a reasoning section</think>",
66 "reasoning": "This is a reasoning section",
@@ -61,8 +75,8 @@ MULTILINE_REASONING = {
75 }
76 ONLY_OPEN_TAG = {
77 "output": "<think>This is a reasoning section",
64 - "reasoning": None,
65 - "content": "<think>This is a reasoning section",
78 + "reasoning": "This is a reasoning section",
79 + "content": None,
80 "is_reasoning_end": False,
81 }
82
@@ -94,6 +108,16 @@ TEST_CASES = [
108 WITHOUT_THINK_STREAM,
109 id="without_think_stream",
110 ),
111 + pytest.param(
112 + False,
113 + WITHOUT_OPEN_THINK,
114 + id="without_open_think",
115 + ),
116 + pytest.param(
117 + True,
118 + WITHOUT_OPEN_THINK_STREAM,
119 + id="without_open_think_stream",
120 + ),
121 pytest.param(
122 False,
123 COMPLETE_REASONING,
tests/tool_parsers/test_glm47_moe_tool_parser.py
+31 -5
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
16 from vllm.tokenizers import get_tokenizer
17 from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
18
19 -MODEL = "zai-org/GLM-4.5"
19 +MODEL = "zai-org/GLM-4.7"
20
21
22 @pytest.fixture(scope="module")
@@ -136,9 +136,10 @@ class TestGlm47Streaming:
136 _reset(glm47_tool_parser)
137 chunks = ["<tool_call>", "get_current_date", "</tool_call>"]
138 current_text = ""
139 + deltas = []
140 for chunk in chunks:
141 current_text += chunk
141 - glm47_tool_parser.extract_tool_calls_streaming(
142 + delta = glm47_tool_parser.extract_tool_calls_streaming(
143 previous_text="",
144 current_text=current_text,
145 delta_text=chunk,
@@ -147,7 +148,23 @@ class TestGlm47Streaming:
148 delta_token_ids=[],
149 request=mock_request,
150 )
150 - assert len(glm47_tool_parser.prev_tool_call_arr) >= 1
151 + if delta:
152 + deltas.append(delta)
153 + tool_calls = [
154 + tool_call for delta in deltas for tool_call in (delta.tool_calls or [])
155 + ]
156 + names = [
157 + tool_call.function.name
158 + for tool_call in tool_calls
159 + if tool_call.function and tool_call.function.name
160 + ]
161 + arguments = [
162 + tool_call.function.arguments
163 + for tool_call in tool_calls
164 + if tool_call.function and tool_call.function.arguments
165 + ]
166 + assert names == ["get_current_date"]
167 + assert "".join(arguments) == "{}"
168
169 def test_with_args(self, glm47_tool_parser, mock_request):
170 _reset(glm47_tool_parser)
@@ -161,9 +178,10 @@ class TestGlm47Streaming:
178 "</tool_call>",
179 ]
180 current_text = ""
181 + deltas = []
182 for chunk in chunks:
183 current_text += chunk
166 - glm47_tool_parser.extract_tool_calls_streaming(
184 + delta = glm47_tool_parser.extract_tool_calls_streaming(
185 previous_text="",
186 current_text=current_text,
187 delta_text=chunk,
@@ -172,5 +190,13 @@ class TestGlm47Streaming:
190 delta_token_ids=[],
191 request=mock_request,
192 )
175 - args = json.loads(glm47_tool_parser.prev_tool_call_arr[0]["arguments"])
193 + if delta:
194 + deltas.append(delta)
195 + arguments = [
196 + tool_call.function.arguments
197 + for delta in deltas
198 + for tool_call in (delta.tool_calls or [])
199 + if tool_call.function and tool_call.function.arguments
200 + ]
201 + args = json.loads("".join(arguments))
202 assert args["city"] == "Beijing"
tests/tool_parsers/test_glm4_moe_tool_parser.py
+147 -1404
@@ -1,1067 +1,57 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Compatibility tests for GLM-4.5 using the shared GLM XML parser."""
4
5 import json
5 -from unittest.mock import Mock
6 -
7 -import pytest
8 -from openai.types.responses import FunctionTool
6 +from typing import Any, TypedDict
7
8 +from tests.parser.engine.replay_harness import MockTokenizer
9 from vllm.entrypoints.openai.chat_completion.protocol import (
10 ChatCompletionRequest,
11 ChatCompletionToolsParam,
12 FunctionDefinition,
13 )
15 -from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
16 -from vllm.tokenizers import get_tokenizer
17 -from vllm.tool_parsers.glm4_moe_tool_parser import (
18 - Glm4MoeModelToolParser,
19 -)
14 +from vllm.tool_parsers import ToolParserManager
15 +from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
16
21 -# Use a common model that is likely to be available
17 MODEL = "zai-org/GLM-4.5"
18
19 +_GLM_VOCAB = {
20 + "<think>": 50,
21 + "</think>": 51,
22 + "<tool_call>": 60,
23 + "</tool_call>": 61,
24 + "<arg_key>": 62,
25 + "</arg_key>": 63,
26 + "<arg_value>": 64,
27 + "</arg_value>": 65,
28 +}
29
25 -@pytest.fixture(scope="module")
26 -def glm4_moe_tokenizer():
27 - return get_tokenizer(tokenizer_name=MODEL)
28 -
29 -
30 -@pytest.fixture
31 -def sample_tools():
32 - return [
33 - ChatCompletionToolsParam(
34 - function=FunctionDefinition(
35 - name="get_weather",
36 - parameters={"city": {"type": "string"}},
37 - ),
38 - ),
39 - ]
40 -
41 -
42 -@pytest.fixture
43 -def glm4_moe_tool_parser(glm4_moe_tokenizer, sample_tools):
44 - return Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=sample_tools)
45 -
46 -
47 -@pytest.fixture
48 -def mock_request(sample_tools) -> ChatCompletionRequest:
49 - request = Mock(spec=ChatCompletionRequest)
50 - request.tools = sample_tools
51 - return request
52 -
53 -
54 -def assert_tool_calls(
55 - actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
56 -):
57 - assert len(actual_tool_calls) == len(expected_tool_calls)
58 -
59 - for actual_tool_call, expected_tool_call in zip(
60 - actual_tool_calls, expected_tool_calls
61 - ):
62 - assert isinstance(actual_tool_call.id, str)
63 - assert len(actual_tool_call.id) > 0
64 -
65 - assert actual_tool_call.type == "function"
66 - assert actual_tool_call.function.name == expected_tool_call.function.name
67 - # Compare arguments as JSON objects to handle formatting differences
68 - actual_args = json.loads(actual_tool_call.function.arguments)
69 - expected_args = json.loads(expected_tool_call.function.arguments)
70 - assert actual_args == expected_args
71 -
72 -
73 -def test_extract_tool_calls_no_tools(glm4_moe_tool_parser, mock_request):
74 - model_output = "This is a test"
75 - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
76 - model_output, request=mock_request
77 - ) # type: ignore[arg-type]
78 - assert not extracted_tool_calls.tools_called
79 - assert extracted_tool_calls.tool_calls == []
80 - assert extracted_tool_calls.content == model_output
81 -
82 -
83 -@pytest.mark.parametrize(
84 - ids=[
85 - "single_tool_call",
86 - "multiple_tool_calls",
87 - "tool_call_with_content_before",
88 - "tool_call_with_mixed_args",
89 - "tool_call_with_chinese_content",
90 - ],
91 - argnames=["model_output", "expected_tool_calls", "expected_content"],
92 - argvalues=[
93 - (
94 - """<tool_call>get_current_weather
95 - <arg_key>city</arg_key>
96 - <arg_value>Dallas</arg_value>
97 - <arg_key>state</arg_key>
98 - <arg_value>TX</arg_value>
99 - <arg_key>unit</arg_key>
100 - <arg_value>fahrenheit</arg_value>
101 - </tool_call>""",
102 - [
103 - ToolCall(
104 - function=FunctionCall(
105 - name="get_current_weather",
106 - arguments=json.dumps(
107 - {
108 - "city": "Dallas",
109 - "state": "TX",
110 - "unit": "fahrenheit",
111 - }
112 - ),
113 - )
114 - )
115 - ],
116 - None,
117 - ),
118 - (
119 - """<tool_call>get_current_weather
120 - <arg_key>city</arg_key>
121 - <arg_value>Dallas</arg_value>
122 - <arg_key>state</arg_key>
123 - <arg_value>TX</arg_value>
124 - <arg_key>unit</arg_key>
125 - <arg_value>fahrenheit</arg_value>
126 - </tool_call>
127 - <tool_call>get_current_weather
128 - <arg_key>city</arg_key>
129 - <arg_value>Orlando</arg_value>
130 - <arg_key>state</arg_key>
131 - <arg_value>FL</arg_value>
132 - <arg_key>unit</arg_key>
133 - <arg_value>fahrenheit</arg_value>
134 - </tool_call>""",
135 - [
136 - ToolCall(
137 - function=FunctionCall(
138 - name="get_current_weather",
139 - arguments=json.dumps(
140 - {
141 - "city": "Dallas",
142 - "state": "TX",
143 - "unit": "fahrenheit",
144 - }
145 - ),
146 - )
147 - ),
148 - ToolCall(
149 - function=FunctionCall(
150 - name="get_current_weather",
151 - arguments=json.dumps(
152 - {
153 - "city": "Orlando",
154 - "state": "FL",
155 - "unit": "fahrenheit",
156 - }
157 - ),
158 - )
159 - ),
160 - ],
161 - None,
162 - ),
163 - (
164 - """I'll help you check the weather. <tool_call>get_current_weather
165 - <arg_key>city</arg_key>
166 - <arg_value>Seattle</arg_value>
167 - <arg_key>state</arg_key>
168 - <arg_value>WA</arg_value>
169 - <arg_key>unit</arg_key>
170 - <arg_value>celsius</arg_value>
171 - </tool_call>""",
172 - [
173 - ToolCall(
174 - function=FunctionCall(
175 - name="get_current_weather",
176 - arguments=json.dumps(
177 - {
178 - "city": "Seattle",
179 - "state": "WA",
180 - "unit": "celsius",
181 - }
182 - ),
183 - )
184 - )
185 - ],
186 - "I'll help you check the weather. ",
187 - ),
188 - (
189 - """<tool_call>get_current_weather
190 - <arg_key>city</arg_key>
191 - <arg_value>New York</arg_value>
192 - <arg_key>state</arg_key>
193 - <arg_value>NY</arg_value>
194 - <arg_key>unit</arg_key>
195 - <arg_value>celsius</arg_value>
196 - </tool_call>""",
197 - [
198 - ToolCall(
199 - function=FunctionCall(
200 - name="get_current_weather",
201 - arguments=json.dumps(
202 - {
203 - "city": "New York",
204 - "state": "NY",
205 - "unit": "celsius",
206 - }
207 - ),
208 - )
209 - )
210 - ],
211 - None,
212 - ),
213 - (
214 - """I will help you get the weather.<tool_call>get_weather
215 - <arg_key>city</arg_key>
216 - <arg_value>Beijing</arg_value>
217 - <arg_key>date</arg_key>
218 - <arg_value>2025-08-01</arg_value>
219 - </tool_call>""",
220 - [
221 - ToolCall(
222 - function=FunctionCall(
223 - name="get_weather",
224 - arguments=json.dumps(
225 - {
226 - "city": "Beijing",
227 - "date": "2025-08-01",
228 - }
229 - ),
230 - )
231 - )
232 - ],
233 - "I will help you get the weather.",
234 - ),
235 - ],
236 -)
237 -def test_extract_tool_calls(
238 - glm4_moe_tool_parser,
239 - mock_request,
240 - model_output,
241 - expected_tool_calls,
242 - expected_content,
243 -):
244 - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
245 - model_output, request=mock_request
246 - ) # type: ignore[arg-type]
247 - assert extracted_tool_calls.tools_called
248 - assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
249 -
250 - assert extracted_tool_calls.content == expected_content
251 -
252 -
253 -def test_extract_tool_calls_with_thinking_tags(glm4_moe_tool_parser, mock_request):
254 - """Test tool extraction when thinking tags are present."""
255 - model_output = """<think>I want to get the weather.</think>
256 -
257 -I will help you get the weather.
258 -<tool_call>get_weather
259 -<arg_key>city</arg_key>
260 -<arg_value>Beijing</arg_value>
261 -<arg_key>date</arg_key>
262 -<arg_value>2025-08-01</arg_value>
263 -</tool_call>"""
264 -
265 - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
266 - model_output, request=mock_request
267 - ) # type: ignore[arg-type]
268 -
269 - assert extracted_tool_calls.tools_called
270 - assert len(extracted_tool_calls.tool_calls) == 1
271 - assert extracted_tool_calls.tool_calls[0].function.name == "get_weather"
272 -
273 - expected_content = """<think>I want to get the weather.</think>
274 -
275 -I will help you get the weather.
276 -"""
277 - assert extracted_tool_calls.content == expected_content
278 -
279 -
280 -def test_extract_tool_calls_malformed_xml(glm4_moe_tool_parser, mock_request):
281 - """Test that malformed XML is handled gracefully."""
282 - model_output = """<tool_call>get_weather
283 -<arg_key>city</arg_key>
284 -<arg_value>Seattle</arg_value>
285 -<arg_key>incomplete_arg
286 -<arg_value>value</arg_value>
287 -</tool_call>"""
288 -
289 - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
290 - model_output, request=mock_request
291 - ) # type: ignore[arg-type]
292 -
293 - # Should handle malformed XML gracefully
294 - # The parser should either extract what it can or return no tool calls
295 - # depending on how robust we want the parsing to be
296 - assert isinstance(extracted_tool_calls.tools_called, bool)
297 - assert isinstance(extracted_tool_calls.tool_calls, list)
298 -
299 -
300 -def test_extract_tool_calls_empty_arguments(glm4_moe_tool_parser, mock_request):
301 - """Test tool calls with no arguments."""
302 - model_output = """<tool_call>get_current_time
303 -</tool_call>"""
304 -
305 - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
306 - model_output, request=mock_request
307 - ) # type: ignore[arg-type]
308 -
309 - assert extracted_tool_calls.tools_called
310 - assert len(extracted_tool_calls.tool_calls) == 1
311 - assert extracted_tool_calls.tool_calls[0].function.name == "get_current_time"
312 - # Empty arguments should result in empty JSON object
313 - assert extracted_tool_calls.tool_calls[0].function.arguments == "{}"
314 -
315 -
316 -def test_extract_tool_calls_mixed_content(glm4_moe_tool_parser, mock_request):
317 - """Test extraction with mixed content and multiple tool calls."""
318 - model_output = """I will help you get the weather info.
319 -
320 -<tool_call>get_weather
321 -<arg_key>city</arg_key>
322 -<arg_value>Beijing</arg_value>
323 -<arg_key>date</arg_key>
324 -<arg_value>2025-08-01</arg_value>
325 -</tool_call>
326 -
327 -meaningwhile, I will also check the weather in Shanghai.
328 -
329 -<tool_call>get_weather
330 -<arg_key>city</arg_key>
331 -<arg_value>Shanghai</arg_value>
332 -<arg_key>date</arg_key>
333 -<arg_value>2025-08-01</arg_value>
334 -</tool_call>"""
335 -
336 - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
337 - model_output, request=mock_request
338 - ) # type: ignore[arg-type]
339 -
340 - assert extracted_tool_calls.tools_called
341 - assert len(extracted_tool_calls.tool_calls) == 2
342 -
343 - # Check first tool call
344 - assert extracted_tool_calls.tool_calls[0].function.name == "get_weather"
345 - args1 = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
346 - assert args1["city"] == "Beijing"
347 - assert args1["date"] == "2025-08-01"
348 -
349 - # Check second tool call
350 - assert extracted_tool_calls.tool_calls[1].function.name == "get_weather"
351 - args2 = json.loads(extracted_tool_calls.tool_calls[1].function.arguments)
352 - assert args2["city"] == "Shanghai"
353 - assert args2["date"] == "2025-08-01"
354 -
355 - # Content should be everything before the first tool call
356 - assert extracted_tool_calls.content == "I will help you get the weather info.\n\n"
357 -
358 -
359 -def test_streaming_basic_functionality(glm4_moe_tool_parser, mock_request):
360 - """Test basic streaming functionality."""
361 - _reset_streaming_state(glm4_moe_tool_parser)
362 -
363 - current_text = """<tool_call>get_weather
364 -<arg_key>city</arg_key>
365 -<arg_value>Beijing</arg_value>
366 -</tool_call>"""
367 -
368 - result = glm4_moe_tool_parser.extract_tool_calls_streaming(
369 - previous_text="",
370 - current_text=current_text,
371 - delta_text=current_text,
372 - previous_token_ids=[],
373 - current_token_ids=[],
374 - delta_token_ids=[],
375 - request=mock_request,
376 - )
377 -
378 - # Should return tool call with name and arguments in one shot
379 - assert result is not None
380 - assert result.tool_calls is not None
381 - assert len(result.tool_calls) >= 1
382 -
383 -
384 -def test_streaming_no_tool_calls(glm4_moe_tool_parser, mock_request):
385 - """Test streaming when there are no tool calls."""
386 - _reset_streaming_state(glm4_moe_tool_parser)
387 -
388 - current_text = "This is just regular text without any tool calls."
389 -
390 - result = glm4_moe_tool_parser.extract_tool_calls_streaming(
391 - previous_text="",
392 - current_text=current_text,
393 - delta_text=current_text,
394 - previous_token_ids=[],
395 - current_token_ids=[],
396 - delta_token_ids=[],
397 - request=mock_request,
398 - )
399 -
400 - # Should return content
401 - assert result is not None
402 - assert result.content == current_text
403 -
404 -
405 -def test_streaming_with_content_before_tool_calls(glm4_moe_tool_parser, mock_request):
406 - """Test streaming when there's content before tool calls."""
407 - _reset_streaming_state(glm4_moe_tool_parser)
408 -
409 - current_text = "I will help you get the weather.<tool_call>"
410 -
411 - result = glm4_moe_tool_parser.extract_tool_calls_streaming(
412 - previous_text="",
413 - current_text=current_text,
414 - delta_text=current_text,
415 - previous_token_ids=[],
416 - current_token_ids=[],
417 - delta_token_ids=[],
418 - request=mock_request,
419 - )
420 -
421 - # Should return content before the <tool_call> tag
422 - assert result is not None
423 - assert result.content == "I will help you get the weather."
424 -
425 -
426 -def test_extract_tool_calls_special_characters(glm4_moe_tool_parser, mock_request):
427 - """Test tool calls with special characters and unicode."""
428 - model_output = """<tool_call>send_message
429 -<arg_key>recipient</arg_key>
430 -<arg_value>Amy</arg_value>
431 -<arg_key>message</arg_key>
432 -<arg_value>It is a nice day</arg_value>
433 -<arg_key>priority</arg_key>
434 -<arg_value>high</arg_value>
435 -</tool_call>"""
436 -
437 - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
438 - model_output, request=mock_request
439 - ) # type: ignore[arg-type]
440 -
441 - assert extracted_tool_calls.tools_called
442 - assert len(extracted_tool_calls.tool_calls) == 1
443 - assert extracted_tool_calls.tool_calls[0].function.name == "send_message"
444 -
445 - args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
446 - assert args["recipient"] == "Amy"
447 - assert args["message"] == "It is a nice day"
448 - assert args["priority"] == "high"
449 -
450 -
451 -def test_extract_tool_calls_incomplete_tool_call(glm4_moe_tool_parser, mock_request):
452 - """Test incomplete tool calls (missing closing tag)."""
453 - model_output = """<tool_call>get_weather
454 -<arg_key>city</arg_key>
455 -<arg_value>Beijing</arg_value>
456 -<arg_key>date</arg_key>
457 -<arg_value>2025-08-01</arg_value>"""
458 -
459 - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
460 - model_output, request=mock_request
461 - ) # type: ignore[arg-type]
462 -
463 - # Incomplete tool calls should not be extracted
464 - assert not extracted_tool_calls.tools_called
465 - assert extracted_tool_calls.tool_calls == []
466 - assert extracted_tool_calls.content == model_output
467 -
468 -
469 -def _reset_streaming_state(parser):
470 - """Helper to reset parser streaming state."""
471 - parser.current_tool_name_sent = False
472 - parser.prev_tool_call_arr = []
473 - parser.current_tool_id = -1
474 - parser.streamed_args_for_tool = []
475 - parser._tool_call_ids = []
476 - parser._sent_content_idx = 0
477 -
478 -
479 -def test_streaming_incremental_string_value(glm4_moe_tool_parser, mock_request):
480 - """Test incremental streaming of string argument values."""
481 - _reset_streaming_state(glm4_moe_tool_parser)
482 -
483 - # Simulate streaming a tool call chunk by chunk
484 - chunks = [
485 - "<tool_call>",
486 - "get_weather\n",
487 - "<arg_key>city</arg_key>",
488 - "<arg_value>",
489 - "Bei",
490 - "jing",
491 - "</arg_value>",
492 - "</tool_call>",
493 - ]
494 -
495 - collected_fragments = []
496 - current_text = ""
497 - for chunk in chunks:
498 - current_text += chunk
499 - result = glm4_moe_tool_parser.extract_tool_calls_streaming(
500 - previous_text="",
501 - current_text=current_text,
502 - delta_text=chunk,
503 - previous_token_ids=[],
504 - current_token_ids=[],
505 - delta_token_ids=[],
506 - request=mock_request,
507 - )
508 - if result is not None and result.tool_calls:
509 - for tc in result.tool_calls:
510 - func = tc.function
511 - if isinstance(func, dict):
512 - if func.get("arguments"):
513 - collected_fragments.append(func["arguments"])
514 - if func.get("name"):
515 - collected_fragments.append(f"name:{func['name']}")
516 - else:
517 - if func.arguments:
518 - collected_fragments.append(func.arguments)
519 - if func.name:
520 - collected_fragments.append(f"name:{func.name}")
521 -
522 - # Verify we got incremental streaming of the argument value
523 - assert len(collected_fragments) > 0
524 - # The fragments should include the tool name and argument pieces
525 - combined = "".join(collected_fragments)
526 - assert "get_weather" in combined or "name:get_weather" in combined
527 -
528 -
529 -def test_streaming_empty_tool_call(glm4_moe_tool_parser, mock_request):
530 - """Test that empty tool calls don't cause infinite loops."""
531 - _reset_streaming_state(glm4_moe_tool_parser)
532 -
533 - current_text = "<tool_call></tool_call>"
534 - result = glm4_moe_tool_parser.extract_tool_calls_streaming(
535 - previous_text="",
536 - current_text=current_text,
537 - delta_text=current_text,
538 - previous_token_ids=[],
539 - current_token_ids=[],
540 - delta_token_ids=[],
541 - request=mock_request,
542 - )
543 -
544 - # Should not hang and should return something (None or content)
545 - # The key is that this completes without hanging
546 - assert result is None or hasattr(result, "content") or hasattr(result, "tool_calls")
547 -
548 -
549 -def test_streaming_prev_tool_call_arr_updates(glm4_moe_tool_parser, mock_request):
550 - """Test that prev_tool_call_arr is populated incrementally."""
551 - _reset_streaming_state(glm4_moe_tool_parser)
552 -
553 - chunks = [
554 - "<tool_call>get_weather\n",
555 - "<arg_key>city</arg_key>",
556 - "<arg_value>Beijing</arg_value>",
557 - "</tool_call>",
558 - ]
559 -
560 - current_text = ""
561 - for chunk in chunks:
562 - current_text += chunk
563 - glm4_moe_tool_parser.extract_tool_calls_streaming(
564 - previous_text="",
565 - current_text=current_text,
566 - delta_text=chunk,
567 - previous_token_ids=[],
568 - current_token_ids=[],
569 - delta_token_ids=[],
570 - request=mock_request,
571 - )
572 -
573 - # After the tool call completes, prev_tool_call_arr should be populated
574 - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1
575 - tool_entry = glm4_moe_tool_parser.prev_tool_call_arr[0]
576 - assert tool_entry.get("name") == "get_weather"
577 -
578 - # arguments is a JSON string in the re-parse approach
579 - args_str = tool_entry.get("arguments")
580 - assert isinstance(args_str, str), f"Expected str, got {type(args_str)}"
581 - parsed = json.loads(args_str)
582 - assert parsed["city"] == "Beijing"
583 -
584 - # streamed_args_for_tool should match prev_tool_call_arr arguments
585 - streamed = glm4_moe_tool_parser.streamed_args_for_tool[0]
586 - assert streamed == args_str
587 -
588 -
589 -def test_streaming_multiple_tool_calls_sequential(glm4_moe_tool_parser, mock_request):
590 - """Test streaming multiple sequential tool calls."""
591 - _reset_streaming_state(glm4_moe_tool_parser)
592 -
593 - chunks = [
594 - "<tool_call>get_weather\n",
595 - "<arg_key>city</arg_key>",
596 - "<arg_value>Beijing</arg_value>",
597 - "</tool_call>",
598 - "<tool_call>get_weather\n",
599 - "<arg_key>city</arg_key>",
600 - "<arg_value>Shanghai</arg_value>",
601 - "</tool_call>",
602 - ]
603 -
604 - current_text = ""
605 - for chunk in chunks:
606 - current_text += chunk
607 - glm4_moe_tool_parser.extract_tool_calls_streaming(
608 - previous_text="",
609 - current_text=current_text,
610 - delta_text=chunk,
611 - previous_token_ids=[],
612 - current_token_ids=[],
613 - delta_token_ids=[],
614 - request=mock_request,
615 - )
616 -
617 - # Should have two tool calls in prev_tool_call_arr
618 - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 2
619 - args0 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[0]["arguments"])
620 - args1 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[1]["arguments"])
621 - assert args0["city"] == "Beijing"
622 - assert args1["city"] == "Shanghai"
623 -
624 -
625 -def test_streaming_json_escape_in_string(glm4_moe_tool_parser, mock_request):
626 - """Test that special characters in string values are properly escaped."""
627 - _reset_streaming_state(glm4_moe_tool_parser)
628 -
629 - chunks = [
630 - "<tool_call>send_message\n",
631 - "<arg_key>message</arg_key>",
632 - '<arg_value>Hello "world"\nNew line</arg_value>',
633 - "</tool_call>",
634 - ]
635 -
636 - current_text = ""
637 - for chunk in chunks:
638 - current_text += chunk
639 - glm4_moe_tool_parser.extract_tool_calls_streaming(
640 - previous_text="",
641 - current_text=current_text,
642 - delta_text=chunk,
643 - previous_token_ids=[],
644 - current_token_ids=[],
645 - delta_token_ids=[],
646 - request=mock_request,
647 - )
648 -
649 - # The streamed_args_for_tool should contain valid JSON
650 - assert len(glm4_moe_tool_parser.streamed_args_for_tool) == 1
651 - args_json = glm4_moe_tool_parser.streamed_args_for_tool[0]
652 - parsed = json.loads(args_json)
653 - assert "message" in parsed
654 - assert '"' in parsed["message"] or "world" in parsed["message"]
655 -
656 -
657 -def test_streaming_long_content_incremental(glm4_moe_tokenizer):
658 - """Test incremental streaming of long content (Issue #32829).
659 -
660 - This is the core fix: for long string values like code (4000+ chars),
661 - the parser should stream incrementally rather than buffering until
662 - complete. This test verifies we get many fragments, not just 1-3.
663 - """
664 -
665 - # Bubble sort example from Issue #32829 - realistic long content
666 - bubble_sort_code = '''#!/usr/bin/env python3
667 -# -*- coding: utf-8 -*-
668 -"""
669 -Bubble Sort Implementation
670 -"""
671 -
672 -def bubble_sort(arr):
673 - n = len(arr)
674 - for i in range(n):
675 - swapped = False
676 - for j in range(0, n - i - 1):
677 - if arr[j] > arr[j + 1]:
678 - arr[j], arr[j + 1] = arr[j + 1], arr[j]
679 - swapped = True
680 - if not swapped:
681 - break
682 - return arr
683 -
684 -if __name__ == "__main__":
685 - test_arr = [64, 34, 25, 12, 22, 11, 90]
686 - print(f"Original: {test_arr}")
687 - sorted_arr = bubble_sort(test_arr.copy())
688 - print(f"Sorted: {sorted_arr}")'''
689 -
690 - # Create tools with schema to enable string type detection
691 - # This is required for incremental streaming of string values
692 - tools = [
693 - ChatCompletionToolsParam(
694 - function=FunctionDefinition(
695 - name="write_to_file",
696 - parameters={
697 - "type": "object",
698 - "properties": {
699 - "file_path": {"type": "string"},
700 - "content": {"type": "string"},
701 - },
702 - },
703 - ),
704 - ),
705 - ]
706 - glm4_moe_tool_parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
707 - request = ChatCompletionRequest(
708 - model=MODEL,
709 - messages=[],
710 - tools=tools,
711 - )
712 -
713 - # Simulate token-based streaming (special tags as single tokens)
714 - chunks = [
715 - "<tool_call>",
716 - "write_to_file\n",
717 - "<arg_key>file_path</arg_key>",
718 - "<arg_value>/tmp/bubble_sort.py</arg_value>",
719 - "<arg_key>content</arg_key>",
720 - "<arg_value>",
721 - ]
722 - # Add content line by line (realistic token streaming)
723 - for line in bubble_sort_code.split("\n"):
724 - chunks.append(line + "\n")
725 - chunks.append("</arg_value>")
726 - chunks.append("</tool_call>")
727 -
728 - # Count argument fragments
729 - fragment_count = 0
730 - current_text = ""
731 - for chunk in chunks:
732 - current_text += chunk
733 - result = glm4_moe_tool_parser.extract_tool_calls_streaming(
734 - previous_text="",
735 - current_text=current_text,
736 - delta_text=chunk,
737 - previous_token_ids=[],
738 - current_token_ids=[],
739 - delta_token_ids=[],
740 - request=request,
741 - )
742 - if result is not None and result.tool_calls:
743 - for tc in result.tool_calls:
744 - func = tc.function
745 - if isinstance(func, dict):
746 - args = func.get("arguments")
747 - else:
748 - args = getattr(func, "arguments", None)
749 - if args:
750 - fragment_count += 1
751 -
752 - # For true incremental streaming, we expect many fragments (10+)
753 - # Old buffered implementation would give only 1-3 fragments
754 - assert fragment_count >= 10, (
755 - f"Expected >=10 fragments for incremental streaming, got {fragment_count}"
756 - )
757 -
758 - # Verify final result is valid JSON
759 - assert len(glm4_moe_tool_parser.streamed_args_for_tool) == 1
760 - args_json = glm4_moe_tool_parser.streamed_args_for_tool[0]
761 - parsed = json.loads(args_json)
762 - assert parsed["file_path"] == "/tmp/bubble_sort.py"
763 - assert "def bubble_sort" in parsed["content"]
764 -
765 -
766 -def test_extract_tool_calls_numeric_deserialization(glm4_moe_tool_parser, mock_request):
767 - """Test that numeric arguments are deserialized as numbers, not strings."""
768 - model_output = """<tool_call>calculate
769 -<arg_key>operation</arg_key>
770 -<arg_value>add</arg_value>
771 -<arg_key>a</arg_key>
772 -<arg_value>42</arg_value>
773 -<arg_key>b</arg_key>
774 -<arg_value>3.14</arg_value>
775 -<arg_key>enabled</arg_key>
776 -<arg_value>true</arg_value>
777 -</tool_call>"""
778 -
779 - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
780 - model_output, request=mock_request
781 - ) # type: ignore[arg-type]
782 -
783 - assert extracted_tool_calls.tools_called
784 - assert len(extracted_tool_calls.tool_calls) == 1
785 -
786 - args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
30
788 - # String should remain string
789 - assert args["operation"] == "add"
790 - assert isinstance(args["operation"], str)
31 +class _CollectedToolDelta(TypedDict):
32 + name: str | None
33 + args_fragments: list[str]
34
792 - # Integer should be deserialized as int
793 - assert args["a"] == 42
794 - assert isinstance(args["a"], int)
35
796 - # Float should be deserialized as float
797 - assert args["b"] == 3.14
798 - assert isinstance(args["b"], float)
36 +def _mock_tokenizer() -> MockTokenizer:
37 + return MockTokenizer(vocab=_GLM_VOCAB, tokens=[])
38
800 - # Boolean should be deserialized as bool
801 - assert args["enabled"] is True
802 - assert isinstance(args["enabled"], bool)
39
804 -
805 -def test_whitespace_preserved_in_arg_values(glm4_moe_tokenizer):
806 - """Test that string arguments preserve leading and trailing whitespace."""
807 - tools = [
40 +def _tools() -> list[ChatCompletionToolsParam]:
41 + return [
42 ChatCompletionToolsParam(
43 function=FunctionDefinition(
810 - name="apply_diff",
44 + name="get_current_weather",
45 parameters={
46 "type": "object",
47 "properties": {
814 - "s": {"type": "string"},
48 + "city": {"type": "string"},
49 + "state": {"type": "string"},
50 + "unit": {"type": "string"},
51 },
816 - "required": ["s"],
52 },
53 ),
54 ),
820 - ]
821 - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
822 - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
823 -
824 - model_output = """<tool_call>apply_diff
825 -<arg_key>s</arg_key>
826 -<arg_value> indented code </arg_value>
827 -</tool_call>"""
828 -
829 - extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)
830 - args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
831 -
832 - assert args["s"] == " indented code "
833 -
834 -
835 -def test_zero_argument_tool_call(glm4_moe_tool_parser, mock_request):
836 - """Regression: zero-argument tool call crash (PR #32321)."""
837 - model_output = """<tool_call>get_time
838 -</tool_call>"""
839 -
840 - extracted = glm4_moe_tool_parser.extract_tool_calls(
841 - model_output, request=mock_request
842 - ) # type: ignore[arg-type]
843 -
844 - assert extracted.tools_called
845 - assert len(extracted.tool_calls) == 1
846 - assert extracted.tool_calls[0].function.name == "get_time"
847 - args = json.loads(extracted.tool_calls[0].function.arguments)
848 - assert args == {}
849 -
850 -
851 -def test_malformed_tool_call_no_regex_match(glm4_moe_tool_parser, mock_request):
852 - """Regression: malformed tool_call with no regex match (PR #32321)."""
853 - model_output = "<tool_call> </tool_call>"
854 -
855 - extracted = glm4_moe_tool_parser.extract_tool_calls(
856 - model_output, request=mock_request
857 - ) # type: ignore[arg-type]
858 -
859 - assert extracted.tools_called is False
860 - assert extracted.tool_calls == []
861 -
862 -
863 -def test_delimiter_preserved_transformers_5x(glm4_moe_tool_parser):
864 - """Regression: adjust_request sets skip_special_tokens=False (PR #31622)."""
865 - # Tools enabled
866 - request_with_tools = ChatCompletionRequest(
867 - model=MODEL,
868 - messages=[],
869 - tools=[
870 - {
871 - "type": "function",
872 - "function": {
873 - "name": "get_weather",
874 - "parameters": {
875 - "type": "object",
876 - "properties": {"city": {"type": "string"}},
877 - },
878 - },
879 - }
880 - ],
881 - ) # type: ignore
882 - adjusted = glm4_moe_tool_parser.adjust_request(request_with_tools)
883 - assert adjusted.skip_special_tokens is False
884 -
885 - # tool_choice="none"
886 - request_no_choice = ChatCompletionRequest(
887 - model=MODEL,
888 - messages=[],
889 - tools=[
890 - {
891 - "type": "function",
892 - "function": {
893 - "name": "get_weather",
894 - "parameters": {
895 - "type": "object",
896 - "properties": {"city": {"type": "string"}},
897 - },
898 - },
899 - }
900 - ],
901 - tool_choice="none",
902 - ) # type: ignore
903 - adjusted_none = glm4_moe_tool_parser.adjust_request(request_no_choice)
904 - assert adjusted_none.skip_special_tokens is True
905 -
906 - # No tools at all
907 - request_no_tools = ChatCompletionRequest(
908 - model=MODEL,
909 - messages=[],
910 - ) # type: ignore
911 - adjusted_empty = glm4_moe_tool_parser.adjust_request(request_no_tools)
912 - assert adjusted_empty.skip_special_tokens is True
913 -
914 -
915 -def test_unicode_characters_preserved(glm4_moe_tool_parser, mock_request):
916 - """Regression: Unicode chars must not be escaped to \\uXXXX (PR #30920)."""
917 - model_output = """<tool_call>send_message
918 -<arg_key>greeting</arg_key>
919 -<arg_value>你好世界</arg_value>
920 -<arg_key>emoji</arg_key>
921 -<arg_value>🎉</arg_value>
922 -</tool_call>"""
923 -
924 - extracted = glm4_moe_tool_parser.extract_tool_calls(
925 - model_output, request=mock_request
926 - ) # type: ignore[arg-type]
927 -
928 - assert extracted.tools_called
929 - assert len(extracted.tool_calls) == 1
930 -
931 - raw_args = extracted.tool_calls[0].function.arguments
932 - assert "你好世界" in raw_args
933 - assert "🎉" in raw_args
934 - assert "\\u4f60" not in raw_args
935 - parsed_args = json.loads(raw_args)
936 - assert parsed_args["greeting"] == "你好世界"
937 - assert parsed_args["emoji"] == "🎉"
938 -
939 -
940 -def test_streaming_multi_token_chunks(glm4_moe_tool_parser, mock_request):
941 - """Test that multi-token chunks (stream_interval > 1) are handled correctly.
942 -
943 - With stream_interval > 1 or MTP, multiple XML tags arrive in one delta.
944 - The old buffer-based parser could only return one delta per call, losing
945 - data on the final output. The re-parse approach handles this correctly.
946 - """
947 - _reset_streaming_state(glm4_moe_tool_parser)
948 -
949 - # Simulate stream_interval=3: chunks contain multiple XML tags
950 - chunks = [
951 - "<tool_call>get_weather\n<arg_key>city</arg_key><arg_value>Bei",
952 - "jing</arg_value>",
953 - "</tool_call>",
954 - ]
955 -
956 - current_text = ""
957 - for chunk in chunks:
958 - current_text += chunk
959 - glm4_moe_tool_parser.extract_tool_calls_streaming(
960 - previous_text="",
961 - current_text=current_text,
962 - delta_text=chunk,
963 - previous_token_ids=[],
964 - current_token_ids=[],
965 - delta_token_ids=[],
966 - request=mock_request,
967 - )
968 -
969 - # All data should be captured despite multi-token chunks
970 - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1
971 - args = json.loads(glm4_moe_tool_parser.streamed_args_for_tool[0])
972 - assert args["city"] == "Beijing"
973 -
974 -
975 -def test_streaming_entire_tool_call_at_once(glm4_moe_tool_parser, mock_request):
976 - """Test that a complete tool call arriving in one delta works.
977 -
978 - This simulates the extreme MTP case where all tokens arrive at once.
979 - """
980 - _reset_streaming_state(glm4_moe_tool_parser)
981 -
982 - full_text = (
983 - "<tool_call>get_weather\n"
984 - "<arg_key>city</arg_key>"
985 - "<arg_value>Beijing</arg_value>"
986 - "</tool_call>"
987 - )
988 -
989 - result = glm4_moe_tool_parser.extract_tool_calls_streaming(
990 - previous_text="",
991 - current_text=full_text,
992 - delta_text=full_text,
993 - previous_token_ids=[],
994 - current_token_ids=[],
995 - delta_token_ids=[],
996 - request=mock_request,
997 - )
998 -
999 - # Should emit tool call with complete arguments in one shot
1000 - assert result is not None
1001 - assert result.tool_calls is not None
1002 -
1003 - # Verify final state
1004 - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1
1005 - args = json.loads(glm4_moe_tool_parser.streamed_args_for_tool[0])
1006 - assert args["city"] == "Beijing"
1007 -
1008 -
1009 -def test_streaming_content_between_tool_calls_multi_token(
1010 - glm4_moe_tool_parser, mock_request
1011 -):
1012 - """Test content between tool calls with multi-token chunks."""
1013 - _reset_streaming_state(glm4_moe_tool_parser)
1014 -
1015 - # Deliver everything at once — worst case for the old buffer parser
1016 - full_text = (
1017 - "I will check.\n"
1018 - "<tool_call>get_weather\n"
1019 - "<arg_key>city</arg_key>"
1020 - "<arg_value>Beijing</arg_value>"
1021 - "</tool_call>"
1022 - "\nAlso Shanghai.\n"
1023 - "<tool_call>get_weather\n"
1024 - "<arg_key>city</arg_key>"
1025 - "<arg_value>Shanghai</arg_value>"
1026 - "</tool_call>"
1027 - )
1028 -
1029 - # First call with partial text (content only)
1030 - partial = "I will check.\n"
1031 - result1 = glm4_moe_tool_parser.extract_tool_calls_streaming(
1032 - previous_text="",
1033 - current_text=partial,
1034 - delta_text=partial,
1035 - previous_token_ids=[],
1036 - current_token_ids=[],
1037 - delta_token_ids=[],
1038 - request=mock_request,
1039 - )
1040 - assert result1 is not None
1041 - assert result1.content == "I will check.\n"
1042 -
1043 - # Second call with everything
1044 - glm4_moe_tool_parser.extract_tool_calls_streaming(
1045 - previous_text="",
1046 - current_text=full_text,
1047 - delta_text=full_text[len(partial) :],
1048 - previous_token_ids=[],
1049 - current_token_ids=[],
1050 - delta_token_ids=[],
1051 - request=mock_request,
1052 - )
1053 -
1054 - # Should have both tool calls
1055 - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 2
1056 - args0 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[0]["arguments"])
1057 - args1 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[1]["arguments"])
1058 - assert args0["city"] == "Beijing"
1059 - assert args1["city"] == "Shanghai"
1060 -
1061 -
1062 -def test_streaming_multi_token_with_multiple_args(glm4_moe_tokenizer):
1063 - """Test multi-token streaming with multiple arguments of mixed types."""
1064 - tools = [
55 ChatCompletionToolsParam(
56 function=FunctionDefinition(
57 name="calculate",
@@ -1071,415 +61,168 @@ def test_streaming_multi_token_with_multiple_args(glm4_moe_tokenizer):
61 "operation": {"type": "string"},
62 "a": {"type": "number"},
63 "b": {"type": "number"},
64 + "enabled": {"type": "boolean"},
65 },
66 },
67 ),
68 ),
1078 - ]
1079 - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1080 - request = ChatCompletionRequest(
1081 - model=MODEL,
1082 - messages=[],
1083 - tools=tools,
1084 - )
1085 -
1086 - # All arguments arrive in two big chunks (simulates stream_interval=5)
1087 - chunks = [
1088 - "<tool_call>calculate\n<arg_key>operation</arg_key><arg_value>add</arg_value><arg_key>a</arg_key>",
1089 - "<arg_value>42</arg_value><arg_key>b</arg_key><arg_value>3.14</arg_value></tool_call>",
1090 - ]
1091 -
1092 - current_text = ""
1093 - for chunk in chunks:
1094 - current_text += chunk
1095 - parser.extract_tool_calls_streaming(
1096 - previous_text="",
1097 - current_text=current_text,
1098 - delta_text=chunk,
1099 - previous_token_ids=[],
1100 - current_token_ids=[],
1101 - delta_token_ids=[],
1102 - request=request,
1103 - )
1104 -
1105 - args = json.loads(parser.streamed_args_for_tool[0])
1106 - assert args["operation"] == "add"
1107 - assert args["a"] == 42
1108 - assert args["b"] == 3.14
1109 -
1110 -
1111 -def _simulate_streaming(tokenizer, parser, request, text, stream_interval=1):
1112 - """Simulate streaming with a given stream_interval.
1113 -
1114 - Tokens are batched into chunks of ``stream_interval`` tokens,
1115 - mimicking how the output processor delivers them.
1116 - Returns a list of non-None DeltaMessages.
1117 - """
1118 - tokens = tokenizer.encode(text)
1119 - previous_text = ""
1120 - deltas = []
1121 - for i in range(0, len(tokens), stream_interval):
1122 - chunk_ids = tokens[i : i + stream_interval]
1123 - delta_text = tokenizer.decode(chunk_ids)
1124 - current_text = previous_text + delta_text
1125 - delta = parser.extract_tool_calls_streaming(
1126 - previous_text=previous_text,
1127 - current_text=current_text,
1128 - delta_text=delta_text,
1129 - previous_token_ids=[],
1130 - current_token_ids=[],
1131 - delta_token_ids=chunk_ids,
1132 - request=request,
1133 - )
1134 - previous_text = current_text
1135 - if delta is not None:
1136 - deltas.append(delta)
1137 - return deltas
1138 -
1139 -
1140 -def _collect_from_deltas(deltas):
1141 - """Reconstruct tool call names/args and content from a delta stream."""
1142 - tools: dict[int, dict] = {}
1143 - content_parts: list[str] = []
1144 - for d in deltas:
1145 - if d.content:
1146 - content_parts.append(d.content)
1147 - if d.tool_calls:
1148 - for tc in d.tool_calls:
1149 - func = tc.function
1150 - if isinstance(func, dict):
1151 - name = func.get("name")
1152 - args = func.get("arguments")
1153 - else:
1154 - name = getattr(func, "name", None)
1155 - args = getattr(func, "arguments", None)
1156 - idx = tc.index
1157 - if idx not in tools:
1158 - tools[idx] = {"name": None, "args_fragments": []}
1159 - if name:
1160 - tools[idx]["name"] = name
1161 - if args:
1162 - tools[idx]["args_fragments"].append(args)
1163 - return content_parts, tools
1164 -
1165 -
1166 -@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
1167 -def test_stream_interval_single_tool_call(glm4_moe_tokenizer, stream_interval):
1168 - """Tool call streaming produces correct name + args at any interval."""
1169 - tools = [
69 ChatCompletionToolsParam(
1171 - function=FunctionDefinition(
1172 - name="get_weather",
1173 - parameters={
1174 - "type": "object",
1175 - "properties": {"city": {"type": "string"}},
1176 - },
1177 - ),
1178 - ),
1179 - ]
1180 - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1181 - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1182 -
1183 - text = (
1184 - "<tool_call>get_weather\n"
1185 - "<arg_key>city</arg_key>"
1186 - "<arg_value>Beijing</arg_value>"
1187 - "</tool_call>"
1188 - )
1189 -
1190 - deltas = _simulate_streaming(
1191 - glm4_moe_tokenizer, parser, request, text, stream_interval
1192 - )
1193 - _, tools_found = _collect_from_deltas(deltas)
1194 -
1195 - assert 0 in tools_found
1196 - assert tools_found[0]["name"] == "get_weather"
1197 - args_json = "".join(tools_found[0]["args_fragments"])
1198 - parsed = json.loads(args_json)
1199 - assert parsed == {"city": "Beijing"}
1200 -
1201 -
1202 -@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
1203 -def test_stream_interval_multiple_tool_calls(glm4_moe_tokenizer, stream_interval):
1204 - """Multiple sequential tool calls with correct indices at any interval."""
1205 - tools = [
1206 - ChatCompletionToolsParam(
1207 - function=FunctionDefinition(
1208 - name="get_weather",
1209 - parameters={
1210 - "type": "object",
1211 - "properties": {"city": {"type": "string"}},
1212 - },
1213 - ),
1214 - ),
1215 - ]
1216 - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1217 - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1218 -
1219 - text = (
1220 - "<tool_call>get_weather\n"
1221 - "<arg_key>city</arg_key>"
1222 - "<arg_value>Beijing</arg_value>"
1223 - "</tool_call>"
1224 - "<tool_call>get_weather\n"
1225 - "<arg_key>city</arg_key>"
1226 - "<arg_value>Shanghai</arg_value>"
1227 - "</tool_call>"
1228 - )
1229 -
1230 - deltas = _simulate_streaming(
1231 - glm4_moe_tokenizer, parser, request, text, stream_interval
1232 - )
1233 - _, tools_found = _collect_from_deltas(deltas)
1234 -
1235 - assert 0 in tools_found and 1 in tools_found
1236 - args0 = json.loads("".join(tools_found[0]["args_fragments"]))
1237 - args1 = json.loads("".join(tools_found[1]["args_fragments"]))
1238 - assert args0 == {"city": "Beijing"}
1239 - assert args1 == {"city": "Shanghai"}
1240 -
1241 -
1242 -@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
1243 -def test_stream_interval_content_then_tool_call(glm4_moe_tokenizer, stream_interval):
1244 - """Content before a tool call is fully emitted before tool deltas."""
1245 - tools = [
1246 - ChatCompletionToolsParam(
1247 - function=FunctionDefinition(
1248 - name="get_weather",
1249 - parameters={
1250 - "type": "object",
1251 - "properties": {"city": {"type": "string"}},
1252 - },
1253 - ),
70 + function=FunctionDefinition(name="get_time", parameters={}),
71 ),
72 ]
1256 - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1257 - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1258 -
1259 - text = (
1260 - "I will check the weather for you.\n"
1261 - "<tool_call>get_weather\n"
1262 - "<arg_key>city</arg_key>"
1263 - "<arg_value>Beijing</arg_value>"
1264 - "</tool_call>"
1265 - )
73
1267 - deltas = _simulate_streaming(
1268 - glm4_moe_tokenizer, parser, request, text, stream_interval
1269 - )
1270 - content_parts, tools_found = _collect_from_deltas(deltas)
74
1272 - # Content must be present and precede tool calls
1273 - full_content = "".join(content_parts)
1274 - assert "I will check the weather" in full_content
75 +def _request(tools: list[ChatCompletionToolsParam]) -> ChatCompletionRequest:
76 + return ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
77
1276 - # Tool call must be correct
1277 - assert 0 in tools_found
1278 - assert tools_found[0]["name"] == "get_weather"
1279 - args = json.loads("".join(tools_found[0]["args_fragments"]))
1280 - assert args == {"city": "Beijing"}
78
79 +def _parser(tools: list[ChatCompletionToolsParam] | None = None):
80 + return Glm47MoeModelToolParser(_mock_tokenizer(), tools=tools)
81
1283 -def test_stream_interval_extreme_single_chunk(glm4_moe_tokenizer):
1284 - """Extreme MTP: entire output arrives in one chunk (interval=9999)."""
1285 - tools = [
1286 - ChatCompletionToolsParam(
1287 - function=FunctionDefinition(
1288 - name="get_weather",
1289 - parameters={
1290 - "type": "object",
1291 - "properties": {"city": {"type": "string"}},
1292 - },
1293 - ),
1294 - ),
1295 - ]
1296 - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1297 - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
82
1299 - text = (
1300 - "Here is the weather.\n"
1301 - "<tool_call>get_weather\n"
1302 - "<arg_key>city</arg_key>"
1303 - "<arg_value>Beijing</arg_value>"
1304 - "</tool_call>"
1305 - )
83 +def _collect_tool_deltas(deltas: Any) -> dict[int, _CollectedToolDelta]:
84 + calls: dict[int, _CollectedToolDelta] = {}
85 + for delta in deltas:
86 + if delta is None or not delta.tool_calls:
87 + continue
88 + for tool_call in delta.tool_calls:
89 + entry = calls.setdefault(
90 + tool_call.index,
91 + {"name": None, "args_fragments": []},
92 + )
93 + function = tool_call.function
94 + if function is None:
95 + continue
96 + if isinstance(function, dict):
97 + name = function.get("name")
98 + arguments = function.get("arguments")
99 + else:
100 + name = function.name
101 + arguments = function.arguments
102 + if isinstance(name, str) and name:
103 + entry["name"] = name
104 + if isinstance(arguments, str) and arguments:
105 + entry["args_fragments"].append(arguments)
106 + return calls
107
1307 - deltas = _simulate_streaming(
1308 - glm4_moe_tokenizer, parser, request, text, stream_interval=9999
1309 - )
1310 - content_parts, tools_found = _collect_from_deltas(deltas)
108
1312 - assert "Here is the weather" in "".join(content_parts)
1313 - assert 0 in tools_found
1314 - assert tools_found[0]["name"] == "get_weather"
1315 - args = json.loads("".join(tools_found[0]["args_fragments"]))
1316 - assert args == {"city": "Beijing"}
109 +def test_glm45_uses_shared_glm47_parser():
110 + assert ToolParserManager.get_tool_parser("glm45") is Glm47MoeModelToolParser
111 + assert ToolParserManager.get_tool_parser("glm47") is Glm47MoeModelToolParser
112
113
1319 -@pytest.mark.parametrize("stream_interval", [1, 2, 5])
1320 -def test_stream_interval_content_between_tool_calls(
1321 - glm4_moe_tokenizer, stream_interval
1322 -):
1323 - """Content between tool calls must be emitted, not silently dropped."""
1324 - tools = [
1325 - ChatCompletionToolsParam(
1326 - function=FunctionDefinition(
1327 - name="get_weather",
1328 - parameters={
1329 - "type": "object",
1330 - "properties": {"city": {"type": "string"}},
1331 - },
1332 - ),
1333 - ),
1334 - ]
1335 - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1336 - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1337 -
1338 - text = (
1339 - "Checking Beijing.\n"
1340 - "<tool_call>get_weather\n"
1341 - "<arg_key>city</arg_key>"
1342 - "<arg_value>Beijing</arg_value>"
1343 - "</tool_call>"
1344 - "\nAlso Shanghai.\n"
1345 - "<tool_call>get_weather\n"
1346 - "<arg_key>city</arg_key>"
1347 - "<arg_value>Shanghai</arg_value>"
1348 - "</tool_call>"
1349 - )
1350 -
1351 - deltas = _simulate_streaming(
1352 - glm4_moe_tokenizer, parser, request, text, stream_interval
1353 - )
1354 - content_parts, tools_found = _collect_from_deltas(deltas)
1355 -
1356 - full_content = "".join(content_parts)
1357 - # Both prefix and inter-tool-call content must appear
1358 - assert "Checking Beijing" in full_content
1359 - assert "Also Shanghai" in full_content
1360 -
1361 - # Both tool calls must be correct
1362 - assert 0 in tools_found and 1 in tools_found
1363 - args0 = json.loads("".join(tools_found[0]["args_fragments"]))
1364 - args1 = json.loads("".join(tools_found[1]["args_fragments"]))
1365 - assert args0 == {"city": "Beijing"}
1366 - assert args1 == {"city": "Shanghai"}
1367 -
1368 -
1369 -# ── FunctionTool (Responses API) tests ──────────────────────────────
1370 -
1371 -
1372 -@pytest.fixture
1373 -def function_tools():
1374 - return [
1375 - FunctionTool(
1376 - type="function",
1377 - name="get_weather",
1378 - parameters={
1379 - "type": "object",
1380 - "properties": {
1381 - "city": {"type": "string"},
1382 - "unit": {"type": "string"},
1383 - },
1384 - },
1385 - ),
1386 - FunctionTool(
1387 - type="function",
1388 - name="calculate",
1389 - parameters={
1390 - "type": "object",
1391 - "properties": {
1392 - "operation": {"type": "string"},
1393 - "a": {"type": "number"},
1394 - "b": {"type": "number"},
1395 - },
1396 - },
1397 - ),
1398 - ]
1399 -
1400 -
1401 -@pytest.fixture
1402 -def glm4_moe_parser_function_tools(glm4_moe_tokenizer, function_tools):
1403 - return Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=function_tools)
1404 -
1405 -
1406 -@pytest.fixture
1407 -def mock_request_function_tools(function_tools) -> ChatCompletionRequest:
1408 - request = Mock(spec=ChatCompletionRequest)
1409 - request.tools = function_tools
1410 - return request
1411 -
1412 -
1413 -def test_extract_tool_calls_with_function_tool(
1414 - glm4_moe_parser_function_tools, mock_request_function_tools
1415 -):
1416 - model_output = """<tool_call>get_weather
114 +def test_extract_tool_calls_with_glm45_newline_format():
115 + tools = _tools()
116 + parser = _parser(tools)
117 + model_output = """I'll check it. <tool_call>get_current_weather
118 <arg_key>city</arg_key>
119 <arg_value>Dallas</arg_value>
120 +<arg_key>state</arg_key>
121 +<arg_value>TX</arg_value>
122 <arg_key>unit</arg_key>
123 <arg_value>fahrenheit</arg_value>
124 </tool_call>"""
125
1423 - extracted = glm4_moe_parser_function_tools.extract_tool_calls(
1424 - model_output, request=mock_request_function_tools
1425 - )
126 + extracted = parser.extract_tool_calls(model_output, request=_request(tools))
127 +
128 assert extracted.tools_called
129 + assert extracted.content == "I'll check it."
130 assert len(extracted.tool_calls) == 1
1428 - assert extracted.tool_calls[0].function.name == "get_weather"
1429 - args = json.loads(extracted.tool_calls[0].function.arguments)
1430 - assert args["city"] == "Dallas"
1431 - assert args["unit"] == "fahrenheit"
131 + tool_call = extracted.tool_calls[0]
132 + assert tool_call.function.name == "get_current_weather"
133 + assert json.loads(tool_call.function.arguments) == {
134 + "city": "Dallas",
135 + "state": "TX",
136 + "unit": "fahrenheit",
137 + }
138 +
139 +
140 +def test_extract_multiple_tool_calls_with_glm45_newline_format():
141 + tools = _tools()
142 + parser = _parser(tools)
143 + model_output = """<tool_call>get_current_weather
144 +<arg_key>city</arg_key><arg_value>Dallas</arg_value>
145 +</tool_call>
146 +<tool_call>get_current_weather
147 +<arg_key>city</arg_key><arg_value>Orlando</arg_value>
148 +</tool_call>"""
149
150 + extracted = parser.extract_tool_calls(model_output, request=_request(tools))
151
1434 -def test_extract_tool_calls_with_function_tool_mixed_types(
1435 - glm4_moe_parser_function_tools, mock_request_function_tools
1436 -):
152 + assert extracted.tools_called
153 + assert [tc.function.name for tc in extracted.tool_calls] == [
154 + "get_current_weather",
155 + "get_current_weather",
156 + ]
157 + assert [
158 + json.loads(tc.function.arguments)["city"] for tc in extracted.tool_calls
159 + ] == ["Dallas", "Orlando"]
160 +
161 +
162 +def test_extract_tool_calls_coerces_schema_types():
163 + tools = _tools()
164 + parser = _parser(tools)
165 model_output = """<tool_call>calculate
1438 -<arg_key>operation</arg_key>
1439 -<arg_value>add</arg_value>
1440 -<arg_key>a</arg_key>
1441 -<arg_value>42</arg_value>
1442 -<arg_key>b</arg_key>
1443 -<arg_value>3.14</arg_value>
166 +<arg_key>operation</arg_key><arg_value>add</arg_value>
167 +<arg_key>a</arg_key><arg_value>42</arg_value>
168 +<arg_key>b</arg_key><arg_value>3.14</arg_value>
169 +<arg_key>enabled</arg_key><arg_value>true</arg_value>
170 </tool_call>"""
171
1446 - extracted = glm4_moe_parser_function_tools.extract_tool_calls(
1447 - model_output, request=mock_request_function_tools
1448 - )
172 + extracted = parser.extract_tool_calls(model_output, request=_request(tools))
173 +
174 assert extracted.tools_called
1450 - args = json.loads(extracted.tool_calls[0].function.arguments)
1451 - assert args["operation"] == "add"
1452 - assert isinstance(args["a"], (int, float))
1453 - assert isinstance(args["b"], float)
175 + assert json.loads(extracted.tool_calls[0].function.arguments) == {
176 + "operation": "add",
177 + "a": 42,
178 + "b": 3.14,
179 + "enabled": True,
180 + }
181
182
1456 -def test_streaming_with_function_tool(
1457 - glm4_moe_parser_function_tools, mock_request_function_tools
1458 -):
1459 - _reset_streaming_state(glm4_moe_parser_function_tools)
183 +def test_extract_zero_argument_tool_call_with_glm45_newline_format():
184 + tools = _tools()
185 + parser = _parser(tools)
186
187 + extracted = parser.extract_tool_calls(
188 + "<tool_call>get_time\n</tool_call>",
189 + request=_request(tools),
190 + )
191 +
192 + assert extracted.tools_called
193 + assert extracted.tool_calls[0].function.name == "get_time"
194 + assert json.loads(extracted.tool_calls[0].function.arguments) == {}
195 +
196 +
197 +def test_streaming_tool_call_with_glm45_newline_format():
198 + tools = _tools()
199 + parser = _parser(tools)
200 + request = _request(tools)
201 chunks = [
1462 - "<tool_call>get_weather\n",
202 + "<tool_call>",
203 + "get_current_weather\n",
204 "<arg_key>city</arg_key>",
205 "<arg_value>Bei",
1465 - "jing",
1466 - "</arg_value>",
206 + "jing</arg_value>",
207 "</tool_call>",
208 ]
1469 -
209 + deltas = []
210 current_text = ""
211 +
212 for chunk in chunks:
213 current_text += chunk
1473 - glm4_moe_parser_function_tools.extract_tool_calls_streaming(
1474 - previous_text="",
1475 - current_text=current_text,
1476 - delta_text=chunk,
1477 - previous_token_ids=[],
1478 - current_token_ids=[],
1479 - delta_token_ids=[],
1480 - request=mock_request_function_tools,
214 + deltas.append(
215 + parser.extract_tool_calls_streaming(
216 + previous_text="",
217 + current_text=current_text,
218 + delta_text=chunk,
219 + previous_token_ids=[],
220 + current_token_ids=[],
221 + delta_token_ids=[],
222 + request=request,
223 + )
224 )
225
1483 - assert len(glm4_moe_parser_function_tools.prev_tool_call_arr) == 1
1484 - args = json.loads(glm4_moe_parser_function_tools.prev_tool_call_arr[0]["arguments"])
1485 - assert args["city"] == "Beijing"
226 + calls = _collect_tool_deltas(deltas)
227 + assert calls[0]["name"] == "get_current_weather"
228 + assert json.loads("".join(calls[0]["args_fragments"])) == {"city": "Beijing"}
vllm/parser/engine/registered_adapters.py
+6
@@ -9,6 +9,7 @@ names so that :class:`ReasoningParserManager` and
9
10 from vllm.parser.engine.adapters import make_adapters
11 from vllm.parser.gemma4 import Gemma4Parser
12 +from vllm.parser.glm47_moe import Glm47MoeParser
13 from vllm.parser.minimax_m2 import MinimaxM2Parser
14 from vllm.parser.nemotron_v3 import NemotronV3Parser
15 from vllm.parser.qwen3 import Qwen3Parser
@@ -32,3 +33,8 @@ from vllm.parser.qwen3 import Qwen3Parser
33 Qwen3ParserReasoningAdapter,
34 Qwen3ParserToolAdapter,
35 ) = make_adapters(Qwen3Parser)
36 +
37 +(
38 + Glm47MoeParserReasoningAdapter,
39 + Glm47MoeParserToolAdapter,
40 +) = make_adapters(Glm47MoeParser)
vllm/parser/glm47_moe.py new
+226
@@ -0,0 +1,226 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""GLM-4.7 parser for reasoning and tool calls.
4 +
5 +GLM-4.7 uses XML-like tool calls::
6 +
7 + <tool_call>func_name<arg_key>key</arg_key><arg_value>value</arg_value></tool_call>
8 +
9 +The function name can be followed directly by the first ``<arg_key>`` tag,
10 +and tool calls may have no arguments.
11 +"""
12 +
13 +from __future__ import annotations
14 +
15 +import functools
16 +import json
17 +from typing import TYPE_CHECKING
18 +
19 +import regex as re
20 +
21 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
22 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
23 +from vllm.parser.engine.events import EventType
24 +from vllm.parser.engine.parser_engine import ParserEngine
25 +from vllm.parser.engine.parser_engine_config import (
26 + ParserEngineConfig,
27 + ParserState,
28 + Transition,
29 +)
30 +
31 +if TYPE_CHECKING:
32 + from vllm.tokenizers import TokenizerLike
33 + from vllm.tool_parsers.abstract_tool_parser import Tool
34 +
35 +THINK_START = "<think>"
36 +THINK_END = "</think>"
37 +TOOL_CALL_START = "<tool_call>"
38 +TOOL_CALL_END = "</tool_call>"
39 +ARG_KEY_START = "<arg_key>"
40 +ARG_KEY_END = "</arg_key>"
41 +ARG_VALUE_START = "<arg_value>"
42 +ARG_VALUE_END = "</arg_value>"
43 +
44 +_ARG_RE = re.compile(
45 + r"<arg_key>(?P<key>.*?)</arg_key>\s*"
46 + r"<arg_value>(?P<value>.*?)</arg_value>",
47 + re.DOTALL,
48 +)
49 +_PARTIAL_ARG_RE = re.compile(
50 + r"<arg_key>(?P<key>.*?)</arg_key>\s*"
51 + r"<arg_value>(?P<value>.*)$",
52 + re.DOTALL,
53 +)
54 +
55 +
56 +def _glm47_arg_converter(raw_args: str, partial: bool) -> str:
57 + params: dict[str, object] = {}
58 +
59 + for match in _ARG_RE.finditer(raw_args):
60 + params[match.group("key").strip()] = match.group("value")
61 +
62 + if partial:
63 + remaining = _ARG_RE.sub("", raw_args)
64 + match = _PARTIAL_ARG_RE.search(remaining)
65 + if match:
66 + key = match.group("key").strip()
67 + if key:
68 + params[key] = match.group("value")
69 +
70 + return json.dumps(params, ensure_ascii=False)
71 +
72 +
73 +@functools.cache
74 +def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig:
75 + arg_tag_transitions = {
76 + (ParserState.TOOL_ARGS, terminal): Transition(
77 + ParserState.TOOL_ARGS,
78 + (EventType.ARG_VALUE_CHUNK,),
79 + )
80 + for terminal in (
81 + "ARG_KEY_START",
82 + "ARG_KEY_END",
83 + "ARG_VALUE_START",
84 + "ARG_VALUE_END",
85 + )
86 + }
87 +
88 + reasoning_terminals = (
89 + {
90 + "THINK_START": THINK_START,
91 + "THINK_END": THINK_END,
92 + }
93 + if thinking
94 + else {}
95 + )
96 + reasoning_token_id_terminals = (
97 + {
98 + "THINK_START": THINK_START,
99 + "THINK_END": THINK_END,
100 + }
101 + if thinking
102 + else {}
103 + )
104 + reasoning_transitions = (
105 + {
106 + (ParserState.CONTENT, "THINK_START"): Transition(
107 + ParserState.REASONING,
108 + (EventType.REASONING_START,),
109 + ),
110 + (ParserState.REASONING, "THINK_END"): Transition(
111 + ParserState.CONTENT,
112 + (EventType.REASONING_END,),
113 + ),
114 + (ParserState.CONTENT, "THINK_END"): Transition(
115 + ParserState.CONTENT,
116 + (),
117 + ),
118 + }
119 + if thinking
120 + else {}
121 + )
122 +
123 + return ParserEngineConfig(
124 + name="glm47_moe",
125 + initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
126 + terminals={
127 + **reasoning_terminals,
128 + "TOOL_START": TOOL_CALL_START,
129 + "TOOL_END": TOOL_CALL_END,
130 + "ARG_KEY_START": ARG_KEY_START,
131 + "ARG_KEY_END": ARG_KEY_END,
132 + "ARG_VALUE_START": ARG_VALUE_START,
133 + "ARG_VALUE_END": ARG_VALUE_END,
134 + },
135 + token_id_terminals={
136 + **reasoning_token_id_terminals,
137 + "TOOL_START": TOOL_CALL_START,
138 + "TOOL_END": TOOL_CALL_END,
139 + },
140 + transitions={
141 + **reasoning_transitions,
142 + (ParserState.REASONING, "THINK_START"): Transition(
143 + ParserState.REASONING,
144 + (),
145 + ),
146 + (ParserState.REASONING, "TOOL_START"): Transition(
147 + ParserState.TOOL_NAME,
148 + (EventType.REASONING_END, EventType.TOOL_CALL_START),
149 + ),
150 + (ParserState.CONTENT, "TOOL_START"): Transition(
151 + ParserState.TOOL_NAME,
152 + (EventType.TOOL_CALL_START,),
153 + ),
154 + (ParserState.TOOL_NAME, "ARG_KEY_START"): Transition(
155 + ParserState.TOOL_ARGS,
156 + (EventType.ARG_VALUE_CHUNK,),
157 + ),
158 + (ParserState.TOOL_NAME, "TOOL_END"): Transition(
159 + ParserState.CONTENT,
160 + (EventType.TOOL_CALL_END,),
161 + ),
162 + (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
163 + ParserState.CONTENT,
164 + (EventType.TOOL_CALL_END,),
165 + ),
166 + **arg_tag_transitions,
167 + },
168 + arg_converter=_glm47_arg_converter,
169 + stream_arg_deltas=True,
170 + tool_args_json=False,
171 + validate_tool_names=True,
172 + )
173 +
174 +
175 +class Glm47MoeParser(ParserEngine):
176 + """GLM-4.7 parser backed by the declarative parser engine."""
177 +
178 + def __init__(
179 + self,
180 + tokenizer: TokenizerLike,
181 + tools: list[Tool] | None = None,
182 + **kwargs,
183 + ) -> None:
184 + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
185 + thinking = chat_kwargs.get("thinking", None)
186 + enable_thinking = chat_kwargs.get("enable_thinking", None)
187 + self.thinking_enabled = (
188 + True
189 + if thinking is None and enable_thinking is None
190 + else bool(thinking) or bool(enable_thinking)
191 + )
192 + kwargs.setdefault(
193 + "parser_engine_config",
194 + glm47_moe_config(thinking=self.thinking_enabled),
195 + )
196 + super().__init__(tokenizer, tools, **kwargs)
197 +
198 + def _emit_name_delta(self, idx: int, deltas, name: str | None) -> None:
199 + if name is not None:
200 + name = name.strip()
201 + super()._emit_name_delta(idx, deltas, name)
202 +
203 + def _handle_tool_end(self, event, deltas) -> None:
204 + idx = event.tool_index
205 + if 0 <= idx < len(self._tool_slots):
206 + self._tool_slots[idx].name = self._tool_slots[idx].name.strip()
207 + super()._handle_tool_end(event, deltas)
208 +
209 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
210 + if not self.thinking_enabled:
211 + return True
212 + return super().is_reasoning_end(input_ids)
213 +
214 + def extract_content_ids(self, input_ids: list[int]) -> list[int]:
215 + if not self.thinking_enabled:
216 + return input_ids
217 + return super().extract_content_ids(input_ids)
218 +
219 + def extract_reasoning(
220 + self,
221 + model_output: str,
222 + request: ChatCompletionRequest | ResponsesRequest,
223 + ) -> tuple[str | None, str | None]:
224 + if not self.thinking_enabled:
225 + return None, model_output
226 + return super().extract_reasoning(model_output, request)
vllm/reasoning/__init__.py
+6 -2
@@ -53,8 +53,12 @@ _REASONING_PARSERS_TO_REGISTER = {
53 "Gemma4ParserReasoningAdapter",
54 ),
55 "glm45": (
56 - "deepseek_v3_reasoning_parser",
57 - "DeepSeekV3ReasoningWithThinkingParser",
56 + "glm47_moe_reasoning_parser",
57 + "Glm47MoeParserReasoningAdapter",
58 + ),
59 + "glm47": (
60 + "glm47_moe_reasoning_parser",
61 + "Glm47MoeParserReasoningAdapter",
62 ),
63 "openai_gptoss": (
64 "gptoss_reasoning_parser",
vllm/reasoning/glm47_moe_reasoning_parser.py new
+6
@@ -0,0 +1,6 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.engine.registered_adapters import Glm47MoeParserReasoningAdapter
5 +
6 +__all__ = ["Glm47MoeParserReasoningAdapter"]
vllm/tool_parsers/__init__.py
+2 -2
@@ -51,8 +51,8 @@ _TOOL_PARSERS_TO_REGISTER = {
51 "Ernie45ToolParser",
52 ),
53 "glm45": (
54 - "glm4_moe_tool_parser",
55 - "Glm4MoeModelToolParser",
54 + "glm47_moe_tool_parser",
55 + "Glm47MoeModelToolParser",
56 ),
57 "glm47": (
58 "glm47_moe_tool_parser",
vllm/tool_parsers/glm47_moe_tool_parser.py
+3 -33
@@ -1,41 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""
4 -GLM-4.7 Tool Call Parser.
3
6 -GLM-4.7 uses a slightly different tool call format compared to GLM-4.5:
7 - - The function name may appear on the same line as ``<tool_call>`` without
8 - a newline separator before the first ``<arg_key>``.
9 - - Tool calls may have zero arguments
10 - (e.g. ``<tool_call>func</tool_call>``).
4 +from __future__ import annotations
5
12 -This parser overrides the parent regex patterns to handle both formats.
13 -"""
6 +from vllm.parser.engine.registered_adapters import Glm47MoeParserToolAdapter
7
15 -import regex as re
8
17 -from vllm.logger import init_logger
18 -from vllm.tokenizers import TokenizerLike
19 -from vllm.tool_parsers.abstract_tool_parser import Tool
20 -from vllm.tool_parsers.glm4_moe_tool_parser import Glm4MoeModelToolParser
21 -
22 -logger = init_logger(__name__)
23 -
24 -
25 -class Glm47MoeModelToolParser(Glm4MoeModelToolParser):
9 +class Glm47MoeModelToolParser(Glm47MoeParserToolAdapter): # type: ignore[valid-type, misc]
10 supports_required_and_named = False
11 structural_tag_model = "glm_4_7"
28 -
29 - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
30 - super().__init__(tokenizer, tools)
31 - # GLM-4.7 format: <tool_call>func_name[<arg_key>...]*</tool_call>
32 - # The function name can be followed by a newline, whitespace, or
33 - # directly by <arg_key> tags (no separator). The arg section is
34 - # optional so that zero-argument calls are supported.
35 - self.func_detail_regex = re.compile(
36 - r"<tool_call>\s*(\S+?)\s*(<arg_key>.*)?</tool_call>", re.DOTALL
37 - )
38 - self.func_arg_regex = re.compile(
39 - r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>",
40 - re.DOTALL,
41 - )
vllm/tool_parsers/glm4_moe_tool_parser.py deleted
-495
@@ -1,495 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""
4 -GLM-4 Tool Call Parser with incremental string streaming support.
5 -
6 -This parser fixes the streaming issue reported in Issue #32829 where long string
7 -parameters (e.g., file content with 4000+ characters of code) are buffered until
8 -complete, causing multi-second delays before the user sees any content.
9 -
10 -The fix streams string values incrementally as they arrive, providing a true
11 -streaming experience for long content.
12 -"""
13 -
14 -import json
15 -from collections.abc import Sequence
16 -from typing import Any
17 -
18 -import regex as re
19 -
20 -from vllm.entrypoints.chat_utils import make_tool_call_id
21 -from vllm.entrypoints.openai.chat_completion.protocol import (
22 - ChatCompletionNamedToolChoiceParam,
23 - ChatCompletionRequest,
24 -)
25 -from vllm.entrypoints.openai.engine.protocol import (
26 - DeltaFunctionCall,
27 - DeltaMessage,
28 - DeltaToolCall,
29 - ExtractedToolCallInformation,
30 - FunctionCall,
31 - ToolCall,
32 -)
33 -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
34 -from vllm.logger import init_logger
35 -from vllm.tokenizers import TokenizerLike
36 -from vllm.tool_parsers.abstract_tool_parser import (
37 - Tool,
38 - ToolParser,
39 -)
40 -from vllm.tool_parsers.utils import (
41 - extract_types_from_schema,
42 - find_tool_properties,
43 - partial_tag_overlap,
44 - safe_literal_eval,
45 -)
46 -
47 -logger = init_logger(__name__)
48 -
49 -
50 -class Glm4MoeModelToolParser(ToolParser):
51 - """Tool parser for GLM-4 models with incremental string streaming.
52 -
53 - On every streaming call the parser re-parses ``current_text`` to find
54 - ``<tool_call>`` regions, builds the JSON arguments string for each tool
55 - call, and diffs against what was previously sent to emit only new content.
56 - """
57 -
58 - supports_required_and_named = False
59 -
60 - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
61 - super().__init__(tokenizer, tools)
62 - # Stateful streaming fields
63 - self.current_tool_name_sent: bool = False
64 - self.prev_tool_call_arr: list[dict[str, Any]] = []
65 - self.current_tool_id: int = -1
66 - self.streamed_args_for_tool: list[str] = []
67 -
68 - self.tool_call_start_token: str = "<tool_call>"
69 - self.tool_call_end_token: str = "</tool_call>"
70 - self.arg_key_start: str = "<arg_key>"
71 - self.arg_key_end: str = "</arg_key>"
72 - self.arg_val_start: str = "<arg_value>"
73 - self.arg_val_end: str = "</arg_value>"
74 -
75 - self.tool_calls_start_token = self.tool_call_start_token
76 -
77 - self.func_call_regex = re.compile(r"<tool_call>.*?</tool_call>", re.DOTALL)
78 - self.func_detail_regex = re.compile(
79 - r"<tool_call>([^\n]*)\n(.*)</tool_call>", re.DOTALL
80 - )
81 - self.func_arg_regex = re.compile(
82 - r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>", re.DOTALL
83 - )
84 -
85 - if not self.model_tokenizer:
86 - raise ValueError(
87 - "The model tokenizer must be passed to the ToolParser "
88 - "constructor during construction."
89 - )
90 -
91 - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
92 - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
93 -
94 - # Pre-compiled pattern for finding the last <arg_key>...</arg_key>
95 - # before a partial <arg_value> (used in _build_args_json_so_far).
96 - self._arg_key_pattern = re.compile(
97 - re.escape(self.arg_key_start) + r"(.*?)" + re.escape(self.arg_key_end),
98 - re.DOTALL,
99 - )
100 -
101 - # Streaming state for re-parse-and-diff approach
102 - self._sent_content_idx: int = 0
103 - self._tool_call_ids: list[str] = []
104 -
105 - @staticmethod
106 - def _deserialize(value: str) -> Any:
107 - try:
108 - return json.loads(value)
109 - except json.JSONDecodeError:
110 - pass
111 -
112 - try:
113 - return safe_literal_eval(value)
114 - except (ValueError, SyntaxError):
115 - pass
116 -
117 - return value
118 -
119 - @staticmethod
120 - def _json_escape_string_content(s: str) -> str:
121 - """JSON-escape string content for incremental streaming.
122 -
123 - This escapes the content that goes INSIDE a JSON string (between quotes),
124 - not including the surrounding quotes themselves.
125 - """
126 - if not s:
127 - return ""
128 - return json.dumps(s, ensure_ascii=False)[1:-1]
129 -
130 - def _is_string_type(self, tool_name: str, arg_name: str) -> bool:
131 - tool_properties = find_tool_properties(self.tools, tool_name)
132 - param_schema = tool_properties.get(arg_name)
133 - if param_schema is None:
134 - return False
135 - param_types = extract_types_from_schema(param_schema)
136 - return set(param_types) - {"null"} == {"string"}
137 -
138 - @staticmethod
139 - def _tools_enabled(request: ChatCompletionRequest) -> bool:
140 - """Return whether tool parsing should be applied for this request."""
141 - try:
142 - tools = getattr(request, "tools", None)
143 - tool_choice = getattr(request, "tool_choice", None)
144 - return bool(tools) and tool_choice != "none"
145 - except Exception:
146 - logger.exception("Failed to determine if tools are enabled.")
147 - return False
148 -
149 - def adjust_request(
150 - self, request: ChatCompletionRequest | ResponsesRequest
151 - ) -> ChatCompletionRequest | ResponsesRequest:
152 - """Adjust request parameters for tool call token handling.
153 -
154 - For required/named tool_choice, skip setting structured_outputs
155 - because GLM models output tool calls in XML format (per chat
156 - template). Guided decoding would force JSON output, conflicting
157 - with the XML format and causing parsing failures.
158 - """
159 - if request.tools:
160 - tc = request.tool_choice
161 - if tc == "required" or isinstance(tc, ChatCompletionNamedToolChoiceParam):
162 - # Do NOT call super().adjust_request() for required/named,
163 - # because it would set structured_outputs and force JSON
164 - # output via guided decoding. GLM models use XML tool-call
165 - # syntax (defined in the chat template), so guided decoding
166 - # must be skipped to let the model output XML freely.
167 - # The tool_parser handles extraction from XML output.
168 - if request.tool_choice != "none":
169 - request.skip_special_tokens = False
170 - return request
171 - request = super().adjust_request(request)
172 - if request.tools and request.tool_choice != "none":
173 - # Ensure tool call tokens (<tool_call>, </tool_call>) are not skipped
174 - # during decoding. Even though they are not marked as special tokens,
175 - # setting skip_special_tokens=False ensures proper handling in
176 - # transformers 5.x where decoding behavior may have changed.
177 - request.skip_special_tokens = False
178 - return request
179 -
180 - def extract_tool_calls(
181 - self,
182 - model_output: str,
183 - request: ChatCompletionRequest,
184 - ) -> ExtractedToolCallInformation:
185 - matched_tool_calls = self.func_call_regex.findall(model_output)
186 - logger.debug("model_output: %s", model_output)
187 - try:
188 - tool_calls: list[ToolCall] = []
189 - for match in matched_tool_calls:
190 - tc_detail = self.func_detail_regex.search(match)
191 - if not tc_detail:
192 - logger.warning(
193 - "Failed to parse tool call details from: %s",
194 - match,
195 - )
196 - continue
197 - tc_name = tc_detail.group(1).strip()
198 - tc_args = tc_detail.group(2)
199 - pairs = self.func_arg_regex.findall(tc_args) if tc_args else []
200 - arg_dct: dict[str, Any] = {}
201 - for key, value in pairs:
202 - arg_key = key.strip()
203 - if self._is_string_type(tc_name, arg_key):
204 - arg_val = value
205 - else:
206 - arg_val = self._deserialize(value.strip())
207 - logger.debug("arg_key = %s, arg_val = %s", arg_key, arg_val)
208 - arg_dct[arg_key] = arg_val
209 - tool_calls.append(
210 - ToolCall(
211 - type="function",
212 - function=FunctionCall(
213 - name=tc_name,
214 - arguments=json.dumps(arg_dct, ensure_ascii=False),
215 - ),
216 - )
217 - )
218 - except Exception:
219 - logger.exception("Failed to extract tool call spec")
220 - return ExtractedToolCallInformation(
221 - tools_called=False, tool_calls=[], content=model_output
222 - )
223 - else:
224 - if len(tool_calls) > 0:
225 - content: str | None = model_output[
226 - : model_output.find(self.tool_calls_start_token)
227 - ]
228 - # Normalize empty/whitespace-only content to None
229 - if not content or not content.strip():
230 - content = None
231 - return ExtractedToolCallInformation(
232 - tools_called=True, tool_calls=tool_calls, content=content
233 - )
234 - return ExtractedToolCallInformation(
235 - tools_called=False, tool_calls=[], content=model_output
236 - )
237 -
238 - def _extract_content(self, current_text: str) -> str | None:
239 - """Return unsent non-tool-call text, or None.
240 -
241 - Collects all text outside ``<tool_call>...</tool_call>`` regions,
242 - including text between consecutive tool calls. Holds back any
243 - suffix that could be a partial ``<tool_call>`` tag.
244 - """
245 - # Build the "sendable index" — the furthest point we can send
246 - # content up to. We scan through the text collecting segments
247 - # that are outside tool-call regions.
248 - content_segments: list[str] = []
249 - pos = self._sent_content_idx
250 -
251 - while pos < len(current_text):
252 - start = current_text.find(self.tool_call_start_token, pos)
253 - if start == -1:
254 - # No more tool calls — send up to (len - partial-tag overlap)
255 - tail = current_text[pos:]
256 - overlap = partial_tag_overlap(tail, self.tool_call_start_token)
257 - sendable = tail[: len(tail) - overlap] if overlap else tail
258 - if sendable:
259 - content_segments.append(sendable)
260 - pos = len(current_text) - overlap
261 - break
262 -
263 - # Text before this <tool_call>
264 - if start > pos:
265 - content_segments.append(current_text[pos:start])
266 -
267 - # Skip past the </tool_call> (or to end if incomplete)
268 - end = current_text.find(self.tool_call_end_token, start)
269 - if end != -1:
270 - pos = end + len(self.tool_call_end_token)
271 - else:
272 - # Incomplete tool call — nothing more to send
273 - pos = start
274 - break
275 -
276 - if content_segments:
277 - self._sent_content_idx = pos
278 - return "".join(content_segments)
279 - # Even if no content, advance past completed tool-call regions
280 - if pos > self._sent_content_idx:
281 - self._sent_content_idx = pos
282 - return None
283 -
284 - def _extract_tool_call_regions(self, text: str) -> list[tuple[str, bool]]:
285 - """Extract ``(inner_text, is_complete)`` for each ``<tool_call>`` region."""
286 - results: list[tuple[str, bool]] = []
287 - pos = 0
288 - while True:
289 - start = text.find(self.tool_call_start_token, pos)
290 - if start == -1:
291 - break
292 - inner_start = start + len(self.tool_call_start_token)
293 - end = text.find(self.tool_call_end_token, inner_start)
294 - if end != -1:
295 - results.append((text[inner_start:end], True))
296 - pos = end + len(self.tool_call_end_token)
297 - else:
298 - # Incomplete tool call — strip partial </tool_call> suffix
299 - raw = text[inner_start:]
300 - overlap = partial_tag_overlap(raw, self.tool_call_end_token)
301 - if overlap:
302 - raw = raw[:-overlap]
303 - results.append((raw, False))
304 - break
305 - return results
306 -
307 - def _extract_tool_name_from_region(self, inner_text: str) -> str | None:
308 - """Extract the tool name from the beginning of a tool-call region.
309 -
310 - The name is everything before the first ``\\n`` or ``<arg_key>``.
311 - Returns ``None`` if the name hasn't fully arrived yet.
312 - """
313 - nl = inner_text.find("\n")
314 - ak = inner_text.find(self.arg_key_start)
315 - candidates = [i for i in [nl, ak] if i != -1]
316 - if not candidates:
317 - return None
318 - cut = min(candidates)
319 - name = inner_text[:cut].strip()
320 - return name if name else None
321 -
322 - def _build_args_json_so_far(
323 - self,
324 - tool_name: str,
325 - inner_text: str,
326 - is_complete: bool,
327 - ) -> str:
328 - """Build the JSON arguments string from the XML pairs seen so far.
329 -
330 - For complete ``<arg_key>/<arg_value>`` pairs the value is fully
331 - formatted. For the last argument whose ``<arg_value>`` has been
332 - opened but not closed, the partial string content is included
333 - (JSON-escaped, with an opening ``"`` but no closing ``"``).
334 -
335 - The closing ``}`` is only appended when ``is_complete`` is True
336 - (i.e. the ``</tool_call>`` tag has arrived).
337 - """
338 - # Find all complete arg pairs
339 - pairs = self.func_arg_regex.findall(inner_text)
340 -
341 - parts: list[str] = []
342 - for key, value in pairs:
343 - key = key.strip()
344 - key_json = json.dumps(key, ensure_ascii=False)
345 - if self._is_string_type(tool_name, key):
346 - # Don't strip string values — whitespace is significant
347 - # and must match the partial-value path for diffing.
348 - val_json = json.dumps(value, ensure_ascii=False)
349 - else:
350 - val_json = json.dumps(
351 - self._deserialize(value.strip()), ensure_ascii=False
352 - )
353 - parts.append(f"{key_json}: {val_json}")
354 -
355 - # Check for a partial (incomplete) arg value
356 - # Find the last <arg_value> that isn't closed
357 - last_val_start = inner_text.rfind(self.arg_val_start)
358 - last_val_end = inner_text.rfind(self.arg_val_end)
359 - has_partial_value = last_val_start != -1 and (
360 - last_val_end == -1 or last_val_end < last_val_start
361 - )
362 -
363 - if has_partial_value:
364 - # Find the key for this partial value
365 - # Look for the last <arg_key>...</arg_key> before this <arg_value>
366 - last_key_match = None
367 - for m in self._arg_key_pattern.finditer(inner_text[:last_val_start]):
368 - last_key_match = m
369 -
370 - if last_key_match:
371 - partial_key = last_key_match.group(1).strip()
372 - partial_content_start = last_val_start + len(self.arg_val_start)
373 - partial_content = inner_text[partial_content_start:]
374 -
375 - # Hold back any partial </arg_value> suffix
376 - overlap = partial_tag_overlap(partial_content, self.arg_val_end)
377 - if overlap:
378 - partial_content = partial_content[:-overlap]
379 -
380 - key_json = json.dumps(partial_key, ensure_ascii=False)
381 - if is_complete:
382 - # Tool call finished but </arg_value> is missing
383 - # (malformed output). Treat partial as complete value
384 - # so the diff naturally closes any open quotes.
385 - if self._is_string_type(tool_name, partial_key):
386 - val_json = json.dumps(partial_content, ensure_ascii=False)
387 - else:
388 - val_json = json.dumps(
389 - self._deserialize(partial_content.strip()),
390 - ensure_ascii=False,
391 - )
392 - parts.append(f"{key_json}: {val_json}")
393 - elif self._is_string_type(tool_name, partial_key):
394 - escaped = self._json_escape_string_content(partial_content)
395 - # Open quote but no close — more content may arrive
396 - parts.append(f'{key_json}: "{escaped}')
397 - else:
398 - # Non-string partial: include raw content, no wrapping
399 - parts.append(f"{key_json}: {partial_content}")
400 -
401 - if not parts:
402 - return "{}" if is_complete else ""
403 -
404 - joined = "{" + ", ".join(parts)
405 - if is_complete:
406 - joined += "}"
407 - return joined
408 -
409 - def _compute_args_diff(self, index: int, args_so_far: str) -> str | None:
410 - """Return new argument text not yet sent for tool *index*, or None."""
411 - if not args_so_far or len(args_so_far) <= len(
412 - self.streamed_args_for_tool[index]
413 - ):
414 - return None
415 - diff = args_so_far[len(self.streamed_args_for_tool[index]) :]
416 - self.streamed_args_for_tool[index] = args_so_far
417 - self.prev_tool_call_arr[index]["arguments"] = args_so_far
418 - return diff
419 -
420 - def _ensure_tool_state_for(self, index: int) -> None:
421 - """Grow state arrays so that *index* is valid."""
422 - while len(self._tool_call_ids) <= index:
423 - self._tool_call_ids.append(
424 - make_tool_call_id(id_type="random", func_name=None, idx=None)
425 - )
426 - while len(self.streamed_args_for_tool) <= index:
427 - self.streamed_args_for_tool.append("")
428 - while len(self.prev_tool_call_arr) <= index:
429 - self.prev_tool_call_arr.append({})
430 -
431 - def extract_tool_calls_streaming(
432 - self,
433 - previous_text: str,
434 - current_text: str,
435 - delta_text: str,
436 - previous_token_ids: Sequence[int],
437 - current_token_ids: Sequence[int],
438 - delta_token_ids: Sequence[int],
439 - request: ChatCompletionRequest,
440 - ) -> DeltaMessage | None:
441 - if not self._tools_enabled(request):
442 - return DeltaMessage(content=delta_text) if delta_text else None
443 -
444 - content = self._extract_content(current_text)
445 - regions = self._extract_tool_call_regions(current_text)
446 - tool_call_deltas: list[DeltaToolCall] = []
447 -
448 - for i, (inner_text, is_complete) in enumerate(regions):
449 - self._ensure_tool_state_for(i)
450 -
451 - # Extract tool name
452 - tool_name = self._extract_tool_name_from_region(inner_text)
453 - if not tool_name:
454 - break
455 -
456 - # Emit tool name (once per tool call)
457 - if "name" not in self.prev_tool_call_arr[i]:
458 - self.prev_tool_call_arr[i]["name"] = tool_name
459 - tool_call_deltas.append(
460 - DeltaToolCall(
461 - index=i,
462 - id=self._tool_call_ids[i],
463 - type="function",
464 - function=DeltaFunctionCall(
465 - name=tool_name,
466 - arguments="",
467 - ).model_dump(exclude_none=True),
468 - )
469 - )
470 -
471 - # Build args JSON so far, diff, emit
472 - args_so_far = self._build_args_json_so_far(
473 - tool_name, inner_text, is_complete
474 - )
475 - diff = self._compute_args_diff(i, args_so_far)
476 - if diff:
477 - tool_call_deltas.append(
478 - DeltaToolCall(
479 - index=i,
480 - function=DeltaFunctionCall(arguments=diff).model_dump(
481 - exclude_none=True
482 - ),
483 - )
484 - )
485 -
486 - # Update current_tool_id for serving layer compatibility
487 - if regions:
488 - self.current_tool_id = len(regions) - 1
489 -
490 - if content or tool_call_deltas:
491 - return DeltaMessage(
492 - content=content,
493 - tool_calls=tool_call_deltas,
494 - )
495 - return None