A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 17:00 UTC
GLM4 tool parser: fix streaming mode (#35208)
Signed-off-by: Robin Nabel <opensource@nabel.co> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Robin Nabel committed
Mar 16, 2026 at 11:48 UTC
bf9a1853958584fe039d33242a43c91cf8786d61
2 files changed
+26
-10
tests/tool_parsers/test_glm4_moe_tool_parser.py
+18
-6
@@ -560,19 +560,23 @@ def test_streaming_empty_tool_call(glm4_moe_tool_parser, mock_request):
560
assert glm4_moe_tool_parser.current_tool_id == -1
561
562
563
-def test_streaming_prev_tool_call_arr_finalization(glm4_moe_tool_parser, mock_request):
563
+def test_streaming_prev_tool_call_arr_updates(glm4_moe_tool_parser, mock_request):
564
"""Test that prev_tool_call_arr contains parsed dict after tool call."""
565
_reset_streaming_state(glm4_moe_tool_parser)
566
567
# Stream a complete tool call
568
+ name_only = {"name": "get_weather", "arguments": {}}
569
+ name_and_args = {"name": "get_weather", "arguments": {"city": "Beijing"}}
570
chunks = [
569
- "<tool_call>get_weather\n",
570
- "<arg_key>city</arg_key>",
571
- "<arg_value>Beijing</arg_value>",
572
- "</tool_call>",
571
+ # Delta, expected streamed_args_for_tool, expected prev_tool_call_arr
572
+ ("<tool_call>get_weather\n", "", name_only),
573
+ ("<arg_key>city</arg_key>", "", name_only),
574
+ ("<arg_value>Beijing</arg_value>", '{"city": "Beijing"', name_only),
575
+ # Note: arguments are only updated when the tool call is complete.
576
+ ("</tool_call>", '{"city": "Beijing"}', name_and_args),
577
]
578
575
- for chunk in chunks:
579
+ for chunk, exp_streamed, exp_prev_tc in chunks:
580
glm4_moe_tool_parser.extract_tool_calls_streaming(
581
previous_text="",
582
current_text="",
@@ -582,6 +586,8 @@ def test_streaming_prev_tool_call_arr_finalization(glm4_moe_tool_parser, mock_re
586
delta_token_ids=[],
587
request=mock_request,
588
)
589
+ assert glm4_moe_tool_parser.streamed_args_for_tool[0] == exp_streamed
590
+ assert glm4_moe_tool_parser.prev_tool_call_arr[0] == exp_prev_tc
591
592
# After the tool call completes, prev_tool_call_arr should have parsed dict
593
assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1
@@ -592,6 +598,12 @@ def test_streaming_prev_tool_call_arr_finalization(glm4_moe_tool_parser, mock_re
598
assert isinstance(args, dict), f"Expected dict, got {type(args)}"
599
assert args.get("city") == "Beijing"
600
601
+ # Test equivalence of prev_tool_call_arr and streamed_args_for_tool
602
+ # Simulates logic in chat_completion/serving.py:chat_completion_stream_generator
603
+ tool_call_json = json.dumps(tool_entry.get("arguments", {}))
604
+ streamed_content = glm4_moe_tool_parser.streamed_args_for_tool[0]
605
+ assert tool_call_json.startswith(streamed_content)
606
+
607
608
def test_streaming_multiple_tool_calls_sequential(glm4_moe_tool_parser, mock_request):
609
"""Test streaming multiple sequential tool calls."""
vllm/tool_parsers/glm4_moe_tool_parser.py
+8
-4
@@ -337,10 +337,10 @@ class Glm4MoeModelToolParser(ToolParser):
337
key_json = json.dumps(key, ensure_ascii=False)
338
339
if not self._args_started[self.current_tool_id]:
340
- frag = "{" + key_json + ':"'
340
+ frag = "{" + key_json + ': "'
341
self._args_started[self.current_tool_id] = True
342
else:
343
- frag = "," + key_json + ':"'
343
+ frag = ", " + key_json + ': "'
344
345
self.streamed_args_for_tool[self.current_tool_id] += frag
346
self._streaming_string_value = True
@@ -447,6 +447,10 @@ class Glm4MoeModelToolParser(ToolParser):
447
self.current_tool_id -= 1
448
449
def _emit_tool_name_delta(self, tool_name: str) -> DeltaMessage:
450
+ self.prev_tool_call_arr[self.current_tool_id] = {
451
+ "name": self._current_tool_name,
452
+ "arguments": {},
453
+ }
454
return DeltaMessage(
455
tool_calls=[
456
DeltaToolCall(
@@ -493,10 +497,10 @@ class Glm4MoeModelToolParser(ToolParser):
497
val_json = json.dumps(val_obj, ensure_ascii=False)
498
499
if not self._args_started[self.current_tool_id]:
496
- fragment = "{" + key_json + ":" + val_json
500
+ fragment = "{" + key_json + ": " + val_json
501
self._args_started[self.current_tool_id] = True
502
else:
499
- fragment = "," + key_json + ":" + val_json
503
+ fragment = "," + key_json + ": " + val_json
504
505
self._seen_keys[self.current_tool_id].add(key)
506
self.streamed_args_for_tool[self.current_tool_id] += fragment