@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0e39202ca9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 06:00 UTC

[Bugfix] Fix GLM tool parser streaming with MTP or stream interval (#39253)

Signed-off-by: sfeng33 <4florafeng@gmail.com>

Flora Feng committed Apr 13, 2026 at 01:10 UTC 0e39202ca911319c7747a2f9d5a0c162fdff4fd9
4 files changed +788 -416
tests/tool_parsers/test_glm47_moe_tool_parser.py
+14 -17
@@ -117,28 +117,24 @@ class TestGlm47ExtractToolCalls:
117
118
119 def _reset(parser):
120 - parser._buffer = ""
121 - parser._in_tool_call = False
120 parser.current_tool_name_sent = False
123 - parser._current_tool_name = None
124 - parser._pending_key = None
125 - parser._streaming_string_value = False
121 parser.prev_tool_call_arr = []
122 parser.current_tool_id = -1
123 parser.streamed_args_for_tool = []
124 parser._tool_call_ids = []
130 - parser._args_started = []
131 - parser._args_closed = []
132 - parser._seen_keys = []
125 + parser._sent_content_idx = 0
126
127
128 class TestGlm47Streaming:
129 def test_no_args(self, glm47_tool_parser, mock_request):
130 _reset(glm47_tool_parser)
138 - for chunk in ["<tool_call>", "get_current_date", "</tool_call>"]:
131 + chunks = ["<tool_call>", "get_current_date", "</tool_call>"]
132 + current_text = ""
133 + for chunk in chunks:
134 + current_text += chunk
135 glm47_tool_parser.extract_tool_calls_streaming(
136 previous_text="",
141 - current_text="",
137 + current_text=current_text,
138 delta_text=chunk,
139 previous_token_ids=[],
140 current_token_ids=[],
@@ -149,10 +145,7 @@ class TestGlm47Streaming:
145
146 def test_with_args(self, glm47_tool_parser, mock_request):
147 _reset(glm47_tool_parser)
152 - # Split chunks so that the incremental string streaming path
153 - # processes the value, its closing tag, and the tool-call closing
154 - # tag in separate calls.
155 - for chunk in [
148 + chunks = [
149 "<tool_call>",
150 "get_weather\n",
151 "<arg_key>city</arg_key>",
@@ -160,14 +153,18 @@ class TestGlm47Streaming:
153 "Beijing",
154 "</arg_value>",
155 "</tool_call>",
163 - ]:
156 + ]
157 + current_text = ""
158 + for chunk in chunks:
159 + current_text += chunk
160 glm47_tool_parser.extract_tool_calls_streaming(
161 previous_text="",
166 - current_text="",
162 + current_text=current_text,
163 delta_text=chunk,
164 previous_token_ids=[],
165 current_token_ids=[],
166 delta_token_ids=[],
167 request=mock_request,
168 )
173 - assert glm47_tool_parser.prev_tool_call_arr[0]["arguments"]["city"] == "Beijing"
169 + args = json.loads(glm47_tool_parser.prev_tool_call_arr[0]["arguments"])
170 + assert args["city"] == "Beijing"
tests/tool_parsers/test_glm4_moe_tool_parser.py
+509 -103
@@ -357,81 +357,69 @@ meaningwhile, I will also check the weather in Shanghai.
357
358 def test_streaming_basic_functionality(glm4_moe_tool_parser, mock_request):
359 """Test basic streaming functionality."""
360 - # Reset streaming state
361 - glm4_moe_tool_parser.current_tool_name_sent = False
362 - glm4_moe_tool_parser.prev_tool_call_arr = []
363 - glm4_moe_tool_parser.current_tool_id = -1
364 - glm4_moe_tool_parser.streamed_args_for_tool = []
360 + _reset_streaming_state(glm4_moe_tool_parser)
361
366 - # Test with a simple tool call
362 current_text = """<tool_call>get_weather
363 <arg_key>city</arg_key>
364 <arg_value>Beijing</arg_value>
365 </tool_call>"""
366
372 - # Mock token IDs for testing
373 - tool_call_start_id = glm4_moe_tool_parser.tool_call_start_token_id or 12345
374 - tool_call_end_id = glm4_moe_tool_parser.tool_call_end_token_id or 12346
375 -
367 result = glm4_moe_tool_parser.extract_tool_calls_streaming(
368 previous_text="",
369 current_text=current_text,
379 - delta_text="</tool_call>",
370 + delta_text=current_text,
371 previous_token_ids=[],
381 - current_token_ids=[tool_call_start_id, tool_call_end_id],
382 - delta_token_ids=[tool_call_end_id],
372 + current_token_ids=[],
373 + delta_token_ids=[],
374 request=mock_request,
375 )
376
386 - # The result behavior depends on the streaming state
387 - # This test mainly ensures no exceptions are thrown
388 - assert result is None or hasattr(result, "tool_calls") or hasattr(result, "content")
377 + # Should return tool call with name and arguments in one shot
378 + assert result is not None
379 + assert result.tool_calls is not None
380 + assert len(result.tool_calls) >= 1
381
382
383 def test_streaming_no_tool_calls(glm4_moe_tool_parser, mock_request):
384 """Test streaming when there are no tool calls."""
385 + _reset_streaming_state(glm4_moe_tool_parser)
386 +
387 current_text = "This is just regular text without any tool calls."
388
389 result = glm4_moe_tool_parser.extract_tool_calls_streaming(
396 - previous_text="This is just regular text",
390 + previous_text="",
391 current_text=current_text,
398 - delta_text=" without any tool calls.",
392 + delta_text=current_text,
393 previous_token_ids=[],
394 current_token_ids=[],
395 delta_token_ids=[],
396 request=mock_request,
397 )
398
405 - # Should return the delta text as content
399 + # Should return content
400 assert result is not None
407 - assert hasattr(result, "content")
408 - assert result.content == " without any tool calls."
401 + assert result.content == current_text
402
403
404 def test_streaming_with_content_before_tool_calls(glm4_moe_tool_parser, mock_request):
405 """Test streaming when there's content before tool calls."""
413 - # Reset streaming state
414 - glm4_moe_tool_parser.current_tool_name_sent = False
415 - glm4_moe_tool_parser.prev_tool_call_arr = []
416 - glm4_moe_tool_parser.current_tool_id = -1
417 - glm4_moe_tool_parser.streamed_args_for_tool = []
406 + _reset_streaming_state(glm4_moe_tool_parser)
407
419 - current_text = "I will help you get the weather<tool_call>"
408 + current_text = "I will help you get the weather.<tool_call>"
409
410 result = glm4_moe_tool_parser.extract_tool_calls_streaming(
422 - previous_text="I will help you",
411 + previous_text="",
412 current_text=current_text,
424 - delta_text="get the weather.<tool_call>",
413 + delta_text=current_text,
414 previous_token_ids=[],
415 current_token_ids=[],
416 delta_token_ids=[],
417 request=mock_request,
418 )
419
431 - # Should return content when no tool call tokens are detected
420 + # Should return content before the <tool_call> tag
421 assert result is not None
433 - assert hasattr(result, "content")
434 - assert result.content == "get the weather."
422 + assert result.content == "I will help you get the weather."
423
424
425 def test_extract_tool_calls_special_characters(glm4_moe_tool_parser, mock_request):
@@ -479,26 +467,19 @@ def test_extract_tool_calls_incomplete_tool_call(glm4_moe_tool_parser, mock_requ
467
468 def _reset_streaming_state(parser):
469 """Helper to reset parser streaming state."""
482 - parser._buffer = ""
483 - parser._in_tool_call = False
470 parser.current_tool_name_sent = False
485 - parser._current_tool_name = None
486 - parser._pending_key = None
487 - parser._streaming_string_value = False
471 parser.prev_tool_call_arr = []
472 parser.current_tool_id = -1
473 parser.streamed_args_for_tool = []
474 parser._tool_call_ids = []
492 - parser._args_started = []
493 - parser._args_closed = []
494 - parser._seen_keys = []
475 + parser._sent_content_idx = 0
476
477
478 def test_streaming_incremental_string_value(glm4_moe_tool_parser, mock_request):
479 """Test incremental streaming of string argument values."""
480 _reset_streaming_state(glm4_moe_tool_parser)
481
501 - # Simulate streaming a tool call character by character
482 + # Simulate streaming a tool call chunk by chunk
483 chunks = [
484 "<tool_call>",
485 "get_weather\n",
@@ -511,30 +492,31 @@ def test_streaming_incremental_string_value(glm4_moe_tool_parser, mock_request):
492 ]
493
494 collected_fragments = []
495 + current_text = ""
496 for chunk in chunks:
497 + current_text += chunk
498 result = glm4_moe_tool_parser.extract_tool_calls_streaming(
499 previous_text="",
517 - current_text="",
500 + current_text=current_text,
501 delta_text=chunk,
502 previous_token_ids=[],
503 current_token_ids=[],
504 delta_token_ids=[],
505 request=mock_request,
506 )
524 - if result is not None and hasattr(result, "tool_calls") and result.tool_calls:
507 + if result is not None and result.tool_calls:
508 for tc in result.tool_calls:
526 - if hasattr(tc, "function") and tc.function:
527 - func = tc.function
528 - if isinstance(func, dict):
529 - if func.get("arguments"):
530 - collected_fragments.append(func["arguments"])
531 - if func.get("name"):
532 - collected_fragments.append(f"name:{func['name']}")
533 - else:
534 - if func.arguments:
535 - collected_fragments.append(func.arguments)
536 - if func.name:
537 - collected_fragments.append(f"name:{func.name}")
509 + func = tc.function
510 + if isinstance(func, dict):
511 + if func.get("arguments"):
512 + collected_fragments.append(func["arguments"])
513 + if func.get("name"):
514 + collected_fragments.append(f"name:{func['name']}")
515 + else:
516 + if func.arguments:
517 + collected_fragments.append(func.arguments)
518 + if func.name:
519 + collected_fragments.append(f"name:{func.name}")
520
521 # Verify we got incremental streaming of the argument value
522 assert len(collected_fragments) > 0
@@ -547,11 +529,11 @@ def test_streaming_empty_tool_call(glm4_moe_tool_parser, mock_request):
529 """Test that empty tool calls don't cause infinite loops."""
530 _reset_streaming_state(glm4_moe_tool_parser)
531
550 - # Empty tool call should be handled gracefully
532 + current_text = "<tool_call></tool_call>"
533 result = glm4_moe_tool_parser.extract_tool_calls_streaming(
534 previous_text="",
553 - current_text="",
554 - delta_text="<tool_call></tool_call>",
535 + current_text=current_text,
536 + delta_text=current_text,
537 previous_token_ids=[],
538 current_token_ids=[],
539 delta_token_ids=[],
@@ -561,60 +543,52 @@ def test_streaming_empty_tool_call(glm4_moe_tool_parser, mock_request):
543 # Should not hang and should return something (None or content)
544 # The key is that this completes without hanging
545 assert result is None or hasattr(result, "content") or hasattr(result, "tool_calls")
564 - # State should be properly reset
565 - assert glm4_moe_tool_parser.current_tool_id == -1
546
547
548 def test_streaming_prev_tool_call_arr_updates(glm4_moe_tool_parser, mock_request):
569 - """Test that prev_tool_call_arr contains parsed dict after tool call."""
549 + """Test that prev_tool_call_arr is populated incrementally."""
550 _reset_streaming_state(glm4_moe_tool_parser)
551
572 - # Stream a complete tool call
573 - name_only = {"name": "get_weather", "arguments": {}}
574 - name_and_args = {"name": "get_weather", "arguments": {"city": "Beijing"}}
552 chunks = [
576 - # Delta, expected streamed_args_for_tool, expected prev_tool_call_arr
577 - ("<tool_call>get_weather\n", "", name_only),
578 - ("<arg_key>city</arg_key>", "", name_only),
579 - ("<arg_value>Beijing</arg_value>", '{"city": "Beijing"', name_only),
580 - # Note: arguments are only updated when the tool call is complete.
581 - ("</tool_call>", '{"city": "Beijing"}', name_and_args),
553 + "<tool_call>get_weather\n",
554 + "<arg_key>city</arg_key>",
555 + "<arg_value>Beijing</arg_value>",
556 + "</tool_call>",
557 ]
558
584 - for chunk, exp_streamed, exp_prev_tc in chunks:
559 + current_text = ""
560 + for chunk in chunks:
561 + current_text += chunk
562 glm4_moe_tool_parser.extract_tool_calls_streaming(
563 previous_text="",
587 - current_text="",
564 + current_text=current_text,
565 delta_text=chunk,
566 previous_token_ids=[],
567 current_token_ids=[],
568 delta_token_ids=[],
569 request=mock_request,
570 )
594 - assert glm4_moe_tool_parser.streamed_args_for_tool[0] == exp_streamed
595 - assert glm4_moe_tool_parser.prev_tool_call_arr[0] == exp_prev_tc
571
597 - # After the tool call completes, prev_tool_call_arr should have parsed dict
572 + # After the tool call completes, prev_tool_call_arr should be populated
573 assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1
574 tool_entry = glm4_moe_tool_parser.prev_tool_call_arr[0]
575 assert tool_entry.get("name") == "get_weather"
601 - # arguments should be a dict, not a string
602 - args = tool_entry.get("arguments")
603 - assert isinstance(args, dict), f"Expected dict, got {type(args)}"
604 - assert args.get("city") == "Beijing"
576
606 - # Test equivalence of prev_tool_call_arr and streamed_args_for_tool
607 - # Simulates logic in chat_completion/serving.py:chat_completion_stream_generator
608 - tool_call_json = json.dumps(tool_entry.get("arguments", {}))
609 - streamed_content = glm4_moe_tool_parser.streamed_args_for_tool[0]
610 - assert tool_call_json.startswith(streamed_content)
577 + # arguments is a JSON string in the re-parse approach
578 + args_str = tool_entry.get("arguments")
579 + assert isinstance(args_str, str), f"Expected str, got {type(args_str)}"
580 + parsed = json.loads(args_str)
581 + assert parsed["city"] == "Beijing"
582 +
583 + # streamed_args_for_tool should match prev_tool_call_arr arguments
584 + streamed = glm4_moe_tool_parser.streamed_args_for_tool[0]
585 + assert streamed == args_str
586
587
588 def test_streaming_multiple_tool_calls_sequential(glm4_moe_tool_parser, mock_request):
589 """Test streaming multiple sequential tool calls."""
590 _reset_streaming_state(glm4_moe_tool_parser)
591
617 - # Stream two tool calls
592 chunks = [
593 "<tool_call>get_weather\n",
594 "<arg_key>city</arg_key>",
@@ -626,10 +600,12 @@ def test_streaming_multiple_tool_calls_sequential(glm4_moe_tool_parser, mock_req
600 "</tool_call>",
601 ]
602
603 + current_text = ""
604 for chunk in chunks:
605 + current_text += chunk
606 glm4_moe_tool_parser.extract_tool_calls_streaming(
607 previous_text="",
632 - current_text="",
608 + current_text=current_text,
609 delta_text=chunk,
610 previous_token_ids=[],
611 current_token_ids=[],
@@ -639,15 +615,16 @@ def test_streaming_multiple_tool_calls_sequential(glm4_moe_tool_parser, mock_req
615
616 # Should have two tool calls in prev_tool_call_arr
617 assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 2
642 - assert glm4_moe_tool_parser.prev_tool_call_arr[0]["arguments"]["city"] == "Beijing"
643 - assert glm4_moe_tool_parser.prev_tool_call_arr[1]["arguments"]["city"] == "Shanghai"
618 + args0 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[0]["arguments"])
619 + args1 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[1]["arguments"])
620 + assert args0["city"] == "Beijing"
621 + assert args1["city"] == "Shanghai"
622
623
624 def test_streaming_json_escape_in_string(glm4_moe_tool_parser, mock_request):
625 """Test that special characters in string values are properly escaped."""
626 _reset_streaming_state(glm4_moe_tool_parser)
627
650 - # String with characters that need JSON escaping
628 chunks = [
629 "<tool_call>send_message\n",
630 "<arg_key>message</arg_key>",
@@ -655,10 +632,12 @@ def test_streaming_json_escape_in_string(glm4_moe_tool_parser, mock_request):
632 "</tool_call>",
633 ]
634
635 + current_text = ""
636 for chunk in chunks:
637 + current_text += chunk
638 glm4_moe_tool_parser.extract_tool_calls_streaming(
639 previous_text="",
661 - current_text="",
640 + current_text=current_text,
641 delta_text=chunk,
642 previous_token_ids=[],
643 current_token_ids=[],
@@ -669,10 +648,8 @@ def test_streaming_json_escape_in_string(glm4_moe_tool_parser, mock_request):
648 # The streamed_args_for_tool should contain valid JSON
649 assert len(glm4_moe_tool_parser.streamed_args_for_tool) == 1
650 args_json = glm4_moe_tool_parser.streamed_args_for_tool[0]
672 - # Should be parseable as JSON
651 parsed = json.loads(args_json)
652 assert "message" in parsed
675 - # The value should preserve the special characters
653 assert '"' in parsed["message"] or "world" in parsed["message"]
654
655
@@ -749,27 +726,27 @@ if __name__ == "__main__":
726
727 # Count argument fragments
728 fragment_count = 0
729 + current_text = ""
730 for chunk in chunks:
731 + current_text += chunk
732 result = glm4_moe_tool_parser.extract_tool_calls_streaming(
733 previous_text="",
755 - current_text="",
734 + current_text=current_text,
735 delta_text=chunk,
736 previous_token_ids=[],
737 current_token_ids=[],
738 delta_token_ids=[],
739 request=request,
740 )
762 - if result is not None and hasattr(result, "tool_calls") and result.tool_calls:
741 + if result is not None and result.tool_calls:
742 for tc in result.tool_calls:
764 - if hasattr(tc, "function") and tc.function:
765 - func = tc.function
766 - args = (
767 - func.get("arguments")
768 - if isinstance(func, dict)
769 - else getattr(func, "arguments", None)
770 - )
771 - if args:
772 - fragment_count += 1
743 + func = tc.function
744 + if isinstance(func, dict):
745 + args = func.get("arguments")
746 + else:
747 + args = getattr(func, "arguments", None)
748 + if args:
749 + fragment_count += 1
750
751 # For true incremental streaming, we expect many fragments (10+)
752 # Old buffered implementation would give only 1-3 fragments
@@ -927,3 +904,432 @@ def test_unicode_characters_preserved(glm4_moe_tool_parser, mock_request):
904 parsed_args = json.loads(raw_args)
905 assert parsed_args["greeting"] == "你好世界"
906 assert parsed_args["emoji"] == "🎉"
907 +
908 +
909 +def test_streaming_multi_token_chunks(glm4_moe_tool_parser, mock_request):
910 + """Test that multi-token chunks (stream_interval > 1) are handled correctly.
911 +
912 + With stream_interval > 1 or MTP, multiple XML tags arrive in one delta.
913 + The old buffer-based parser could only return one delta per call, losing
914 + data on the final output. The re-parse approach handles this correctly.
915 + """
916 + _reset_streaming_state(glm4_moe_tool_parser)
917 +
918 + # Simulate stream_interval=3: chunks contain multiple XML tags
919 + chunks = [
920 + "<tool_call>get_weather\n<arg_key>city</arg_key><arg_value>Bei",
921 + "jing</arg_value>",
922 + "</tool_call>",
923 + ]
924 +
925 + current_text = ""
926 + for chunk in chunks:
927 + current_text += chunk
928 + glm4_moe_tool_parser.extract_tool_calls_streaming(
929 + previous_text="",
930 + current_text=current_text,
931 + delta_text=chunk,
932 + previous_token_ids=[],
933 + current_token_ids=[],
934 + delta_token_ids=[],
935 + request=mock_request,
936 + )
937 +
938 + # All data should be captured despite multi-token chunks
939 + assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1
940 + args = json.loads(glm4_moe_tool_parser.streamed_args_for_tool[0])
941 + assert args["city"] == "Beijing"
942 +
943 +
944 +def test_streaming_entire_tool_call_at_once(glm4_moe_tool_parser, mock_request):
945 + """Test that a complete tool call arriving in one delta works.
946 +
947 + This simulates the extreme MTP case where all tokens arrive at once.
948 + """
949 + _reset_streaming_state(glm4_moe_tool_parser)
950 +
951 + full_text = (
952 + "<tool_call>get_weather\n"
953 + "<arg_key>city</arg_key>"
954 + "<arg_value>Beijing</arg_value>"
955 + "</tool_call>"
956 + )
957 +
958 + result = glm4_moe_tool_parser.extract_tool_calls_streaming(
959 + previous_text="",
960 + current_text=full_text,
961 + delta_text=full_text,
962 + previous_token_ids=[],
963 + current_token_ids=[],
964 + delta_token_ids=[],
965 + request=mock_request,
966 + )
967 +
968 + # Should emit tool call with complete arguments in one shot
969 + assert result is not None
970 + assert result.tool_calls is not None
971 +
972 + # Verify final state
973 + assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1
974 + args = json.loads(glm4_moe_tool_parser.streamed_args_for_tool[0])
975 + assert args["city"] == "Beijing"
976 +
977 +
978 +def test_streaming_content_between_tool_calls_multi_token(
979 + glm4_moe_tool_parser, mock_request
980 +):
981 + """Test content between tool calls with multi-token chunks."""
982 + _reset_streaming_state(glm4_moe_tool_parser)
983 +
984 + # Deliver everything at once — worst case for the old buffer parser
985 + full_text = (
986 + "I will check.\n"
987 + "<tool_call>get_weather\n"
988 + "<arg_key>city</arg_key>"
989 + "<arg_value>Beijing</arg_value>"
990 + "</tool_call>"
991 + "\nAlso Shanghai.\n"
992 + "<tool_call>get_weather\n"
993 + "<arg_key>city</arg_key>"
994 + "<arg_value>Shanghai</arg_value>"
995 + "</tool_call>"
996 + )
997 +
998 + # First call with partial text (content only)
999 + partial = "I will check.\n"
1000 + result1 = glm4_moe_tool_parser.extract_tool_calls_streaming(
1001 + previous_text="",
1002 + current_text=partial,
1003 + delta_text=partial,
1004 + previous_token_ids=[],
1005 + current_token_ids=[],
1006 + delta_token_ids=[],
1007 + request=mock_request,
1008 + )
1009 + assert result1 is not None
1010 + assert result1.content == "I will check.\n"
1011 +
1012 + # Second call with everything
1013 + glm4_moe_tool_parser.extract_tool_calls_streaming(
1014 + previous_text="",
1015 + current_text=full_text,
1016 + delta_text=full_text[len(partial) :],
1017 + previous_token_ids=[],
1018 + current_token_ids=[],
1019 + delta_token_ids=[],
1020 + request=mock_request,
1021 + )
1022 +
1023 + # Should have both tool calls
1024 + assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 2
1025 + args0 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[0]["arguments"])
1026 + args1 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[1]["arguments"])
1027 + assert args0["city"] == "Beijing"
1028 + assert args1["city"] == "Shanghai"
1029 +
1030 +
1031 +def test_streaming_multi_token_with_multiple_args(glm4_moe_tokenizer):
1032 + """Test multi-token streaming with multiple arguments of mixed types."""
1033 + tools = [
1034 + ChatCompletionToolsParam(
1035 + function=FunctionDefinition(
1036 + name="calculate",
1037 + parameters={
1038 + "type": "object",
1039 + "properties": {
1040 + "operation": {"type": "string"},
1041 + "a": {"type": "number"},
1042 + "b": {"type": "number"},
1043 + },
1044 + },
1045 + ),
1046 + ),
1047 + ]
1048 + parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1049 + request = ChatCompletionRequest(
1050 + model=MODEL,
1051 + messages=[],
1052 + tools=tools,
1053 + )
1054 +
1055 + # All arguments arrive in two big chunks (simulates stream_interval=5)
1056 + chunks = [
1057 + "<tool_call>calculate\n<arg_key>operation</arg_key><arg_value>add</arg_value><arg_key>a</arg_key>",
1058 + "<arg_value>42</arg_value><arg_key>b</arg_key><arg_value>3.14</arg_value></tool_call>",
1059 + ]
1060 +
1061 + current_text = ""
1062 + for chunk in chunks:
1063 + current_text += chunk
1064 + parser.extract_tool_calls_streaming(
1065 + previous_text="",
1066 + current_text=current_text,
1067 + delta_text=chunk,
1068 + previous_token_ids=[],
1069 + current_token_ids=[],
1070 + delta_token_ids=[],
1071 + request=request,
1072 + )
1073 +
1074 + args = json.loads(parser.streamed_args_for_tool[0])
1075 + assert args["operation"] == "add"
1076 + assert args["a"] == 42
1077 + assert args["b"] == 3.14
1078 +
1079 +
1080 +def _simulate_streaming(tokenizer, parser, request, text, stream_interval=1):
1081 + """Simulate streaming with a given stream_interval.
1082 +
1083 + Tokens are batched into chunks of ``stream_interval`` tokens,
1084 + mimicking how the output processor delivers them.
1085 + Returns a list of non-None DeltaMessages.
1086 + """
1087 + tokens = tokenizer.encode(text)
1088 + previous_text = ""
1089 + deltas = []
1090 + for i in range(0, len(tokens), stream_interval):
1091 + chunk_ids = tokens[i : i + stream_interval]
1092 + delta_text = tokenizer.decode(chunk_ids)
1093 + current_text = previous_text + delta_text
1094 + delta = parser.extract_tool_calls_streaming(
1095 + previous_text=previous_text,
1096 + current_text=current_text,
1097 + delta_text=delta_text,
1098 + previous_token_ids=[],
1099 + current_token_ids=[],
1100 + delta_token_ids=chunk_ids,
1101 + request=request,
1102 + )
1103 + previous_text = current_text
1104 + if delta is not None:
1105 + deltas.append(delta)
1106 + return deltas
1107 +
1108 +
1109 +def _collect_from_deltas(deltas):
1110 + """Reconstruct tool call names/args and content from a delta stream."""
1111 + tools: dict[int, dict] = {}
1112 + content_parts: list[str] = []
1113 + for d in deltas:
1114 + if d.content:
1115 + content_parts.append(d.content)
1116 + if d.tool_calls:
1117 + for tc in d.tool_calls:
1118 + func = tc.function
1119 + if isinstance(func, dict):
1120 + name = func.get("name")
1121 + args = func.get("arguments")
1122 + else:
1123 + name = getattr(func, "name", None)
1124 + args = getattr(func, "arguments", None)
1125 + idx = tc.index
1126 + if idx not in tools:
1127 + tools[idx] = {"name": None, "args_fragments": []}
1128 + if name:
1129 + tools[idx]["name"] = name
1130 + if args:
1131 + tools[idx]["args_fragments"].append(args)
1132 + return content_parts, tools
1133 +
1134 +
1135 +@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
1136 +def test_stream_interval_single_tool_call(glm4_moe_tokenizer, stream_interval):
1137 + """Tool call streaming produces correct name + args at any interval."""
1138 + tools = [
1139 + ChatCompletionToolsParam(
1140 + function=FunctionDefinition(
1141 + name="get_weather",
1142 + parameters={
1143 + "type": "object",
1144 + "properties": {"city": {"type": "string"}},
1145 + },
1146 + ),
1147 + ),
1148 + ]
1149 + parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1150 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1151 +
1152 + text = (
1153 + "<tool_call>get_weather\n"
1154 + "<arg_key>city</arg_key>"
1155 + "<arg_value>Beijing</arg_value>"
1156 + "</tool_call>"
1157 + )
1158 +
1159 + deltas = _simulate_streaming(
1160 + glm4_moe_tokenizer, parser, request, text, stream_interval
1161 + )
1162 + _, tools_found = _collect_from_deltas(deltas)
1163 +
1164 + assert 0 in tools_found
1165 + assert tools_found[0]["name"] == "get_weather"
1166 + args_json = "".join(tools_found[0]["args_fragments"])
1167 + parsed = json.loads(args_json)
1168 + assert parsed == {"city": "Beijing"}
1169 +
1170 +
1171 +@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
1172 +def test_stream_interval_multiple_tool_calls(glm4_moe_tokenizer, stream_interval):
1173 + """Multiple sequential tool calls with correct indices at any interval."""
1174 + tools = [
1175 + ChatCompletionToolsParam(
1176 + function=FunctionDefinition(
1177 + name="get_weather",
1178 + parameters={
1179 + "type": "object",
1180 + "properties": {"city": {"type": "string"}},
1181 + },
1182 + ),
1183 + ),
1184 + ]
1185 + parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1186 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1187 +
1188 + text = (
1189 + "<tool_call>get_weather\n"
1190 + "<arg_key>city</arg_key>"
1191 + "<arg_value>Beijing</arg_value>"
1192 + "</tool_call>"
1193 + "<tool_call>get_weather\n"
1194 + "<arg_key>city</arg_key>"
1195 + "<arg_value>Shanghai</arg_value>"
1196 + "</tool_call>"
1197 + )
1198 +
1199 + deltas = _simulate_streaming(
1200 + glm4_moe_tokenizer, parser, request, text, stream_interval
1201 + )
1202 + _, tools_found = _collect_from_deltas(deltas)
1203 +
1204 + assert 0 in tools_found and 1 in tools_found
1205 + args0 = json.loads("".join(tools_found[0]["args_fragments"]))
1206 + args1 = json.loads("".join(tools_found[1]["args_fragments"]))
1207 + assert args0 == {"city": "Beijing"}
1208 + assert args1 == {"city": "Shanghai"}
1209 +
1210 +
1211 +@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
1212 +def test_stream_interval_content_then_tool_call(glm4_moe_tokenizer, stream_interval):
1213 + """Content before a tool call is fully emitted before tool deltas."""
1214 + tools = [
1215 + ChatCompletionToolsParam(
1216 + function=FunctionDefinition(
1217 + name="get_weather",
1218 + parameters={
1219 + "type": "object",
1220 + "properties": {"city": {"type": "string"}},
1221 + },
1222 + ),
1223 + ),
1224 + ]
1225 + parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1226 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1227 +
1228 + text = (
1229 + "I will check the weather for you.\n"
1230 + "<tool_call>get_weather\n"
1231 + "<arg_key>city</arg_key>"
1232 + "<arg_value>Beijing</arg_value>"
1233 + "</tool_call>"
1234 + )
1235 +
1236 + deltas = _simulate_streaming(
1237 + glm4_moe_tokenizer, parser, request, text, stream_interval
1238 + )
1239 + content_parts, tools_found = _collect_from_deltas(deltas)
1240 +
1241 + # Content must be present and precede tool calls
1242 + full_content = "".join(content_parts)
1243 + assert "I will check the weather" in full_content
1244 +
1245 + # Tool call must be correct
1246 + assert 0 in tools_found
1247 + assert tools_found[0]["name"] == "get_weather"
1248 + args = json.loads("".join(tools_found[0]["args_fragments"]))
1249 + assert args == {"city": "Beijing"}
1250 +
1251 +
1252 +def test_stream_interval_extreme_single_chunk(glm4_moe_tokenizer):
1253 + """Extreme MTP: entire output arrives in one chunk (interval=9999)."""
1254 + tools = [
1255 + ChatCompletionToolsParam(
1256 + function=FunctionDefinition(
1257 + name="get_weather",
1258 + parameters={
1259 + "type": "object",
1260 + "properties": {"city": {"type": "string"}},
1261 + },
1262 + ),
1263 + ),
1264 + ]
1265 + parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1266 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1267 +
1268 + text = (
1269 + "Here is the weather.\n"
1270 + "<tool_call>get_weather\n"
1271 + "<arg_key>city</arg_key>"
1272 + "<arg_value>Beijing</arg_value>"
1273 + "</tool_call>"
1274 + )
1275 +
1276 + deltas = _simulate_streaming(
1277 + glm4_moe_tokenizer, parser, request, text, stream_interval=9999
1278 + )
1279 + content_parts, tools_found = _collect_from_deltas(deltas)
1280 +
1281 + assert "Here is the weather" in "".join(content_parts)
1282 + assert 0 in tools_found
1283 + assert tools_found[0]["name"] == "get_weather"
1284 + args = json.loads("".join(tools_found[0]["args_fragments"]))
1285 + assert args == {"city": "Beijing"}
1286 +
1287 +
1288 +@pytest.mark.parametrize("stream_interval", [1, 2, 5])
1289 +def test_stream_interval_content_between_tool_calls(
1290 + glm4_moe_tokenizer, stream_interval
1291 +):
1292 + """Content between tool calls must be emitted, not silently dropped."""
1293 + tools = [
1294 + ChatCompletionToolsParam(
1295 + function=FunctionDefinition(
1296 + name="get_weather",
1297 + parameters={
1298 + "type": "object",
1299 + "properties": {"city": {"type": "string"}},
1300 + },
1301 + ),
1302 + ),
1303 + ]
1304 + parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools)
1305 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
1306 +
1307 + text = (
1308 + "Checking Beijing.\n"
1309 + "<tool_call>get_weather\n"
1310 + "<arg_key>city</arg_key>"
1311 + "<arg_value>Beijing</arg_value>"
1312 + "</tool_call>"
1313 + "\nAlso Shanghai.\n"
1314 + "<tool_call>get_weather\n"
1315 + "<arg_key>city</arg_key>"
1316 + "<arg_value>Shanghai</arg_value>"
1317 + "</tool_call>"
1318 + )
1319 +
1320 + deltas = _simulate_streaming(
1321 + glm4_moe_tokenizer, parser, request, text, stream_interval
1322 + )
1323 + content_parts, tools_found = _collect_from_deltas(deltas)
1324 +
1325 + full_content = "".join(content_parts)
1326 + # Both prefix and inter-tool-call content must appear
1327 + assert "Checking Beijing" in full_content
1328 + assert "Also Shanghai" in full_content
1329 +
1330 + # Both tool calls must be correct
1331 + assert 0 in tools_found and 1 in tools_found
1332 + args0 = json.loads("".join(tools_found[0]["args_fragments"]))
1333 + args1 = json.loads("".join(tools_found[1]["args_fragments"]))
1334 + assert args0 == {"city": "Beijing"}
1335 + assert args1 == {"city": "Shanghai"}
vllm/tool_parsers/glm4_moe_tool_parser.py
+252 -296
@@ -37,6 +37,7 @@ from vllm.tool_parsers.abstract_tool_parser import (
37 Tool,
38 ToolParser,
39 )
40 +from vllm.tool_parsers.utils import partial_tag_overlap
41
42 logger = init_logger(__name__)
43
@@ -44,9 +45,9 @@ logger = init_logger(__name__)
45 class Glm4MoeModelToolParser(ToolParser):
46 """Tool parser for GLM-4 models with incremental string streaming.
47
47 - This parser emits tool-call deltas incrementally as arguments arrive.
48 - For string-type parameters, content is streamed character-by-character
49 - rather than waiting for the complete </arg_value> tag.
48 + On every streaming call the parser re-parses ``current_text`` to find
49 + ``<tool_call>`` regions, builds the JSON arguments string for each tool
50 + call, and diffs against what was previously sent to emit only new content.
51 """
52
53 def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
@@ -82,17 +83,17 @@ class Glm4MoeModelToolParser(ToolParser):
83
84 self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
85 self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
85 - self._buffer: str = ""
86
87 - # Streaming state for incremental tool-call streaming
88 - self._in_tool_call: bool = False
89 - self._current_tool_name: str | None = None
90 - self._pending_key: str | None = None
91 - self._streaming_string_value: bool = False
87 + # Pre-compiled pattern for finding the last <arg_key>...</arg_key>
88 + # before a partial <arg_value> (used in _build_args_json_so_far).
89 + self._arg_key_pattern = re.compile(
90 + re.escape(self.arg_key_start) + r"(.*?)" + re.escape(self.arg_key_end),
91 + re.DOTALL,
92 + )
93 +
94 + # Streaming state for re-parse-and-diff approach
95 + self._sent_content_idx: int = 0
96 self._tool_call_ids: list[str] = []
93 - self._args_started: list[bool] = []
94 - self._args_closed: list[bool] = []
95 - self._seen_keys: list[set[str]] = []
97
98 @staticmethod
99 def _deserialize(value: str) -> Any:
@@ -222,6 +223,199 @@ class Glm4MoeModelToolParser(ToolParser):
223 tools_called=False, tool_calls=[], content=model_output
224 )
225
226 + def _extract_content(self, current_text: str) -> str | None:
227 + """Return unsent non-tool-call text, or None.
228 +
229 + Collects all text outside ``<tool_call>...</tool_call>`` regions,
230 + including text between consecutive tool calls. Holds back any
231 + suffix that could be a partial ``<tool_call>`` tag.
232 + """
233 + # Build the "sendable index" — the furthest point we can send
234 + # content up to. We scan through the text collecting segments
235 + # that are outside tool-call regions.
236 + content_segments: list[str] = []
237 + pos = self._sent_content_idx
238 +
239 + while pos < len(current_text):
240 + start = current_text.find(self.tool_call_start_token, pos)
241 + if start == -1:
242 + # No more tool calls — send up to (len - partial-tag overlap)
243 + tail = current_text[pos:]
244 + overlap = partial_tag_overlap(tail, self.tool_call_start_token)
245 + sendable = tail[: len(tail) - overlap] if overlap else tail
246 + if sendable:
247 + content_segments.append(sendable)
248 + pos = len(current_text) - overlap
249 + break
250 +
251 + # Text before this <tool_call>
252 + if start > pos:
253 + content_segments.append(current_text[pos:start])
254 +
255 + # Skip past the </tool_call> (or to end if incomplete)
256 + end = current_text.find(self.tool_call_end_token, start)
257 + if end != -1:
258 + pos = end + len(self.tool_call_end_token)
259 + else:
260 + # Incomplete tool call — nothing more to send
261 + pos = start
262 + break
263 +
264 + if content_segments:
265 + self._sent_content_idx = pos
266 + return "".join(content_segments)
267 + # Even if no content, advance past completed tool-call regions
268 + if pos > self._sent_content_idx:
269 + self._sent_content_idx = pos
270 + return None
271 +
272 + def _extract_tool_call_regions(self, text: str) -> list[tuple[str, bool]]:
273 + """Extract ``(inner_text, is_complete)`` for each ``<tool_call>`` region."""
274 + results: list[tuple[str, bool]] = []
275 + pos = 0
276 + while True:
277 + start = text.find(self.tool_call_start_token, pos)
278 + if start == -1:
279 + break
280 + inner_start = start + len(self.tool_call_start_token)
281 + end = text.find(self.tool_call_end_token, inner_start)
282 + if end != -1:
283 + results.append((text[inner_start:end], True))
284 + pos = end + len(self.tool_call_end_token)
285 + else:
286 + # Incomplete tool call — strip partial </tool_call> suffix
287 + raw = text[inner_start:]
288 + overlap = partial_tag_overlap(raw, self.tool_call_end_token)
289 + if overlap:
290 + raw = raw[:-overlap]
291 + results.append((raw, False))
292 + break
293 + return results
294 +
295 + def _extract_tool_name_from_region(self, inner_text: str) -> str | None:
296 + """Extract the tool name from the beginning of a tool-call region.
297 +
298 + The name is everything before the first ``\\n`` or ``<arg_key>``.
299 + Returns ``None`` if the name hasn't fully arrived yet.
300 + """
301 + nl = inner_text.find("\n")
302 + ak = inner_text.find(self.arg_key_start)
303 + candidates = [i for i in [nl, ak] if i != -1]
304 + if not candidates:
305 + return None
306 + cut = min(candidates)
307 + name = inner_text[:cut].strip()
308 + return name if name else None
309 +
310 + def _build_args_json_so_far(
311 + self,
312 + tool_name: str,
313 + inner_text: str,
314 + is_complete: bool,
315 + ) -> str:
316 + """Build the JSON arguments string from the XML pairs seen so far.
317 +
318 + For complete ``<arg_key>/<arg_value>`` pairs the value is fully
319 + formatted. For the last argument whose ``<arg_value>`` has been
320 + opened but not closed, the partial string content is included
321 + (JSON-escaped, with an opening ``"`` but no closing ``"``).
322 +
323 + The closing ``}`` is only appended when ``is_complete`` is True
324 + (i.e. the ``</tool_call>`` tag has arrived).
325 + """
326 + # Find all complete arg pairs
327 + pairs = self.func_arg_regex.findall(inner_text)
328 +
329 + parts: list[str] = []
330 + for key, value in pairs:
331 + key = key.strip()
332 + key_json = json.dumps(key, ensure_ascii=False)
333 + if self._is_string_type(tool_name, key, self.tools):
334 + # Don't strip string values — whitespace is significant
335 + # and must match the partial-value path for diffing.
336 + val_json = json.dumps(value, ensure_ascii=False)
337 + else:
338 + val_json = json.dumps(
339 + self._deserialize(value.strip()), ensure_ascii=False
340 + )
341 + parts.append(f"{key_json}: {val_json}")
342 +
343 + # Check for a partial (incomplete) arg value
344 + # Find the last <arg_value> that isn't closed
345 + last_val_start = inner_text.rfind(self.arg_val_start)
346 + last_val_end = inner_text.rfind(self.arg_val_end)
347 + has_partial_value = last_val_start != -1 and (
348 + last_val_end == -1 or last_val_end < last_val_start
349 + )
350 +
351 + if has_partial_value:
352 + # Find the key for this partial value
353 + # Look for the last <arg_key>...</arg_key> before this <arg_value>
354 + last_key_match = None
355 + for m in self._arg_key_pattern.finditer(inner_text[:last_val_start]):
356 + last_key_match = m
357 +
358 + if last_key_match:
359 + partial_key = last_key_match.group(1).strip()
360 + partial_content_start = last_val_start + len(self.arg_val_start)
361 + partial_content = inner_text[partial_content_start:]
362 +
363 + # Hold back any partial </arg_value> suffix
364 + overlap = partial_tag_overlap(partial_content, self.arg_val_end)
365 + if overlap:
366 + partial_content = partial_content[:-overlap]
367 +
368 + key_json = json.dumps(partial_key, ensure_ascii=False)
369 + if is_complete:
370 + # Tool call finished but </arg_value> is missing
371 + # (malformed output). Treat partial as complete value
372 + # so the diff naturally closes any open quotes.
373 + if self._is_string_type(tool_name, partial_key, self.tools):
374 + val_json = json.dumps(partial_content, ensure_ascii=False)
375 + else:
376 + val_json = json.dumps(
377 + self._deserialize(partial_content.strip()),
378 + ensure_ascii=False,
379 + )
380 + parts.append(f"{key_json}: {val_json}")
381 + elif self._is_string_type(tool_name, partial_key, self.tools):
382 + escaped = self._json_escape_string_content(partial_content)
383 + # Open quote but no close — more content may arrive
384 + parts.append(f'{key_json}: "{escaped}')
385 + else:
386 + # Non-string partial: include raw content, no wrapping
387 + parts.append(f"{key_json}: {partial_content}")
388 +
389 + if not parts:
390 + return "{}" if is_complete else ""
391 +
392 + joined = "{" + ", ".join(parts)
393 + if is_complete:
394 + joined += "}"
395 + return joined
396 +
397 + def _compute_args_diff(self, index: int, args_so_far: str) -> str | None:
398 + """Return new argument text not yet sent for tool *index*, or None."""
399 + if not args_so_far or len(args_so_far) <= len(
400 + self.streamed_args_for_tool[index]
401 + ):
402 + return None
403 + diff = args_so_far[len(self.streamed_args_for_tool[index]) :]
404 + self.streamed_args_for_tool[index] = args_so_far
405 + self.prev_tool_call_arr[index]["arguments"] = args_so_far
406 + return diff
407 +
408 + def _ensure_tool_state_for(self, index: int) -> None:
409 + """Grow state arrays so that *index* is valid."""
410 + while len(self._tool_call_ids) <= index:
411 + self._tool_call_ids.append(
412 + make_tool_call_id(id_type="random", func_name=None, idx=None)
413 + )
414 + while len(self.streamed_args_for_tool) <= index:
415 + self.streamed_args_for_tool.append("")
416 + while len(self.prev_tool_call_arr) <= index:
417 + self.prev_tool_call_arr.append({})
418 +
419 def extract_tool_calls_streaming(
420 self,
421 previous_text: str,
@@ -235,293 +429,55 @@ class Glm4MoeModelToolParser(ToolParser):
429 if not self._tools_enabled(request):
430 return DeltaMessage(content=delta_text) if delta_text else None
431
238 - self._buffer += delta_text
239 -
240 - while True:
241 - if not self._in_tool_call:
242 - start_idx = self._buffer.find(self.tool_call_start_token)
243 - if start_idx == -1:
244 - # Check for partial start token at end of buffer
245 - for i in range(1, len(self.tool_call_start_token)):
246 - if self._buffer.endswith(self.tool_call_start_token[:i]):
247 - out = self._buffer[:-i]
248 - self._buffer = self._buffer[-i:]
249 - return DeltaMessage(content=out) if out else None
250 - out = self._buffer
251 - self._buffer = ""
252 - return DeltaMessage(content=out) if out else None
253 -
254 - if start_idx > 0:
255 - out = self._buffer[:start_idx]
256 - self._buffer = self._buffer[start_idx:]
257 - return DeltaMessage(content=out) if out else None
258 -
259 - self._buffer = self._buffer[len(self.tool_call_start_token) :]
260 - self._begin_tool_call()
261 - continue
262 -
263 - # Parse tool name first
264 - if not self.current_tool_name_sent:
265 - nl = self._buffer.find("\n")
266 - ak = self._buffer.find(self.arg_key_start)
267 - end = self._buffer.find(self.tool_call_end_token)
268 - candidates = [i for i in [nl, ak, end] if i != -1]
269 - if not candidates:
270 - return None
271 - cut = min(candidates)
272 - tool_name = self._buffer[:cut].strip()
273 - if tool_name == "" and cut == end:
274 - # Handle empty tool call like `<tool_call></tool_call>`.
275 - # Consume the tokens and reset state to avoid infinite loop.
276 - self._buffer = self._buffer[end + len(self.tool_call_end_token) :]
277 - self._finish_tool_call()
278 - self._revert_last_tool_call_state()
279 - continue
280 -
281 - if cut == nl:
282 - self._buffer = self._buffer[nl + 1 :]
283 - else:
284 - self._buffer = self._buffer[cut:]
285 -
286 - self._current_tool_name = tool_name
287 - self.current_tool_name_sent = True
288 - return self._emit_tool_name_delta(tool_name)
289 -
290 - assert self._current_tool_name is not None
291 -
292 - # Handle incremental string value streaming
293 - if self._streaming_string_value:
294 - val_end = self._buffer.find(self.arg_val_end)
295 - if val_end != -1:
296 - raw_content = self._buffer[:val_end]
297 - self._buffer = self._buffer[val_end + len(self.arg_val_end) :]
298 - self._streaming_string_value = False
299 - self._pending_key = None
300 -
301 - escaped = self._json_escape_string_content(raw_content)
302 - frag = escaped + '"'
303 - self.streamed_args_for_tool[self.current_tool_id] += frag
304 - return self._emit_tool_args_delta(frag)
305 - else:
306 - # Check for partial </arg_value> at end
307 - safe_len = len(self._buffer)
308 - for i in range(1, len(self.arg_val_end)):
309 - if self._buffer.endswith(self.arg_val_end[:i]):
310 - safe_len = len(self._buffer) - i
311 - break
312 -
313 - if safe_len > 0:
314 - to_emit = self._buffer[:safe_len]
315 - self._buffer = self._buffer[safe_len:]
316 - escaped = self._json_escape_string_content(to_emit)
317 - if escaped:
318 - self.streamed_args_for_tool[self.current_tool_id] += escaped
319 - return self._emit_tool_args_delta(escaped)
320 - return None
321 -
322 - # If we have a pending key, parse its value
323 - if self._pending_key is not None:
324 - val_pos = self._buffer.find(self.arg_val_start)
325 - if val_pos == -1:
326 - return None
327 - if val_pos > 0:
328 - self._buffer = self._buffer[val_pos:]
329 -
330 - key = (self._pending_key or "").strip()
331 -
332 - is_string = self._is_string_type(
333 - self._current_tool_name, key, self.tools
334 - )
335 -
336 - if is_string:
337 - # String type: stream incrementally
338 - self._buffer = self._buffer[len(self.arg_val_start) :]
339 -
340 - if key in self._seen_keys[self.current_tool_id]:
341 - self._pending_key = None
342 - continue
343 -
344 - self._seen_keys[self.current_tool_id].add(key)
345 - key_json = json.dumps(key, ensure_ascii=False)
346 -
347 - if not self._args_started[self.current_tool_id]:
348 - frag = "{" + key_json + ': "'
349 - self._args_started[self.current_tool_id] = True
350 - else:
351 - frag = ", " + key_json + ': "'
352 -
353 - self.streamed_args_for_tool[self.current_tool_id] += frag
354 - self._streaming_string_value = True
355 - return self._emit_tool_args_delta(frag)
356 - else:
357 - # Non-string type: wait for complete value
358 - val_end = self._buffer.find(self.arg_val_end)
359 - if val_end == -1:
360 - return None
361 -
362 - raw_val = self._buffer[len(self.arg_val_start) : val_end].strip()
363 - self._buffer = self._buffer[val_end + len(self.arg_val_end) :]
364 - self._pending_key = None
365 -
366 - frag_or_none = self._append_arg_fragment(key=key, raw_val=raw_val)
367 - if frag_or_none:
368 - return self._emit_tool_args_delta(frag_or_none)
369 - continue
370 -
371 - # Parse next arg or close
372 - end_pos = self._buffer.find(self.tool_call_end_token)
373 - key_pos = self._buffer.find(self.arg_key_start)
374 - if end_pos != -1 and (key_pos == -1 or end_pos < key_pos):
375 - self._buffer = self._buffer[end_pos + len(self.tool_call_end_token) :]
376 - frag_or_none = self._close_args_if_needed()
377 - # Finalize prev_tool_call_arr with complete parsed arguments
378 - if self._current_tool_name:
379 - try:
380 - full_args_str = self.streamed_args_for_tool[
381 - self.current_tool_id
382 - ]
383 - args_dict = json.loads(full_args_str)
384 - self.prev_tool_call_arr[self.current_tool_id] = {
385 - "name": self._current_tool_name,
386 - "arguments": args_dict,
387 - }
388 - except (json.JSONDecodeError, IndexError) as e:
389 - logger.warning(
390 - "Failed to finalize tool call state for tool %d: %s",
391 - self.current_tool_id,
392 - e,
393 - )
394 - self._finish_tool_call()
395 - return (
396 - self._emit_tool_args_delta(frag_or_none) if frag_or_none else None
432 + content = self._extract_content(current_text)
433 + regions = self._extract_tool_call_regions(current_text)
434 + tool_call_deltas: list[DeltaToolCall] = []
435 +
436 + for i, (inner_text, is_complete) in enumerate(regions):
437 + self._ensure_tool_state_for(i)
438 +
439 + # Extract tool name
440 + tool_name = self._extract_tool_name_from_region(inner_text)
441 + if not tool_name:
442 + break
443 +
444 + # Emit tool name (once per tool call)
445 + if "name" not in self.prev_tool_call_arr[i]:
446 + self.prev_tool_call_arr[i]["name"] = tool_name
447 + tool_call_deltas.append(
448 + DeltaToolCall(
449 + index=i,
450 + id=self._tool_call_ids[i],
451 + type="function",
452 + function=DeltaFunctionCall(
453 + name=tool_name,
454 + arguments="",
455 + ).model_dump(exclude_none=True),
456 + )
457 )
458
399 - if key_pos == -1:
400 - return None
401 - if key_pos > 0:
402 - self._buffer = self._buffer[key_pos:]
403 - key_end = self._buffer.find(self.arg_key_end)
404 - if key_end == -1:
405 - return None
406 - key = self._buffer[len(self.arg_key_start) : key_end]
407 - self._buffer = self._buffer[key_end + len(self.arg_key_end) :]
408 - self._pending_key = key
409 - continue
410 -
411 - def _ensure_tool_state(self) -> None:
412 - while len(self._tool_call_ids) <= self.current_tool_id:
413 - self._tool_call_ids.append(
414 - make_tool_call_id(id_type="random", func_name=None, idx=None)
459 + # Build args JSON so far, diff, emit
460 + args_so_far = self._build_args_json_so_far(
461 + tool_name, inner_text, is_complete
462 )
416 - while len(self.streamed_args_for_tool) <= self.current_tool_id:
417 - self.streamed_args_for_tool.append("")
418 - while len(self.prev_tool_call_arr) <= self.current_tool_id:
419 - self.prev_tool_call_arr.append({})
420 - while len(self._args_started) <= self.current_tool_id:
421 - self._args_started.append(False)
422 - while len(self._args_closed) <= self.current_tool_id:
423 - self._args_closed.append(False)
424 - while len(self._seen_keys) <= self.current_tool_id:
425 - self._seen_keys.append(set())
426 -
427 - def _begin_tool_call(self) -> None:
428 - if self.current_tool_id == -1:
429 - self.current_tool_id = 0
430 - else:
431 - self.current_tool_id += 1
432 - self._ensure_tool_state()
433 - self.current_tool_name_sent = False
434 - self._current_tool_name = None
435 - self._pending_key = None
436 - self._streaming_string_value = False
437 - self._in_tool_call = True
438 -
439 - def _finish_tool_call(self) -> None:
440 - self._in_tool_call = False
441 - self._current_tool_name = None
442 - self._pending_key = None
443 - self._streaming_string_value = False
444 -
445 - def _revert_last_tool_call_state(self) -> None:
446 - """Revert the state allocation for the last tool call."""
447 - if self.current_tool_id < 0:
448 - return
449 - self._tool_call_ids.pop()
450 - self.streamed_args_for_tool.pop()
451 - self.prev_tool_call_arr.pop()
452 - self._args_started.pop()
453 - self._args_closed.pop()
454 - self._seen_keys.pop()
455 - self.current_tool_id -= 1
456 -
457 - def _emit_tool_name_delta(self, tool_name: str) -> DeltaMessage:
458 - self.prev_tool_call_arr[self.current_tool_id] = {
459 - "name": self._current_tool_name,
460 - "arguments": {},
461 - }
462 - return DeltaMessage(
463 - tool_calls=[
464 - DeltaToolCall(
465 - index=self.current_tool_id,
466 - id=self._tool_call_ids[self.current_tool_id],
467 - type="function",
468 - function=DeltaFunctionCall(
469 - name=tool_name,
470 - arguments="",
471 - ).model_dump(exclude_none=True),
472 - )
473 - ]
474 - )
475 -
476 - def _emit_tool_args_delta(self, fragment: str) -> DeltaMessage:
477 - return DeltaMessage(
478 - tool_calls=[
479 - DeltaToolCall(
480 - index=self.current_tool_id,
481 - function=DeltaFunctionCall(arguments=fragment).model_dump(
482 - exclude_none=True
483 - ),
463 + diff = self._compute_args_diff(i, args_so_far)
464 + if diff:
465 + tool_call_deltas.append(
466 + DeltaToolCall(
467 + index=i,
468 + function=DeltaFunctionCall(arguments=diff).model_dump(
469 + exclude_none=True
470 + ),
471 + )
472 )
485 - ]
486 - )
487 -
488 - def _append_arg_fragment(
489 - self,
490 - *,
491 - key: str,
492 - raw_val: str,
493 - ) -> str | None:
494 - key = key.strip()
495 - if not key:
496 - return None
497 - if key in self._seen_keys[self.current_tool_id]:
498 - return None
499 -
500 - # This function is only called for non-string types (already checked
501 - # by _is_string_type in the caller), so we always deserialize.
502 - val_obj: Any = self._deserialize(raw_val)
503 -
504 - key_json = json.dumps(key, ensure_ascii=False)
505 - val_json = json.dumps(val_obj, ensure_ascii=False)
506 -
507 - if not self._args_started[self.current_tool_id]:
508 - fragment = "{" + key_json + ": " + val_json
509 - self._args_started[self.current_tool_id] = True
510 - else:
511 - fragment = "," + key_json + ": " + val_json
473
513 - self._seen_keys[self.current_tool_id].add(key)
514 - self.streamed_args_for_tool[self.current_tool_id] += fragment
515 - return fragment
474 + # Update current_tool_id for serving layer compatibility
475 + if regions:
476 + self.current_tool_id = len(regions) - 1
477
517 - def _close_args_if_needed(self) -> str | None:
518 - if self._args_closed[self.current_tool_id]:
519 - return None
520 - self._args_closed[self.current_tool_id] = True
521 - if not self._args_started[self.current_tool_id]:
522 - fragment = "{}"
523 - self.streamed_args_for_tool[self.current_tool_id] = fragment
524 - else:
525 - fragment = "}"
526 - self.streamed_args_for_tool[self.current_tool_id] += fragment
527 - return fragment
478 + if content or tool_call_deltas:
479 + return DeltaMessage(
480 + content=content,
481 + tool_calls=tool_call_deltas,
482 + )
483 + return None
vllm/tool_parsers/utils.py
+13
@@ -31,6 +31,19 @@ Tool: TypeAlias = ChatCompletionToolsParam | ResponsesTool
31 logger = init_logger(__name__)
32
33
34 +def partial_tag_overlap(text: str, tag: str) -> int:
35 + """Length of the longest prefix of *tag* that matches a suffix of *text*.
36 +
37 + E.g. text ending in ``"<tool_"`` returns 6 when tag is ``"<tool_call>"``.
38 + Returns 0 when there is no overlap.
39 + """
40 + max_check = min(len(tag) - 1, len(text))
41 + for k in range(max_check, 0, -1):
42 + if text.endswith(tag[:k]):
43 + return k
44 + return 0
45 +
46 +
47 def find_common_prefix(s1: str, s2: str) -> str:
48 """
49 Finds a common prefix that is shared between two strings, if there is one.