@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c036432fc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 03:45 UTC

[Bugfix] GLM-4 tool parser: incremental string streaming (#33218)

Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com>

jack committed Feb 2, 2026 at 11:13 UTC 7c036432fc3e545487540460bec42aef58a8874d
2 files changed +725 -96
tests/tool_parsers/test_glm4_moe_tool_parser.py
+336
@@ -6,6 +6,7 @@ import json
6
7 import pytest
8
9 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
10 from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
11 from vllm.tokenizers import get_tokenizer
12 from vllm.tool_parsers.glm4_moe_tool_parser import (
@@ -447,3 +448,338 @@ def test_extract_tool_calls_incomplete_tool_call(glm4_moe_tool_parser):
448 assert not extracted_tool_calls.tools_called
449 assert extracted_tool_calls.tool_calls == []
450 assert extracted_tool_calls.content == model_output
451 +
452 +
453 +def _reset_streaming_state(parser):
454 + """Helper to reset parser streaming state."""
455 + parser._buffer = ""
456 + parser._in_tool_call = False
457 + parser.current_tool_name_sent = False
458 + parser._current_tool_name = None
459 + parser._pending_key = None
460 + parser._streaming_string_value = False
461 + parser.prev_tool_call_arr = []
462 + parser.current_tool_id = -1
463 + parser.streamed_args_for_tool = []
464 + parser._tool_call_ids = []
465 + parser._args_started = []
466 + parser._args_closed = []
467 + parser._seen_keys = []
468 +
469 +
470 +def test_streaming_incremental_string_value(glm4_moe_tool_parser):
471 + """Test incremental streaming of string argument values."""
472 + _reset_streaming_state(glm4_moe_tool_parser)
473 +
474 + # Simulate streaming a tool call character by character
475 + chunks = [
476 + "<tool_call>",
477 + "get_weather\n",
478 + "<arg_key>city</arg_key>",
479 + "<arg_value>",
480 + "Bei",
481 + "jing",
482 + "</arg_value>",
483 + "</tool_call>",
484 + ]
485 +
486 + collected_fragments = []
487 + for chunk in chunks:
488 + result = glm4_moe_tool_parser.extract_tool_calls_streaming(
489 + previous_text="",
490 + current_text="",
491 + delta_text=chunk,
492 + previous_token_ids=[],
493 + current_token_ids=[],
494 + delta_token_ids=[],
495 + request=None,
496 + )
497 + if result is not None and hasattr(result, "tool_calls") and result.tool_calls:
498 + for tc in result.tool_calls:
499 + if hasattr(tc, "function") and tc.function:
500 + func = tc.function
501 + if isinstance(func, dict):
502 + if func.get("arguments"):
503 + collected_fragments.append(func["arguments"])
504 + if func.get("name"):
505 + collected_fragments.append(f"name:{func['name']}")
506 + else:
507 + if func.arguments:
508 + collected_fragments.append(func.arguments)
509 + if func.name:
510 + collected_fragments.append(f"name:{func.name}")
511 +
512 + # Verify we got incremental streaming of the argument value
513 + assert len(collected_fragments) > 0
514 + # The fragments should include the tool name and argument pieces
515 + combined = "".join(collected_fragments)
516 + assert "get_weather" in combined or "name:get_weather" in combined
517 +
518 +
519 +def test_streaming_empty_tool_call(glm4_moe_tool_parser):
520 + """Test that empty tool calls don't cause infinite loops."""
521 + _reset_streaming_state(glm4_moe_tool_parser)
522 +
523 + # Empty tool call should be handled gracefully
524 + result = glm4_moe_tool_parser.extract_tool_calls_streaming(
525 + previous_text="",
526 + current_text="",
527 + delta_text="<tool_call></tool_call>",
528 + previous_token_ids=[],
529 + current_token_ids=[],
530 + delta_token_ids=[],
531 + request=None,
532 + )
533 +
534 + # Should not hang and should return something (None or content)
535 + # The key is that this completes without hanging
536 + assert result is None or hasattr(result, "content") or hasattr(result, "tool_calls")
537 + # State should be properly reset
538 + assert glm4_moe_tool_parser.current_tool_id == -1
539 +
540 +
541 +def test_streaming_prev_tool_call_arr_finalization(glm4_moe_tool_parser):
542 + """Test that prev_tool_call_arr contains parsed dict after tool call."""
543 + _reset_streaming_state(glm4_moe_tool_parser)
544 +
545 + # Stream a complete tool call
546 + chunks = [
547 + "<tool_call>get_weather\n",
548 + "<arg_key>city</arg_key>",
549 + "<arg_value>Beijing</arg_value>",
550 + "</tool_call>",
551 + ]
552 +
553 + for chunk in chunks:
554 + glm4_moe_tool_parser.extract_tool_calls_streaming(
555 + previous_text="",
556 + current_text="",
557 + delta_text=chunk,
558 + previous_token_ids=[],
559 + current_token_ids=[],
560 + delta_token_ids=[],
561 + request=None,
562 + )
563 +
564 + # After the tool call completes, prev_tool_call_arr should have parsed dict
565 + assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1
566 + tool_entry = glm4_moe_tool_parser.prev_tool_call_arr[0]
567 + assert tool_entry.get("name") == "get_weather"
568 + # arguments should be a dict, not a string
569 + args = tool_entry.get("arguments")
570 + assert isinstance(args, dict), f"Expected dict, got {type(args)}"
571 + assert args.get("city") == "Beijing"
572 +
573 +
574 +def test_streaming_multiple_tool_calls_sequential(glm4_moe_tool_parser):
575 + """Test streaming multiple sequential tool calls."""
576 + _reset_streaming_state(glm4_moe_tool_parser)
577 +
578 + # Stream two tool calls
579 + chunks = [
580 + "<tool_call>get_weather\n",
581 + "<arg_key>city</arg_key>",
582 + "<arg_value>Beijing</arg_value>",
583 + "</tool_call>",
584 + "<tool_call>get_weather\n",
585 + "<arg_key>city</arg_key>",
586 + "<arg_value>Shanghai</arg_value>",
587 + "</tool_call>",
588 + ]
589 +
590 + for chunk in chunks:
591 + glm4_moe_tool_parser.extract_tool_calls_streaming(
592 + previous_text="",
593 + current_text="",
594 + delta_text=chunk,
595 + previous_token_ids=[],
596 + current_token_ids=[],
597 + delta_token_ids=[],
598 + request=None,
599 + )
600 +
601 + # Should have two tool calls in prev_tool_call_arr
602 + assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 2
603 + assert glm4_moe_tool_parser.prev_tool_call_arr[0]["arguments"]["city"] == "Beijing"
604 + assert glm4_moe_tool_parser.prev_tool_call_arr[1]["arguments"]["city"] == "Shanghai"
605 +
606 +
607 +def test_streaming_json_escape_in_string(glm4_moe_tool_parser):
608 + """Test that special characters in string values are properly escaped."""
609 + _reset_streaming_state(glm4_moe_tool_parser)
610 +
611 + # String with characters that need JSON escaping
612 + chunks = [
613 + "<tool_call>send_message\n",
614 + "<arg_key>message</arg_key>",
615 + '<arg_value>Hello "world"\nNew line</arg_value>',
616 + "</tool_call>",
617 + ]
618 +
619 + for chunk in chunks:
620 + glm4_moe_tool_parser.extract_tool_calls_streaming(
621 + previous_text="",
622 + current_text="",
623 + delta_text=chunk,
624 + previous_token_ids=[],
625 + current_token_ids=[],
626 + delta_token_ids=[],
627 + request=None,
628 + )
629 +
630 + # The streamed_args_for_tool should contain valid JSON
631 + assert len(glm4_moe_tool_parser.streamed_args_for_tool) == 1
632 + args_json = glm4_moe_tool_parser.streamed_args_for_tool[0]
633 + # Should be parseable as JSON
634 + parsed = json.loads(args_json)
635 + assert "message" in parsed
636 + # The value should preserve the special characters
637 + assert '"' in parsed["message"] or "world" in parsed["message"]
638 +
639 +
640 +def test_streaming_long_content_incremental(glm4_moe_tool_parser):
641 + """Test incremental streaming of long content (Issue #32829).
642 +
643 + This is the core fix: for long string values like code (4000+ chars),
644 + the parser should stream incrementally rather than buffering until
645 + complete. This test verifies we get many fragments, not just 1-3.
646 + """
647 + _reset_streaming_state(glm4_moe_tool_parser)
648 +
649 + # Bubble sort example from Issue #32829 - realistic long content
650 + bubble_sort_code = '''#!/usr/bin/env python3
651 +# -*- coding: utf-8 -*-
652 +"""
653 +Bubble Sort Implementation
654 +"""
655 +
656 +def bubble_sort(arr):
657 + n = len(arr)
658 + for i in range(n):
659 + swapped = False
660 + for j in range(0, n - i - 1):
661 + if arr[j] > arr[j + 1]:
662 + arr[j], arr[j + 1] = arr[j + 1], arr[j]
663 + swapped = True
664 + if not swapped:
665 + break
666 + return arr
667 +
668 +if __name__ == "__main__":
669 + test_arr = [64, 34, 25, 12, 22, 11, 90]
670 + print(f"Original: {test_arr}")
671 + sorted_arr = bubble_sort(test_arr.copy())
672 + print(f"Sorted: {sorted_arr}")'''
673 +
674 + # Create a request with tool schema to enable string type detection
675 + # This is required for incremental streaming of string values
676 + request = ChatCompletionRequest(
677 + model=MODEL,
678 + messages=[],
679 + tools=[
680 + {
681 + "type": "function",
682 + "function": {
683 + "name": "write_to_file",
684 + "parameters": {
685 + "type": "object",
686 + "properties": {
687 + "file_path": {"type": "string"},
688 + "content": {"type": "string"},
689 + },
690 + },
691 + },
692 + }
693 + ],
694 + )
695 +
696 + # Simulate token-based streaming (special tags as single tokens)
697 + chunks = [
698 + "<tool_call>",
699 + "write_to_file\n",
700 + "<arg_key>file_path</arg_key>",
701 + "<arg_value>/tmp/bubble_sort.py</arg_value>",
702 + "<arg_key>content</arg_key>",
703 + "<arg_value>",
704 + ]
705 + # Add content line by line (realistic token streaming)
706 + for line in bubble_sort_code.split("\n"):
707 + chunks.append(line + "\n")
708 + chunks.append("</arg_value>")
709 + chunks.append("</tool_call>")
710 +
711 + # Count argument fragments
712 + fragment_count = 0
713 + for chunk in chunks:
714 + result = glm4_moe_tool_parser.extract_tool_calls_streaming(
715 + previous_text="",
716 + current_text="",
717 + delta_text=chunk,
718 + previous_token_ids=[],
719 + current_token_ids=[],
720 + delta_token_ids=[],
721 + request=request,
722 + )
723 + if result is not None and hasattr(result, "tool_calls") and result.tool_calls:
724 + for tc in result.tool_calls:
725 + if hasattr(tc, "function") and tc.function:
726 + func = tc.function
727 + args = (
728 + func.get("arguments")
729 + if isinstance(func, dict)
730 + else getattr(func, "arguments", None)
731 + )
732 + if args:
733 + fragment_count += 1
734 +
735 + # For true incremental streaming, we expect many fragments (10+)
736 + # Old buffered implementation would give only 1-3 fragments
737 + assert fragment_count >= 10, (
738 + f"Expected >=10 fragments for incremental streaming, got {fragment_count}"
739 + )
740 +
741 + # Verify final result is valid JSON
742 + assert len(glm4_moe_tool_parser.streamed_args_for_tool) == 1
743 + args_json = glm4_moe_tool_parser.streamed_args_for_tool[0]
744 + parsed = json.loads(args_json)
745 + assert parsed["file_path"] == "/tmp/bubble_sort.py"
746 + assert "def bubble_sort" in parsed["content"]
747 +
748 +
749 +def test_extract_tool_calls_numeric_deserialization(glm4_moe_tool_parser):
750 + """Test that numeric arguments are deserialized as numbers, not strings."""
751 + model_output = """<tool_call>calculate
752 +<arg_key>operation</arg_key>
753 +<arg_value>add</arg_value>
754 +<arg_key>a</arg_key>
755 +<arg_value>42</arg_value>
756 +<arg_key>b</arg_key>
757 +<arg_value>3.14</arg_value>
758 +<arg_key>enabled</arg_key>
759 +<arg_value>true</arg_value>
760 +</tool_call>"""
761 +
762 + extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls(
763 + model_output, request=None
764 + ) # type: ignore[arg-type]
765 +
766 + assert extracted_tool_calls.tools_called
767 + assert len(extracted_tool_calls.tool_calls) == 1
768 +
769 + args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments)
770 +
771 + # String should remain string
772 + assert args["operation"] == "add"
773 + assert isinstance(args["operation"], str)
774 +
775 + # Integer should be deserialized as int
776 + assert args["a"] == 42
777 + assert isinstance(args["a"], int)
778 +
779 + # Float should be deserialized as float
780 + assert args["b"] == 3.14
781 + assert isinstance(args["b"], float)
782 +
783 + # Boolean should be deserialized as bool
784 + assert args["enabled"] is True
785 + assert isinstance(args["enabled"], bool)
vllm/tool_parsers/glm4_moe_tool_parser.py
+389 -96
@@ -1,5 +1,15 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +GLM-4 Tool Call Parser with incremental string streaming support.
5 +
6 +This parser fixes the streaming issue reported in Issue #32829 where long string
7 +parameters (e.g., file content with 4000+ characters of code) are buffered until
8 +complete, causing multi-second delays before the user sees any content.
9 +
10 +The fix streams string values incrementally as they arrive, providing a true
11 +streaming experience for long content.
12 +"""
13
14 import ast
15 import json
@@ -8,6 +18,7 @@ from typing import Any
18
19 import regex as re
20
21 +from vllm.entrypoints.chat_utils import make_tool_call_id
22 from vllm.entrypoints.openai.chat_completion.protocol import (
23 ChatCompletionRequest,
24 ChatCompletionToolsParam,
@@ -30,14 +41,27 @@ logger = init_logger(__name__)
41
42
43 class Glm4MoeModelToolParser(ToolParser):
44 + """Tool parser for GLM-4 models with incremental string streaming.
45 +
46 + This parser emits tool-call deltas incrementally as arguments arrive.
47 + For string-type parameters, content is streamed character-by-character
48 + rather than waiting for the complete </arg_value> tag.
49 + """
50 +
51 def __init__(self, tokenizer: TokenizerLike):
52 super().__init__(tokenizer)
35 - self.current_tool_name_sent = False
36 - self.prev_tool_call_arr: list[dict] = []
37 - self.current_tool_id = -1
53 + # Stateful streaming fields
54 + self.current_tool_name_sent: bool = False
55 + self.prev_tool_call_arr: list[dict[str, Any]] = []
56 + self.current_tool_id: int = -1
57 self.streamed_args_for_tool: list[str] = []
39 - self.tool_call_start_token = "<tool_call>"
40 - self.tool_call_end_token = "</tool_call>"
58 +
59 + self.tool_call_start_token: str = "<tool_call>"
60 + self.tool_call_end_token: str = "</tool_call>"
61 + self.arg_key_start: str = "<arg_key>"
62 + self.arg_key_end: str = "</arg_key>"
63 + self.arg_val_start: str = "<arg_value>"
64 + self.arg_val_end: str = "</arg_value>"
65
66 self.tool_calls_start_token = self.tool_call_start_token
67
@@ -48,6 +72,7 @@ class Glm4MoeModelToolParser(ToolParser):
72 self.func_arg_regex = re.compile(
73 r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>", re.DOTALL
74 )
75 +
76 if not self.model_tokenizer:
77 raise ValueError(
78 "The model tokenizer must be passed to the ToolParser "
@@ -56,13 +81,78 @@ class Glm4MoeModelToolParser(ToolParser):
81
82 self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
83 self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
59 - self._buffer = ""
84 + self._buffer: str = ""
85
61 - def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
62 - """
63 - Adjust request parameters to ensure tool call tokens are not skipped
64 - during tokenizer decoding.
86 + # Streaming state for incremental tool-call streaming
87 + self._in_tool_call: bool = False
88 + self._current_tool_name: str | None = None
89 + self._pending_key: str | None = None
90 + self._streaming_string_value: bool = False
91 + self._tool_call_ids: list[str] = []
92 + self._args_started: list[bool] = []
93 + self._args_closed: list[bool] = []
94 + self._seen_keys: list[set[str]] = []
95 +
96 + @staticmethod
97 + def _deserialize(value: str) -> Any:
98 + try:
99 + return json.loads(value)
100 + except json.JSONDecodeError:
101 + pass
102 +
103 + try:
104 + return ast.literal_eval(value)
105 + except (ValueError, SyntaxError):
106 + pass
107 +
108 + return value
109 +
110 + @staticmethod
111 + def _json_escape_string_content(s: str) -> str:
112 + """JSON-escape string content for incremental streaming.
113 +
114 + This escapes the content that goes INSIDE a JSON string (between quotes),
115 + not including the surrounding quotes themselves.
116 """
117 + if not s:
118 + return ""
119 + return json.dumps(s, ensure_ascii=False)[1:-1]
120 +
121 + @staticmethod
122 + def _is_string_type(
123 + tool_name: str,
124 + arg_name: str,
125 + tools: list[ChatCompletionToolsParam] | None,
126 + ) -> bool:
127 + if tools is None:
128 + return False
129 + for tool in tools:
130 + if tool.function.name != tool_name:
131 + continue
132 + if tool.function.parameters is None:
133 + return False
134 + arg_type = (
135 + tool.function.parameters.get("properties", {})
136 + .get(arg_name, {})
137 + .get("type", None)
138 + )
139 + return arg_type == "string"
140 + logger.debug("No tool named '%s'.", tool_name)
141 + return False
142 +
143 + @staticmethod
144 + def _tools_enabled(request: ChatCompletionRequest) -> bool:
145 + """Return whether tool parsing should be applied for this request."""
146 + try:
147 + tools = getattr(request, "tools", None)
148 + tool_choice = getattr(request, "tool_choice", None)
149 + return bool(tools) and tool_choice != "none"
150 + except Exception:
151 + logger.exception("Failed to determine if tools are enabled.")
152 + return False
153 +
154 + def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
155 + """Adjust request parameters for tool call token handling."""
156 request = super().adjust_request(request)
157 if request.tools and request.tool_choice != "none":
158 # Ensure tool call tokens (<tool_call>, </tool_call>) are not skipped
@@ -77,42 +167,10 @@ class Glm4MoeModelToolParser(ToolParser):
167 model_output: str,
168 request: ChatCompletionRequest,
169 ) -> ExtractedToolCallInformation:
80 - def _is_string_type(
81 - tool_name: str,
82 - arg_name: str,
83 - tools: list[ChatCompletionToolsParam] | None,
84 - ) -> bool:
85 - if tools is None:
86 - return False
87 - for tool in tools:
88 - if tool.function.name == tool_name:
89 - if tool.function.parameters is None:
90 - return False
91 - arg_type = (
92 - tool.function.parameters.get("properties", {})
93 - .get(arg_name, {})
94 - .get("type", None)
95 - )
96 - return arg_type == "string"
97 - logger.debug("No tool named '%s'.", tool_name)
98 - return False
99 -
100 - def _deserialize(value: str) -> Any:
101 - try:
102 - return json.loads(value)
103 - except Exception:
104 - pass
105 -
106 - try:
107 - return ast.literal_eval(value)
108 - except Exception:
109 - pass
110 - return value
111 -
170 matched_tool_calls = self.func_call_regex.findall(model_output)
171 logger.debug("model_output: %s", model_output)
172 try:
115 - tool_calls = []
173 + tool_calls: list[ToolCall] = []
174 for match in matched_tool_calls:
175 tc_detail = self.func_detail_regex.search(match)
176 if not tc_detail:
@@ -121,15 +179,15 @@ class Glm4MoeModelToolParser(ToolParser):
179 match,
180 )
181 continue
124 - tc_name = tc_detail.group(1)
182 + tc_name = tc_detail.group(1).strip()
183 tc_args = tc_detail.group(2)
184 pairs = self.func_arg_regex.findall(tc_args) if tc_args else []
127 - arg_dct = {}
185 + arg_dct: dict[str, Any] = {}
186 for key, value in pairs:
187 arg_key = key.strip()
188 arg_val = value.strip()
131 - if not _is_string_type(tc_name, arg_key, request.tools):
132 - arg_val = _deserialize(arg_val)
189 + if not self._is_string_type(tc_name, arg_key, request.tools):
190 + arg_val = self._deserialize(arg_val)
191 logger.debug("arg_key = %s, arg_val = %s", arg_key, arg_val)
192 arg_dct[arg_key] = arg_val
193 tool_calls.append(
@@ -166,58 +224,293 @@ class Glm4MoeModelToolParser(ToolParser):
224 delta_token_ids: Sequence[int],
225 request: ChatCompletionRequest,
226 ) -> DeltaMessage | None:
227 + if not self._tools_enabled(request):
228 + return DeltaMessage(content=delta_text) if delta_text else None
229 +
230 self._buffer += delta_text
170 - cur_text = self._buffer
171 - start_idx = cur_text.find(self.tool_call_start_token)
172 - if start_idx == -1:
173 - self._buffer = ""
174 - if self.current_tool_id > 0:
175 - cur_text = ""
176 - return DeltaMessage(content=cur_text)
177 - logger.debug("cur_text = %s", cur_text)
178 - end_idx = cur_text.find(self.tool_call_end_token)
179 - if end_idx != -1:
180 - if self.current_tool_id == -1:
181 - self.current_tool_id = 0
182 - self.prev_tool_call_arr = []
183 - self.streamed_args_for_tool = []
184 - while len(self.prev_tool_call_arr) <= self.current_tool_id:
185 - self.prev_tool_call_arr.append({})
186 - while len(self.streamed_args_for_tool) <= self.current_tool_id:
187 - self.streamed_args_for_tool.append("")
188 -
189 - extracted_tool_calls = self.extract_tool_calls(
190 - cur_text[: end_idx + len(self.tool_call_end_token)], request
191 - )
231
193 - if len(extracted_tool_calls.tool_calls) == 0:
194 - logger.warning("Failed to extract any tool calls.")
195 - return None
196 - tool_call = extracted_tool_calls.tool_calls[0]
197 - self.prev_tool_call_arr[self.current_tool_id] = {
198 - "name": tool_call.function.name,
199 - "arguments": json.loads(tool_call.function.arguments),
200 - }
201 - self.streamed_args_for_tool[self.current_tool_id] = (
202 - tool_call.function.arguments
203 - )
204 - delta = DeltaMessage(
205 - content=extracted_tool_calls.content,
206 - tool_calls=[
207 - DeltaToolCall(
208 - index=self.current_tool_id,
209 - id=tool_call.id,
210 - type=tool_call.type,
211 - function=DeltaFunctionCall(
212 - name=tool_call.function.name,
213 - arguments=tool_call.function.arguments,
214 - ),
232 + while True:
233 + if not self._in_tool_call:
234 + start_idx = self._buffer.find(self.tool_call_start_token)
235 + if start_idx == -1:
236 + # Check for partial start token at end of buffer
237 + for i in range(1, len(self.tool_call_start_token)):
238 + if self._buffer.endswith(self.tool_call_start_token[:i]):
239 + out = self._buffer[:-i]
240 + self._buffer = self._buffer[-i:]
241 + return DeltaMessage(content=out) if out else None
242 + out = self._buffer
243 + self._buffer = ""
244 + return DeltaMessage(content=out) if out else None
245 +
246 + if start_idx > 0:
247 + out = self._buffer[:start_idx]
248 + self._buffer = self._buffer[start_idx:]
249 + return DeltaMessage(content=out) if out else None
250 +
251 + self._buffer = self._buffer[len(self.tool_call_start_token) :]
252 + self._begin_tool_call()
253 + continue
254 +
255 + # Parse tool name first
256 + if not self.current_tool_name_sent:
257 + nl = self._buffer.find("\n")
258 + ak = self._buffer.find(self.arg_key_start)
259 + end = self._buffer.find(self.tool_call_end_token)
260 + candidates = [i for i in [nl, ak, end] if i != -1]
261 + if not candidates:
262 + return None
263 + cut = min(candidates)
264 + tool_name = self._buffer[:cut].strip()
265 + if tool_name == "" and cut == end:
266 + # Handle empty tool call like `<tool_call></tool_call>`.
267 + # Consume the tokens and reset state to avoid infinite loop.
268 + self._buffer = self._buffer[end + len(self.tool_call_end_token) :]
269 + self._finish_tool_call()
270 + self._revert_last_tool_call_state()
271 + continue
272 +
273 + if cut == nl:
274 + self._buffer = self._buffer[nl + 1 :]
275 + else:
276 + self._buffer = self._buffer[cut:]
277 +
278 + self._current_tool_name = tool_name
279 + self.current_tool_name_sent = True
280 + return self._emit_tool_name_delta(tool_name)
281 +
282 + assert self._current_tool_name is not None
283 +
284 + # Handle incremental string value streaming
285 + if self._streaming_string_value:
286 + val_end = self._buffer.find(self.arg_val_end)
287 + if val_end != -1:
288 + raw_content = self._buffer[:val_end]
289 + self._buffer = self._buffer[val_end + len(self.arg_val_end) :]
290 + self._streaming_string_value = False
291 + self._pending_key = None
292 +
293 + escaped = self._json_escape_string_content(raw_content)
294 + frag = escaped + '"'
295 + self.streamed_args_for_tool[self.current_tool_id] += frag
296 + return self._emit_tool_args_delta(frag)
297 + else:
298 + # Check for partial </arg_value> at end
299 + safe_len = len(self._buffer)
300 + for i in range(1, len(self.arg_val_end)):
301 + if self._buffer.endswith(self.arg_val_end[:i]):
302 + safe_len = len(self._buffer) - i
303 + break
304 +
305 + if safe_len > 0:
306 + to_emit = self._buffer[:safe_len]
307 + self._buffer = self._buffer[safe_len:]
308 + escaped = self._json_escape_string_content(to_emit)
309 + if escaped:
310 + self.streamed_args_for_tool[self.current_tool_id] += escaped
311 + return self._emit_tool_args_delta(escaped)
312 + return None
313 +
314 + # If we have a pending key, parse its value
315 + if self._pending_key is not None:
316 + val_pos = self._buffer.find(self.arg_val_start)
317 + if val_pos == -1:
318 + return None
319 + if val_pos > 0:
320 + self._buffer = self._buffer[val_pos:]
321 +
322 + key = (self._pending_key or "").strip()
323 +
324 + is_string = self._is_string_type(
325 + self._current_tool_name, key, request.tools
326 + )
327 +
328 + if is_string:
329 + # String type: stream incrementally
330 + self._buffer = self._buffer[len(self.arg_val_start) :]
331 +
332 + if key in self._seen_keys[self.current_tool_id]:
333 + self._pending_key = None
334 + continue
335 +
336 + self._seen_keys[self.current_tool_id].add(key)
337 + key_json = json.dumps(key, ensure_ascii=False)
338 +
339 + if not self._args_started[self.current_tool_id]:
340 + frag = "{" + key_json + ':"'
341 + self._args_started[self.current_tool_id] = True
342 + else:
343 + frag = "," + key_json + ':"'
344 +
345 + self.streamed_args_for_tool[self.current_tool_id] += frag
346 + self._streaming_string_value = True
347 + return self._emit_tool_args_delta(frag)
348 + else:
349 + # Non-string type: wait for complete value
350 + val_end = self._buffer.find(self.arg_val_end)
351 + if val_end == -1:
352 + return None
353 +
354 + raw_val = self._buffer[len(self.arg_val_start) : val_end].strip()
355 + self._buffer = self._buffer[val_end + len(self.arg_val_end) :]
356 + self._pending_key = None
357 +
358 + frag = self._append_arg_fragment(
359 + key=key,
360 + raw_val=raw_val,
361 )
216 - ],
362 + if frag:
363 + return self._emit_tool_args_delta(frag)
364 + continue
365 +
366 + # Parse next arg or close
367 + end_pos = self._buffer.find(self.tool_call_end_token)
368 + key_pos = self._buffer.find(self.arg_key_start)
369 + if end_pos != -1 and (key_pos == -1 or end_pos < key_pos):
370 + self._buffer = self._buffer[end_pos + len(self.tool_call_end_token) :]
371 + frag = self._close_args_if_needed()
372 + # Finalize prev_tool_call_arr with complete parsed arguments
373 + if self._current_tool_name:
374 + try:
375 + full_args_str = self.streamed_args_for_tool[
376 + self.current_tool_id
377 + ]
378 + args_dict = json.loads(full_args_str)
379 + self.prev_tool_call_arr[self.current_tool_id] = {
380 + "name": self._current_tool_name,
381 + "arguments": args_dict,
382 + }
383 + except (json.JSONDecodeError, IndexError) as e:
384 + logger.warning(
385 + "Failed to finalize tool call state for tool %d: %s",
386 + self.current_tool_id,
387 + e,
388 + )
389 + self._finish_tool_call()
390 + return self._emit_tool_args_delta(frag) if frag else None
391 +
392 + if key_pos == -1:
393 + return None
394 + if key_pos > 0:
395 + self._buffer = self._buffer[key_pos:]
396 + key_end = self._buffer.find(self.arg_key_end)
397 + if key_end == -1:
398 + return None
399 + key = self._buffer[len(self.arg_key_start) : key_end]
400 + self._buffer = self._buffer[key_end + len(self.arg_key_end) :]
401 + self._pending_key = key
402 + continue
403 +
404 + def _ensure_tool_state(self) -> None:
405 + while len(self._tool_call_ids) <= self.current_tool_id:
406 + self._tool_call_ids.append(
407 + make_tool_call_id(id_type="random", func_name=None, idx=None)
408 )
409 + while len(self.streamed_args_for_tool) <= self.current_tool_id:
410 + self.streamed_args_for_tool.append("")
411 + while len(self.prev_tool_call_arr) <= self.current_tool_id:
412 + self.prev_tool_call_arr.append({})
413 + while len(self._args_started) <= self.current_tool_id:
414 + self._args_started.append(False)
415 + while len(self._args_closed) <= self.current_tool_id:
416 + self._args_closed.append(False)
417 + while len(self._seen_keys) <= self.current_tool_id:
418 + self._seen_keys.append(set())
419 +
420 + def _begin_tool_call(self) -> None:
421 + if self.current_tool_id == -1:
422 + self.current_tool_id = 0
423 + else:
424 self.current_tool_id += 1
219 - self._buffer = cur_text[end_idx + len(self.tool_call_end_token) :]
220 - return delta
425 + self._ensure_tool_state()
426 + self.current_tool_name_sent = False
427 + self._current_tool_name = None
428 + self._pending_key = None
429 + self._streaming_string_value = False
430 + self._in_tool_call = True
431 +
432 + def _finish_tool_call(self) -> None:
433 + self._in_tool_call = False
434 + self._current_tool_name = None
435 + self._pending_key = None
436 + self._streaming_string_value = False
437 +
438 + def _revert_last_tool_call_state(self) -> None:
439 + """Revert the state allocation for the last tool call."""
440 + if self.current_tool_id < 0:
441 + return
442 + self._tool_call_ids.pop()
443 + self.streamed_args_for_tool.pop()
444 + self.prev_tool_call_arr.pop()
445 + self._args_started.pop()
446 + self._args_closed.pop()
447 + self._seen_keys.pop()
448 + self.current_tool_id -= 1
449 +
450 + def _emit_tool_name_delta(self, tool_name: str) -> DeltaMessage:
451 + return DeltaMessage(
452 + tool_calls=[
453 + DeltaToolCall(
454 + index=self.current_tool_id,
455 + id=self._tool_call_ids[self.current_tool_id],
456 + type="function",
457 + function=DeltaFunctionCall(
458 + name=tool_name,
459 + arguments="",
460 + ).model_dump(exclude_none=True),
461 + )
462 + ]
463 + )
464 +
465 + def _emit_tool_args_delta(self, fragment: str) -> DeltaMessage:
466 + return DeltaMessage(
467 + tool_calls=[
468 + DeltaToolCall(
469 + index=self.current_tool_id,
470 + function=DeltaFunctionCall(arguments=fragment).model_dump(
471 + exclude_none=True
472 + ),
473 + )
474 + ]
475 + )
476 +
477 + def _append_arg_fragment(
478 + self,
479 + *,
480 + key: str,
481 + raw_val: str,
482 + ) -> str | None:
483 + key = key.strip()
484 + if not key:
485 + return None
486 + if key in self._seen_keys[self.current_tool_id]:
487 + return None
488 +
489 + # This function is only called for non-string types (already checked
490 + # by _is_string_type in the caller), so we always deserialize.
491 + val_obj: Any = self._deserialize(raw_val)
492
222 - self._buffer = cur_text[start_idx:]
223 - return DeltaMessage(content=cur_text[:start_idx])
493 + key_json = json.dumps(key, ensure_ascii=False)
494 + val_json = json.dumps(val_obj, ensure_ascii=False)
495 +
496 + if not self._args_started[self.current_tool_id]:
497 + fragment = "{" + key_json + ":" + val_json
498 + self._args_started[self.current_tool_id] = True
499 + else:
500 + fragment = "," + key_json + ":" + val_json
501 +
502 + self._seen_keys[self.current_tool_id].add(key)
503 + self.streamed_args_for_tool[self.current_tool_id] += fragment
504 + return fragment
505 +
506 + def _close_args_if_needed(self) -> str | None:
507 + if self._args_closed[self.current_tool_id]:
508 + return None
509 + self._args_closed[self.current_tool_id] = True
510 + if not self._args_started[self.current_tool_id]:
511 + fragment = "{}"
512 + self.streamed_args_for_tool[self.current_tool_id] = fragment
513 + else:
514 + fragment = "}"
515 + self.streamed_args_for_tool[self.current_tool_id] += fragment
516 + return fragment