@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8647c6cf51
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 20:30 UTC

[Bugfix] Fix minimax_m2 tool parser when stream interval > 1 (#35895)

Signed-off-by: sfeng33 <4florafeng@gmail.com>

Flora Feng committed Mar 11, 2026 at 22:25 UTC 8647c6cf510bbb0c22fe0820681b993e33406e32
3 files changed +534 -532
tests/tool_parsers/test_minimax_m2_tool_parser.py new
+444
@@ -0,0 +1,444 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +
6 +import pytest
7 +
8 +from vllm.tool_parsers.minimax_m2_tool_parser import (
9 + MinimaxM2ToolParser,
10 +)
11 +
12 +pytestmark = pytest.mark.cpu_test
13 +
14 +# Token IDs matching FakeTokenizer.vocab
15 +TC_START_ID = 1
16 +TC_END_ID = 2
17 +EOS_ID = 99
18 +
19 +
20 +class FakeTokenizer:
21 + """Minimal fake tokenizer for unit tests."""
22 +
23 + def __init__(self):
24 + self.model_tokenizer = True
25 + self.vocab = {
26 + "<minimax:tool_call>": TC_START_ID,
27 + "</minimax:tool_call>": TC_END_ID,
28 + }
29 +
30 + def get_vocab(self):
31 + return self.vocab
32 +
33 +
34 +@pytest.fixture
35 +def parser():
36 + return MinimaxM2ToolParser(FakeTokenizer())
37 +
38 +
39 +# ---------------------------------------------------------------------------
40 +# Helpers
41 +# ---------------------------------------------------------------------------
42 +
43 +
44 +def _feed(parser, chunks, request=None):
45 + """Feed chunks through the streaming parser and collect results.
46 +
47 + Each element in *chunks* is either:
48 + - a ``str``: used as delta_text (current_text accumulates automatically)
49 + - a ``(delta_text, delta_token_ids)`` tuple for special-token scenarios
50 +
51 + Returns a list of non-None DeltaMessage objects.
52 + """
53 + previous = ""
54 + results = []
55 + for chunk in chunks:
56 + if isinstance(chunk, tuple):
57 + delta, delta_ids = chunk
58 + else:
59 + delta = chunk
60 + delta_ids = []
61 +
62 + current = previous + delta
63 + result = parser.extract_tool_calls_streaming(
64 + previous_text=previous,
65 + current_text=current,
66 + delta_text=delta,
67 + previous_token_ids=[],
68 + current_token_ids=[],
69 + delta_token_ids=delta_ids,
70 + request=request,
71 + )
72 + if result is not None:
73 + results.append(result)
74 + previous = current
75 +
76 + return results
77 +
78 +
79 +def _collect_content(results):
80 + """Join all content strings from a list of DeltaMessages."""
81 + return "".join(r.content for r in results if r.content)
82 +
83 +
84 +def _collect_tool_calls(results):
85 + """Aggregate tool calls by index from a list of DeltaMessages.
86 +
87 + Returns a dict: index -> {"id": ..., "name": ..., "arguments": ...}
88 + """
89 + tool_calls = {}
90 + for r in results:
91 + for tc in r.tool_calls or []:
92 + if tc.index not in tool_calls:
93 + tool_calls[tc.index] = {
94 + "id": None,
95 + "name": "",
96 + "arguments": "",
97 + }
98 + if tc.id:
99 + tool_calls[tc.index]["id"] = tc.id
100 + if tc.function:
101 + if tc.function.name:
102 + tool_calls[tc.index]["name"] += tc.function.name
103 + if tc.function.arguments:
104 + tool_calls[tc.index]["arguments"] += tc.function.arguments
105 + return tool_calls
106 +
107 +
108 +# ---------------------------------------------------------------------------
109 +# Phase 1: content before tool calls
110 +# ---------------------------------------------------------------------------
111 +
112 +
113 +class TestContentStreaming:
114 + """Tests for plain content (no tool calls)."""
115 +
116 + def test_plain_content(self, parser):
117 + """No tool call tokens — all text is streamed as content."""
118 + results = _feed(parser, ["Hello ", "world"])
119 + assert _collect_content(results) == "Hello world"
120 + assert not parser.prev_tool_call_arr
121 +
122 + def test_content_before_tool_call(self, parser):
123 + """Text before <minimax:tool_call> is streamed as content."""
124 + results = _feed(
125 + parser,
126 + [
127 + "Let me check. ",
128 + '<minimax:tool_call><invoke name="get_weather">'
129 + '<parameter name="city">Seattle</parameter>'
130 + "</invoke></minimax:tool_call>",
131 + ],
132 + )
133 + assert _collect_content(results) == "Let me check. "
134 + assert len(parser.prev_tool_call_arr) == 1
135 +
136 + def test_empty_delta_no_crash(self, parser):
137 + """Empty delta_text with no token IDs returns None."""
138 + results = _feed(parser, [("", [])])
139 + assert results == []
140 +
141 +
142 +# ---------------------------------------------------------------------------
143 +# Phase 2: tool call parsing
144 +# ---------------------------------------------------------------------------
145 +
146 +
147 +class TestSingleInvoke:
148 + """Tests for a single <invoke> block."""
149 +
150 + def test_incremental_chunks(self, parser):
151 + """Each XML element arrives in a separate chunk."""
152 + results = _feed(
153 + parser,
154 + [
155 + "<minimax:tool_call>",
156 + '<invoke name="get_weather">',
157 + '<parameter name="city">Seattle</parameter>',
158 + "</invoke></minimax:tool_call>",
159 + ],
160 + )
161 + tc = _collect_tool_calls(results)
162 + assert len(tc) == 1
163 + assert tc[0]["name"] == "get_weather"
164 + assert json.loads(tc[0]["arguments"]) == {"city": "Seattle"}
165 + assert tc[0]["id"] is not None
166 +
167 + def test_single_chunk_complete(self, parser):
168 + """Entire tool call arrives in one delta."""
169 + results = _feed(
170 + parser,
171 + [
172 + '<minimax:tool_call><invoke name="get_weather">'
173 + '<parameter name="city">Seattle</parameter>'
174 + "</invoke></minimax:tool_call>",
175 + ],
176 + )
177 + tc = _collect_tool_calls(results)
178 + assert len(tc) == 1
179 + assert json.loads(tc[0]["arguments"]) == {"city": "Seattle"}
180 +
181 + def test_multiple_params(self, parser):
182 + """Multiple parameters in one invoke."""
183 + results = _feed(
184 + parser,
185 + [
186 + "<minimax:tool_call>",
187 + '<invoke name="get_weather">',
188 + '<parameter name="city">Seattle</parameter>',
189 + '<parameter name="days">5</parameter>',
190 + "</invoke></minimax:tool_call>",
191 + ],
192 + )
193 + tc = _collect_tool_calls(results)
194 + assert json.loads(tc[0]["arguments"]) == {
195 + "city": "Seattle",
196 + "days": "5",
197 + }
198 +
199 +
200 +class TestMultipleInvokes:
201 + """Tests for multiple <invoke> blocks in one tool call."""
202 +
203 + def test_two_invokes_incremental(self, parser):
204 + """Two invokes arriving one chunk at a time."""
205 + results = _feed(
206 + parser,
207 + [
208 + "<minimax:tool_call>",
209 + '<invoke name="search_web">'
210 + '<parameter name="query">OpenAI</parameter>'
211 + "</invoke>",
212 + '<invoke name="search_web">'
213 + '<parameter name="query">Gemini</parameter>'
214 + "</invoke>",
215 + "</minimax:tool_call>",
216 + ],
217 + )
218 + tc = _collect_tool_calls(results)
219 + assert len(tc) == 2
220 + assert tc[0]["name"] == "search_web"
221 + assert tc[1]["name"] == "search_web"
222 + assert json.loads(tc[0]["arguments"]) == {"query": "OpenAI"}
223 + assert json.loads(tc[1]["arguments"]) == {"query": "Gemini"}
224 +
225 + def test_two_invokes_in_single_delta(self, parser):
226 + """Both invokes close in the same delta — loop must emit both."""
227 + results = _feed(
228 + parser,
229 + [
230 + "<minimax:tool_call>",
231 + '<invoke name="fn_a"><parameter name="x">1</parameter></invoke>'
232 + '<invoke name="fn_b"><parameter name="y">2</parameter></invoke>',
233 + "</minimax:tool_call>",
234 + ],
235 + )
236 + tc = _collect_tool_calls(results)
237 + assert len(tc) == 2
238 + assert tc[0]["name"] == "fn_a"
239 + assert tc[1]["name"] == "fn_b"
240 +
241 + def test_different_functions(self, parser):
242 + """Parallel calls to different functions."""
243 + results = _feed(
244 + parser,
245 + [
246 + "<minimax:tool_call>",
247 + '<invoke name="get_weather">'
248 + '<parameter name="city">NYC</parameter>'
249 + "</invoke>",
250 + '<invoke name="get_stock">'
251 + '<parameter name="ticker">AAPL</parameter>'
252 + "</invoke>",
253 + "</minimax:tool_call>",
254 + ],
255 + )
256 + tc = _collect_tool_calls(results)
257 + assert tc[0]["name"] == "get_weather"
258 + assert tc[1]["name"] == "get_stock"
259 +
260 +
261 +# ---------------------------------------------------------------------------
262 +# Internal state: prev_tool_call_arr
263 +# ---------------------------------------------------------------------------
264 +
265 +
266 +class TestInternalState:
267 + """Verify prev_tool_call_arr is correct."""
268 +
269 + def test_prev_tool_call_arr_single(self, parser):
270 + _feed(
271 + parser,
272 + [
273 + '<minimax:tool_call><invoke name="fn">'
274 + '<parameter name="a">1</parameter>'
275 + "</invoke></minimax:tool_call>",
276 + ],
277 + )
278 + assert len(parser.prev_tool_call_arr) == 1
279 + assert parser.prev_tool_call_arr[0]["name"] == "fn"
280 + assert parser.prev_tool_call_arr[0]["arguments"] == {"a": "1"}
281 +
282 + def test_prev_tool_call_arr_multiple(self, parser):
283 + """prev_tool_call_arr records each invoke with correct arguments."""
284 + _feed(
285 + parser,
286 + [
287 + "<minimax:tool_call>",
288 + '<invoke name="search"><parameter name="q">hello</parameter></invoke>',
289 + '<invoke name="search"><parameter name="q">world</parameter></invoke>',
290 + "</minimax:tool_call>",
291 + ],
292 + )
293 + assert len(parser.prev_tool_call_arr) == 2
294 + assert parser.prev_tool_call_arr[0]["name"] == "search"
295 + assert parser.prev_tool_call_arr[0]["arguments"] == {"q": "hello"}
296 + assert parser.prev_tool_call_arr[1]["name"] == "search"
297 + assert parser.prev_tool_call_arr[1]["arguments"] == {"q": "world"}
298 +
299 +
300 +# ---------------------------------------------------------------------------
301 +# DeltaMessage structure
302 +# ---------------------------------------------------------------------------
303 +
304 +
305 +class TestDeltaMessageFormat:
306 + """Verify the shape of emitted DeltaMessage / DeltaToolCall."""
307 +
308 + def test_tool_call_fields(self, parser):
309 + """Each emitted tool call has id, name, arguments, type, index."""
310 + results = _feed(
311 + parser,
312 + [
313 + '<minimax:tool_call><invoke name="fn">'
314 + '<parameter name="k">v</parameter>'
315 + "</invoke></minimax:tool_call>",
316 + ],
317 + )
318 + tc_deltas = [tc for r in results for tc in (r.tool_calls or [])]
319 + assert len(tc_deltas) == 1
320 + tc = tc_deltas[0]
321 + assert tc.index == 0
322 + assert tc.type == "function"
323 + assert tc.id is not None and tc.id.startswith("call_")
324 + assert tc.function.name == "fn"
325 + assert json.loads(tc.function.arguments) == {"k": "v"}
326 +
327 + def test_multi_invoke_indices(self, parser):
328 + """Multiple invokes get sequential indices."""
329 + results = _feed(
330 + parser,
331 + [
332 + "<minimax:tool_call>",
333 + '<invoke name="a"><parameter name="x">1</parameter></invoke>',
334 + '<invoke name="b"><parameter name="x">2</parameter></invoke>',
335 + "</minimax:tool_call>",
336 + ],
337 + )
338 + tc_deltas = [tc for r in results for tc in (r.tool_calls or [])]
339 + indices = [tc.index for tc in tc_deltas]
340 + assert indices == [0, 1]
341 +
342 +
343 +# ---------------------------------------------------------------------------
344 +# Phase 3: EOS handling
345 +# ---------------------------------------------------------------------------
346 +
347 +
348 +class TestEOSHandling:
349 + """Tests for the end-of-stream phase."""
350 +
351 + def test_eos_after_tool_calls(self, parser):
352 + """EOS token (empty delta, non-special token id) returns content=''."""
353 + results = _feed(
354 + parser,
355 + [
356 + "<minimax:tool_call>",
357 + '<invoke name="fn"><parameter name="k">v</parameter></invoke>',
358 + "</minimax:tool_call>",
359 + # EOS: empty delta_text, non-special token id
360 + ("", [EOS_ID]),
361 + ],
362 + )
363 + # Last result should be the EOS empty-content signal
364 + assert results[-1].content == ""
365 +
366 + def test_end_token_ignored(self, parser):
367 + """</minimax:tool_call> special token should NOT trigger EOS."""
368 + results = _feed(
369 + parser,
370 + [
371 + "<minimax:tool_call>",
372 + '<invoke name="fn"><parameter name="k">v</parameter></invoke>',
373 + # </minimax:tool_call> arrives as special token
374 + ("", [TC_END_ID]),
375 + ],
376 + )
377 + # The tool call delta should be emitted, but no EOS signal
378 + assert not any(r.content == "" and r.tool_calls is None for r in results)
379 +
380 +
381 +# ---------------------------------------------------------------------------
382 +# Start token detection via token IDs
383 +# ---------------------------------------------------------------------------
384 +
385 +
386 +class TestSpecialTokenDetection:
387 + """Start token arrives as a special token (not in delta_text)."""
388 +
389 + def test_start_token_via_id(self, parser):
390 + """<minimax:tool_call> detected via delta_token_ids, not text."""
391 + results = _feed(parser, ["Hello "])
392 + assert _collect_content(results) == "Hello "
393 +
394 + # Start token as special token (empty delta_text)
395 + previous = "Hello "
396 + result = parser.extract_tool_calls_streaming(
397 + previous_text=previous,
398 + current_text=previous,
399 + delta_text="",
400 + previous_token_ids=[],
401 + current_token_ids=[],
402 + delta_token_ids=[TC_START_ID],
403 + request=None,
404 + )
405 + assert result is None # no content to emit
406 + assert parser.is_tool_call_started is True
407 +
408 +
409 +# ---------------------------------------------------------------------------
410 +# Large chunks (stream_interval > 1)
411 +# ---------------------------------------------------------------------------
412 +
413 +
414 +class TestLargeChunks:
415 + """Simulate stream_interval > 1 where many tokens arrive at once."""
416 +
417 + def test_header_and_params_in_separate_chunks(self, parser):
418 + """Header in chunk 1, all params + close in chunk 2, then EOS."""
419 + chunk1 = '<minimax:tool_call><invoke name="get_weather">'
420 + chunk2 = (
421 + '<parameter name="city">Seattle</parameter>'
422 + '<parameter name="days">5</parameter>'
423 + "</invoke></minimax:tool_call>"
424 + )
425 +
426 + results = _feed(
427 + parser,
428 + [
429 + chunk1,
430 + chunk2,
431 + ("", [EOS_ID]),
432 + ],
433 + )
434 +
435 + tc = _collect_tool_calls(results)
436 + assert len(tc) == 1
437 + parsed = json.loads(tc[0]["arguments"])
438 + assert parsed == {"city": "Seattle", "days": "5"}
439 +
440 + assert len(parser.prev_tool_call_arr) == 1
441 + assert parser.prev_tool_call_arr[0]["arguments"] == {
442 + "city": "Seattle",
443 + "days": "5",
444 + }
tests/tool_use/test_minimax_m2_tool_parser.py deleted
-119
@@ -1,119 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import json
5 -
6 -import pytest
7 -
8 -from vllm.tool_parsers.minimax_m2_tool_parser import (
9 - MinimaxM2ToolParser,
10 -)
11 -
12 -pytestmark = pytest.mark.cpu_test
13 -
14 -
15 -class FakeTokenizer:
16 - """Minimal fake tokenizer that exposes the attributes used by the
17 - parser: a truthy model_tokenizer marker and a vocab mapping for the
18 - special tokens.
19 - """
20 -
21 - def __init__(self):
22 - self.model_tokenizer = True
23 - # The parser will look up start/end tokens by their literal strings
24 - self.vocab = {
25 - "<minimax:tool_call>": 1,
26 - "</minimax:tool_call>": 2,
27 - }
28 -
29 - def get_vocab(self):
30 - return self.vocab
31 -
32 -
33 -@pytest.fixture
34 -def minimax_m2_tool_parser():
35 - return MinimaxM2ToolParser(FakeTokenizer())
36 -
37 -
38 -def test_extract_tool_calls_streaming_incremental(minimax_m2_tool_parser):
39 - parser = minimax_m2_tool_parser
40 - parser._reset_streaming_state()
41 - chunks = [
42 - "<minimax:tool_call>",
43 - '<invoke name="get_weather">',
44 - '<parameter name="city">',
45 - "Seattle</parameter>",
46 - "</invoke></minimax:tool_call>",
47 - ]
48 - previous = ""
49 - for chunk in chunks:
50 - current = previous + chunk
51 - delta = chunk
52 - parser.extract_tool_calls_streaming(
53 - previous_text=previous,
54 - current_text=current,
55 - delta_text=delta,
56 - previous_token_ids=[],
57 - current_token_ids=[],
58 - delta_token_ids=[],
59 - request=None,
60 - )
61 - previous = current
62 -
63 - assert len(parser.prev_tool_call_arr) == 1
64 - entry = parser.prev_tool_call_arr[0]
65 -
66 - assert entry["name"] == "get_weather"
67 - args = entry["arguments"]
68 - assert args["city"] == "Seattle"
69 -
70 -
71 -def test_streaming_minimax_m2_multiple_invokes(minimax_m2_tool_parser):
72 - parser = minimax_m2_tool_parser
73 - parser._reset_streaming_state()
74 -
75 - chunks = [
76 - "<minimax:tool_call>",
77 - '<invoke name="search_web">',
78 - '<parameter name="query_tag">',
79 - '["technology", "events"]</parameter>',
80 - '<parameter name="query_list">',
81 - '["OpenAI", "latest", "release"]</parameter>',
82 - "</invoke>",
83 - '<invoke name="search_web">',
84 - '<parameter name="query_tag">',
85 - '["technology", "events"]</parameter>',
86 - '<parameter name="query_list">',
87 - '["Gemini", "latest", "release"]</parameter>',
88 - "</invoke>",
89 - "</minimax:tool_call>",
90 - ]
91 - previous = ""
92 - for chunk in chunks:
93 - current = previous + chunk
94 - delta = chunk
95 - parser.extract_tool_calls_streaming(
96 - previous_text=previous,
97 - current_text=current,
98 - delta_text=delta,
99 - previous_token_ids=[],
100 - current_token_ids=[],
101 - delta_token_ids=[],
102 - request=None,
103 - )
104 - previous = current
105 -
106 - assert len(parser.prev_tool_call_arr) == 2
107 -
108 - for entry, expect_model in zip(parser.prev_tool_call_arr, ["OpenAI", "Gemini"]):
109 - assert entry["name"] == "search_web"
110 - args = json.dumps(entry["arguments"])
111 - assert "technology" in args and "events" in args
112 - assert expect_model in args
113 -
114 - # check streamed_args_for_tool for serving_chat.py
115 - for index in range(2):
116 - expected_call = parser.prev_tool_call_arr[index].get("arguments", {})
117 - expected_call = json.dumps(expected_call)
118 - actual_call = parser.streamed_args_for_tool[index]
119 - assert expected_call == actual_call
vllm/tool_parsers/minimax_m2_tool_parser.py
+90 -413
@@ -37,37 +37,10 @@ class MinimaxM2ToolParser(ToolParser):
37 # Sentinel tokens
38 self.tool_call_start_token: str = "<minimax:tool_call>"
39 self.tool_call_end_token: str = "</minimax:tool_call>"
40 - self.invoke_start_prefix: str = "<invoke name="
41 - self.invoke_end_token: str = "</invoke>"
42 - self.parameter_prefix: str = "<parameter name="
43 - self.parameter_end_token: str = "</parameter>"
44 -
45 - # Streaming state variables
46 - self.current_tool_name_sent: bool = False
47 - # Override base class type - we use string IDs for tool calls
48 - self.current_tool_id: str | None = None # type: ignore
49 - self.streamed_args_for_tool: list[str] = []
50 - self.is_tool_call_started: bool = False
51 - self.failed_count: int = 0
40
53 - # Initialize streaming state variables
41 + # Streaming state
42 + self.is_tool_call_started: bool = False
43 self.current_tool_index: int = 0
55 - self.invoke_index: int = 0
56 - self.header_sent: bool = False
57 - self.current_function_name: str | None = None
58 - self.current_param_name: str | None = None
59 - self.current_param_value: str = ""
60 - self.param_count: int = 0
61 - self.in_param: bool = False
62 - self.in_function: bool = False
63 - self.accumulated_text: str = ""
64 - self.json_started: bool = False
65 - self.json_closed: bool = False
66 - self.accumulated_params: dict = {}
67 - self.streaming_request: ChatCompletionRequest | None = None
68 -
69 - # Enhanced streaming state - reset for each new message
70 - self._reset_streaming_state()
44
45 # Regex patterns for complete parsing
46 self.tool_call_complete_regex = re.compile(
@@ -103,46 +76,15 @@ class MinimaxM2ToolParser(ToolParser):
76 """Generate a unique tool call ID."""
77 return f"call_{uuid.uuid4().hex[:24]}"
78
106 - def _reset_streaming_state(self):
107 - """Reset all streaming state."""
108 - self.current_tool_index = 0
109 - self.invoke_index = 0
110 - self.is_tool_call_started = False
111 - self.header_sent = False
112 - self.current_tool_id = None
113 - self.current_function_name = None
114 - self.current_param_name = None
115 - self.current_param_value = ""
116 - self.param_count = 0
117 - self.in_param = False
118 - self.in_function = False
119 - self.accumulated_text = ""
120 - self.json_started = False
121 - self.json_closed = False
122 - # Store accumulated parameters for type conversion
123 - self.accumulated_params = {}
124 - self.streaming_request = None
125 - # Clear previous tool call history to avoid state pollution
126 - self.prev_tool_call_arr.clear()
127 - # Reset streamed args tracking
128 - self.streamed_args_for_tool.clear()
129 -
79 def _extract_name(self, name_str: str) -> str:
80 """Extract name from quoted string."""
81 name_str = name_str.strip()
133 - if (
134 - name_str.startswith('"')
135 - and name_str.endswith('"')
136 - or name_str.startswith("'")
137 - and name_str.endswith("'")
82 + if (name_str.startswith('"') and name_str.endswith('"')) or (
83 + name_str.startswith("'") and name_str.endswith("'")
84 ):
85 return name_str[1:-1]
86 return name_str
87
142 - def _convert_param_value(self, value: str, param_type: str) -> Any:
143 - """Convert parameter value to the correct type (legacy single-type version)."""
144 - return self._convert_param_value_with_types(value, [param_type])
145 -
88 def _extract_types_from_schema(self, schema: Any) -> list[str]:
89 """
90 Extract all possible types from a JSON schema definition.
@@ -331,10 +273,6 @@ class MinimaxM2ToolParser(ToolParser):
273 if param_match:
274 param_name = self._extract_name(param_match.group(1))
275 param_value = param_match.group(2).strip()
334 - if param_value.startswith("\n"):
335 - param_value = param_value[1:]
336 - if param_value.endswith("\n"):
337 - param_value = param_value[:-1]
276
277 # Get parameter types (supports anyOf/oneOf/allOf)
278 param_type = self._get_param_types_from_config(param_name, param_config)
@@ -352,6 +290,54 @@ class MinimaxM2ToolParser(ToolParser):
290 ),
291 )
292
293 + def _extract_delta_tool_calls(
294 + self,
295 + current_text: str,
296 + request: ChatCompletionRequest | None,
297 + ) -> list[DeltaToolCall]:
298 + """Extract DeltaToolCalls from newly completed <invoke> blocks.
299 +
300 + Tracks progress via ``current_tool_index`` so each block is
301 + extracted exactly once across successive streaming calls.
302 + """
303 + complete_invokes = self.invoke_complete_regex.findall(current_text)
304 + delta_tool_calls: list[DeltaToolCall] = []
305 +
306 + while len(complete_invokes) > self.current_tool_index:
307 + invoke_str = complete_invokes[self.current_tool_index]
308 + tool_call = self._parse_single_invoke(
309 + invoke_str,
310 + request.tools if request else None,
311 + )
312 + if not tool_call:
313 + self.current_tool_index += 1
314 + continue
315 +
316 + args_json = tool_call.function.arguments
317 + idx = self.current_tool_index
318 + self.current_tool_index += 1
319 +
320 + self.prev_tool_call_arr.append(
321 + {
322 + "name": tool_call.function.name,
323 + "arguments": json.loads(args_json),
324 + }
325 + )
326 + self.streamed_args_for_tool.append(args_json)
327 + delta_tool_calls.append(
328 + DeltaToolCall(
329 + index=idx,
330 + id=self._generate_tool_call_id(),
331 + function=DeltaFunctionCall(
332 + name=tool_call.function.name,
333 + arguments=args_json,
334 + ),
335 + type="function",
336 + )
337 + )
338 +
339 + return delta_tool_calls
340 +
341 def extract_tool_calls(
342 self,
343 model_output: str,
@@ -416,360 +402,51 @@ class MinimaxM2ToolParser(ToolParser):
402 delta_token_ids: Sequence[int],
403 request: ChatCompletionRequest,
404 ) -> DeltaMessage | None:
419 - """Extract tool calls from streaming model output."""
420 -
421 - # Store request for type conversion
422 - if not previous_text or self.tool_call_start_token in delta_text:
423 - self._reset_streaming_state()
424 - self.streaming_request = request
425 -
426 - # If no delta text, return None unless it's an EOS token after tools
427 - if not delta_text:
428 - # Check if this is an EOS token after all tool calls are complete
429 - if delta_token_ids and self.tool_call_end_token_id not in delta_token_ids:
430 - # Count complete tool calls
431 - complete_calls = len(
432 - self.tool_call_complete_regex.findall(current_text)
433 - )
405 + """Extract tool calls from streaming model output.
406
435 - # If we have completed tool calls and populated prev_tool_call_arr
436 - if complete_calls > 0 and len(self.prev_tool_call_arr) > 0:
437 - # Check if all tool calls are closed
438 - open_calls = current_text.count(
439 - self.tool_call_start_token
440 - ) - current_text.count(self.tool_call_end_token)
441 - if open_calls == 0:
442 - # Return empty delta for finish_reason processing
443 - return DeltaMessage(content="")
444 - elif not self.is_tool_call_started and current_text:
445 - # This is a regular content response that's now complete
446 - return DeltaMessage(content="")
447 - return None
407 + Uses a buffer-until-complete-invoke strategy: tokens are buffered
408 + until a complete ``<invoke>...</invoke>`` block is available, then
409 + parsed and emitted in one shot.
410 + """
411
449 - # Update accumulated text
450 - self.accumulated_text = current_text
412 + start_in_text = self.tool_call_start_token in delta_text
413 + start_in_ids = self.tool_call_start_token_id in delta_token_ids
414 + tool_call_starting = start_in_text or start_in_ids
415 + # Reset state on new request (parser is reused) or new tool-call block.
416 + if not previous_text or tool_call_starting:
417 + self.current_tool_index = 0
418 + self.prev_tool_call_arr.clear()
419 + self.streamed_args_for_tool.clear()
420 + self.is_tool_call_started = tool_call_starting
421
452 - # Check if we need to advance to next tool
453 - if self.json_closed and not self.in_function:
454 - # Check if this tool call has ended
455 - invoke_ends = current_text.count(self.invoke_end_token)
456 - if invoke_ends > self.current_tool_index:
457 - # This tool has ended, advance to next
458 - self.current_tool_index += 1
459 - self.header_sent = False
460 - self.param_count = 0
461 - self.json_started = False
462 - self.json_closed = False
463 - self.in_function = False # Now we can safely set this to False
464 - self.accumulated_params = {}
465 - # Continue processing next tool
466 - return None
467 -
468 - # Handle normal content before tool calls
422 + # Pass through content before any tool call.
423 if not self.is_tool_call_started:
470 - # Check if tool call is starting
471 - if (
472 - self.tool_call_start_token_id in delta_token_ids
473 - or self.tool_call_start_token in delta_text
474 - ):
475 - self.is_tool_call_started = True
476 - # Return any content before the tool call
477 - if self.tool_call_start_token in delta_text:
478 - content_before = delta_text[
479 - : delta_text.index(self.tool_call_start_token)
480 - ]
481 - if content_before:
482 - return DeltaMessage(content=content_before)
483 - return None
484 - else:
485 - # Check if we're between tool calls - skip whitespace
486 - if (
487 - current_text.rstrip().endswith(self.tool_call_end_token)
488 - and delta_text.strip() == ""
489 - ):
490 - # We just ended a tool call, skip whitespace
491 - return None
492 - # Normal content, no tool call
493 - return DeltaMessage(content=delta_text)
494 -
495 - # Check if we're between tool calls (waiting for next one)
496 - invoke_starts_count = current_text.count(self.invoke_start_prefix)
497 - if self.current_tool_index >= invoke_starts_count:
498 - # We're past all tool calls, shouldn't be here
499 - return None
424 + return DeltaMessage(content=delta_text) if delta_text else None
425
501 - # Find the current tool call portion
502 - invoke_start_positions: list[int] = []
503 - idx = 0
504 - while True:
505 - idx = current_text.find(self.invoke_start_prefix, idx)
506 - if idx == -1:
507 - break
508 - invoke_start_positions.append(idx)
509 - idx += len(self.invoke_start_prefix)
510 -
511 - if self.current_tool_index >= len(invoke_start_positions):
512 - # No more tool calls to process yet
513 - return None
426 + # Capture content before the start token.
427 + content_before = None
428 + if start_in_text:
429 + before = delta_text[: delta_text.index(self.tool_call_start_token)]
430 + content_before = before or None
431
515 - invoke_start_idx = invoke_start_positions[self.current_tool_index]
516 - # Find where this tool call ends (or current position if not ended yet)
517 - invoke_end_idx = current_text.find(self.invoke_end_token, invoke_start_idx)
518 - if invoke_end_idx == -1:
519 - tool_text = current_text[invoke_start_idx:]
520 - else:
521 - tool_text = current_text[
522 - invoke_start_idx : invoke_end_idx + len(self.invoke_end_token)
523 - ]
524 -
525 - # Looking for function header
526 - if not self.header_sent:
527 - if self.invoke_start_prefix in tool_text:
528 - func_start = tool_text.find(self.invoke_start_prefix) + len(
529 - self.invoke_start_prefix
530 - )
531 - # Find the end quote for the function name
532 - func_end = tool_text.find(">", func_start)
533 -
534 - if func_end != -1:
535 - # Found complete function name
536 - function_name_raw = tool_text[func_start:func_end]
537 - self.current_function_name = self._extract_name(function_name_raw)
538 - self.current_tool_id = self._generate_tool_call_id()
539 - self.header_sent = True
540 - self.in_function = True
541 -
542 - # Add to prev_tool_call_arr immediately when we detect a tool call
543 - # Each tool call should be recorded regardless of function name
544 - # Ensure we don't add the same tool call index multiple times
545 - if len(self.prev_tool_call_arr) <= self.current_tool_index:
546 - self.prev_tool_call_arr.append(
547 - {
548 - "name": self.current_function_name,
549 - "arguments": {}, # Placeholder, will be updated later
550 - }
551 - )
552 - # Initialize streamed_args_for_tool for this tool call
553 - if len(self.streamed_args_for_tool) <= self.current_tool_index:
554 - self.streamed_args_for_tool.append("")
555 -
556 - # Send header with function info
557 - return DeltaMessage(
558 - tool_calls=[
559 - DeltaToolCall(
560 - index=self.current_tool_index,
561 - id=self.current_tool_id,
562 - function=DeltaFunctionCall(
563 - name=self.current_function_name, arguments=""
564 - ),
565 - type="function",
566 - )
567 - ]
568 - )
569 - return None
432 + # Extract newly completed <invoke> blocks as DeltaToolCalls.
433 + delta_tool_calls = self._extract_delta_tool_calls(current_text, request)
434
571 - # We've sent header, now handle function body
572 - if self.in_function:
573 - # Send opening brace if not sent yet
574 - if self.in_function and not self.json_started:
575 - self.json_started = True
576 - # Update streamed_args_for_tool for opening brace
577 - if self.current_tool_index < len(self.streamed_args_for_tool):
578 - self.streamed_args_for_tool[self.current_tool_index] += "{"
579 - return DeltaMessage(
580 - tool_calls=[
581 - DeltaToolCall(
582 - index=self.current_tool_index,
583 - function=DeltaFunctionCall(arguments="{"),
584 - )
585 - ]
586 - )
587 -
588 - # Make sure json_started is set if we're processing parameters
589 - if not self.json_started:
590 - self.json_started = True
591 -
592 - # Check for function end in accumulated text
593 - if not self.json_closed and self.invoke_end_token in tool_text:
594 - # Count total parameters in the tool text
595 - total_param_count = tool_text.count(self.parameter_prefix)
596 -
597 - # Only close JSON if all parameters have been processed
598 - if self.param_count >= total_param_count:
599 - # Close JSON
600 - self.json_closed = True
435 + if delta_tool_calls or content_before:
436 + return DeltaMessage(
437 + content=content_before,
438 + tool_calls=delta_tool_calls,
439 + )
440
602 - # Extract complete tool call
603 - # Find the invoke content
604 - invoke_start = tool_text.find(self.invoke_start_prefix) + len(
605 - self.invoke_start_prefix
606 - )
607 - invoke_content_end = tool_text.find(
608 - self.invoke_end_token, invoke_start
609 - )
610 - if invoke_content_end != -1:
611 - invoke_content = tool_text[invoke_start:invoke_content_end]
612 - # Parse to get the complete arguments
613 - try:
614 - parsed_tool = self._parse_single_invoke(
615 - invoke_content,
616 - self.streaming_request.tools
617 - if self.streaming_request
618 - else None,
619 - )
620 - if parsed_tool and self.current_tool_index < len(
621 - self.prev_tool_call_arr
622 - ):
623 - # Update existing entry in prev_tool_call_arr
624 - args = parsed_tool.function.arguments
625 - self.prev_tool_call_arr[self.current_tool_index][
626 - "arguments"
627 - ] = json.loads(args)
628 - except Exception:
629 - pass # Ignore parsing errors during streaming
630 -
631 - result = DeltaMessage(
632 - tool_calls=[
633 - DeltaToolCall(
634 - index=self.current_tool_index,
635 - function=DeltaFunctionCall(arguments="}"),
636 - )
637 - ]
638 - )
639 - # Update streamed_args_for_tool for closing brace
640 - if self.current_tool_index < len(self.streamed_args_for_tool):
641 - self.streamed_args_for_tool[self.current_tool_index] += "}"
642 - # Reset state for next tool
643 - self.json_closed = True
644 - self.in_function = False
645 - self.accumulated_params = {}
646 -
647 - logger.debug("[M2_STREAMING] Tool call completed")
648 -
649 - return result
650 - else:
651 - # Don't close JSON yet, continue processing parameters
652 - return None
653 -
654 - # Look for parameters
655 - # Find all parameter starts
656 - param_starts = []
657 - idx = 0
658 - while True:
659 - idx = tool_text.find(self.parameter_prefix, idx)
660 - if idx == -1:
661 - break
662 - param_starts.append(idx)
663 - idx += len(self.parameter_prefix)
664 -
665 - # Check if we should start a new parameter
666 - if (
667 - not self.in_param
668 - and self.param_count < len(param_starts)
669 - and len(param_starts) > self.param_count
670 - ):
671 - # Process the next parameter
672 - param_idx = param_starts[self.param_count]
673 - param_start = param_idx + len(self.parameter_prefix)
674 - remaining = tool_text[param_start:]
675 -
676 - if ">" in remaining:
677 - # We have the complete parameter name
678 - name_end = remaining.find(">")
679 - param_name_raw = remaining[:name_end]
680 - self.current_param_name = self._extract_name(param_name_raw)
681 -
682 - # Find the parameter value
683 - value_start = param_start + name_end + 1
684 - value_text = tool_text[value_start:]
685 - if value_text.startswith("\n"):
686 - value_text = value_text[1:]
687 -
688 - # Find where this parameter ends
689 - param_end_idx = value_text.find(self.parameter_end_token)
690 - if param_end_idx == -1:
691 - # No closing tag, look for next parameter or function end
692 - next_param_idx = value_text.find(self.parameter_prefix)
693 - func_end_idx = value_text.find(self.invoke_end_token)
694 -
695 - if next_param_idx != -1 and (
696 - func_end_idx == -1 or next_param_idx < func_end_idx
697 - ):
698 - param_end_idx = next_param_idx
699 - elif func_end_idx != -1:
700 - param_end_idx = func_end_idx
701 - else:
702 - # Neither found, check if tool call is complete
703 - if self.invoke_end_token in tool_text:
704 - # Tool call and parameter is complete
705 - param_end_idx = len(value_text)
706 - else:
707 - # Still streaming, wait for more content
708 - return None
709 -
710 - if param_end_idx != -1:
711 - # Complete parameter found
712 - param_value = value_text[:param_end_idx]
713 - if param_value.endswith("\n"):
714 - param_value = param_value[:-1]
715 -
716 - # Store raw value for later processing
717 - self.accumulated_params[self.current_param_name] = param_value
718 -
719 - # Get parameter configuration with anyOf support
720 - param_config = {}
721 - if self.streaming_request and self.streaming_request.tools:
722 - for tool in self.streaming_request.tools:
723 - if (
724 - hasattr(tool, "function")
725 - and tool.function.name == self.current_function_name
726 - and hasattr(tool.function, "parameters")
727 - ):
728 - params = tool.function.parameters
729 - if (
730 - isinstance(params, dict)
731 - and "properties" in params
732 - ):
733 - param_config = params["properties"]
734 - break
735 -
736 - # Get parameter types (supports anyOf/oneOf/allOf)
737 - param_type = self._get_param_types_from_config(
738 - self.current_param_name, param_config
739 - )
740 -
741 - converted_value = self._convert_param_value_with_types(
742 - param_value, param_type
743 - )
744 -
745 - # Build JSON fragment based on the converted type
746 - # Use json.dumps to properly serialize the value
747 - serialized_value = json.dumps(
748 - converted_value, ensure_ascii=False
749 - )
750 -
751 - if self.param_count == 0:
752 - json_fragment = (
753 - f'"{self.current_param_name}": {serialized_value}'
754 - )
755 - else:
756 - json_fragment = (
757 - f', "{self.current_param_name}": {serialized_value}'
758 - )
759 -
760 - self.param_count += 1
761 - # Update streamed_args_for_tool for this tool call
762 - if self.current_tool_index < len(self.streamed_args_for_tool):
763 - self.streamed_args_for_tool[self.current_tool_index] += (
764 - json_fragment
765 - )
766 - return DeltaMessage(
767 - tool_calls=[
768 - DeltaToolCall(
769 - index=self.current_tool_index,
770 - function=DeltaFunctionCall(arguments=json_fragment),
771 - )
772 - ]
773 - )
441 + # EOS and </minimax:tool_call> both arrive as special tokens with
442 + # no decoded text. Return non-None for EOS so the serving framework
443 + # reaches the finish-reason handling path instead of skipping.
444 + if (
445 + not delta_text
446 + and delta_token_ids
447 + and self.prev_tool_call_arr
448 + and self.tool_call_end_token_id not in delta_token_ids
449 + ):
450 + return DeltaMessage(content="")
451
452 return None