A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[Bugfix] Fix Gemma4 streaming tool call corruption for split boolean/number values (#39114)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Apr 8, 2026 at 12:46 UTC
13151a4df43decadce2e962078635529f954d9d8
2 files changed
+78
-8
tests/tool_parsers/test_gemma4_tool_parser.py
+45
@@ -491,6 +491,51 @@ class TestStreamingExtraction:
491
assert parsed_args["count"] == 42
492
assert parsed_args["active"] is True
493
494
+ def test_streaming_boolean_split_across_chunks(self, parser, mock_request):
495
+ """Boolean value split across token boundaries must not corrupt JSON."""
496
+ chunks = [
497
+ "<|tool_call>",
498
+ "call:search{input:{all:" + "true"[:3],
499
+ "e}}",
500
+ "<tool_call|>",
501
+ ]
502
+
503
+ results = self._simulate_streaming(parser, mock_request, chunks)
504
+ args_text = self._collect_arguments(results)
505
+ assert args_text, "No arguments were streamed"
506
+ parsed_args = json.loads(args_text)
507
+ assert parsed_args["input"]["all"] is True
508
+
509
+ def test_streaming_false_split_across_chunks(self, parser, mock_request):
510
+ """Boolean false split across chunks."""
511
+ chunks = [
512
+ "<|tool_call>",
513
+ "call:set{flag:" + "false"[:4],
514
+ "e}",
515
+ "<tool_call|>",
516
+ ]
517
+
518
+ results = self._simulate_streaming(parser, mock_request, chunks)
519
+ args_text = self._collect_arguments(results)
520
+ assert args_text, "No arguments were streamed"
521
+ parsed_args = json.loads(args_text)
522
+ assert parsed_args["flag"] is False
523
+
524
+ def test_streaming_number_split_across_chunks(self, parser, mock_request):
525
+ """Number split across chunks must not change type."""
526
+ chunks = [
527
+ "<|tool_call>",
528
+ "call:set{count:4",
529
+ "2}",
530
+ "<tool_call|>",
531
+ ]
532
+
533
+ results = self._simulate_streaming(parser, mock_request, chunks)
534
+ args_text = self._collect_arguments(results)
535
+ assert args_text, "No arguments were streamed"
536
+ parsed_args = json.loads(args_text)
537
+ assert parsed_args["count"] == 42
538
+
539
def test_streaming_empty_args(self, parser, mock_request):
540
"""Tool call with no arguments."""
541
chunks = [
vllm/tool_parsers/gemma4_tool_parser.py
+33
-8
@@ -78,7 +78,7 @@ def _parse_gemma4_value(value_str: str) -> object:
78
return value_str
79
80
81
-def _parse_gemma4_args(args_str: str) -> dict:
81
+def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
82
"""Parse Gemma4's custom key:value format into a Python dict.
83
84
Format examples::
@@ -89,6 +89,12 @@ def _parse_gemma4_args(args_str: str) -> dict:
89
nested:{inner_key:<|"|>val<|"|>}
90
items:[<|"|>a<|"|>,<|"|>b<|"|>]
91
92
+ Args:
93
+ args_str: The raw Gemma4 argument string.
94
+ partial: When True (streaming), bare values at end of string are
95
+ omitted because they may be incomplete and type-unstable
96
+ (e.g. partial boolean parsed as bare string).
97
+
98
Returns a dict ready for ``json.dumps()``.
99
"""
100
if not args_str or not args_str.strip():
@@ -155,7 +161,12 @@ def _parse_gemma4_args(args_str: str) -> dict:
161
elif args_str[i] == "}":
162
depth -= 1
163
i += 1
158
- result[key] = _parse_gemma4_args(args_str[obj_start : i - 1])
164
+ if depth > 0:
165
+ # Incomplete nested object — use i (not i-1) to avoid
166
+ # dropping the last char, and recurse as partial.
167
+ result[key] = _parse_gemma4_args(args_str[obj_start:i], partial=True)
168
+ else:
169
+ result[key] = _parse_gemma4_args(args_str[obj_start : i - 1])
170
171
# Array: [...]
172
elif args_str[i] == "[":
@@ -173,20 +184,26 @@ def _parse_gemma4_args(args_str: str) -> dict:
184
elif args_str[i] == "]":
185
depth -= 1
186
i += 1
176
- arr_content = args_str[arr_start : i - 1]
177
- result[key] = _parse_gemma4_array(arr_content)
187
+ if depth > 0:
188
+ result[key] = _parse_gemma4_array(args_str[arr_start:i], partial=True)
189
+ else:
190
+ result[key] = _parse_gemma4_array(args_str[arr_start : i - 1])
191
192
# Bare value (number, boolean, etc.)
193
else:
194
val_start = i
195
while i < n and args_str[i] not in (",", "}", "]"):
196
i += 1
197
+ if partial and i >= n:
198
+ # Value may be incomplete (e.g. partial boolean) —
199
+ # withhold to avoid type instability during streaming.
200
+ break
201
result[key] = _parse_gemma4_value(args_str[val_start:i])
202
203
return result
204
205
189
-def _parse_gemma4_array(arr_str: str) -> list:
206
+def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
207
"""Parse a Gemma4 array content string into a Python list."""
208
items: list = []
209
i = 0
@@ -224,7 +241,10 @@ def _parse_gemma4_array(arr_str: str) -> list:
241
elif arr_str[i] == "}":
242
depth -= 1
243
i += 1
227
- items.append(_parse_gemma4_args(arr_str[obj_start : i - 1]))
244
+ if depth > 0:
245
+ items.append(_parse_gemma4_args(arr_str[obj_start:i], partial=True))
246
+ else:
247
+ items.append(_parse_gemma4_args(arr_str[obj_start : i - 1]))
248
249
# Nested array
250
elif arr_str[i] == "[":
@@ -237,13 +257,18 @@ def _parse_gemma4_array(arr_str: str) -> list:
257
elif arr_str[i] == "]":
258
depth -= 1
259
i += 1
240
- items.append(_parse_gemma4_array(arr_str[sub_start : i - 1]))
260
+ if depth > 0:
261
+ items.append(_parse_gemma4_array(arr_str[sub_start:i], partial=True))
262
+ else:
263
+ items.append(_parse_gemma4_array(arr_str[sub_start : i - 1]))
264
265
# Bare value
266
else:
267
val_start = i
268
while i < n and arr_str[i] not in (",", "]"):
269
i += 1
270
+ if partial and i >= n:
271
+ break
272
items.append(_parse_gemma4_value(arr_str[val_start:i]))
273
274
return items
@@ -663,7 +688,7 @@ class Gemma4ToolParser(ToolParser):
688
DeltaMessage with the argument diff, or None if no new content.
689
"""
690
try:
666
- current_args = _parse_gemma4_args(raw_args_str)
691
+ current_args = _parse_gemma4_args(raw_args_str, partial=True)
692
except Exception:
693
logger.debug(
694
"Could not parse partial Gemma4 args yet: %s",