A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 12:15 UTC
[Bugfix] Fix invalid JSON in Gemma 4 streaming tool calls by stripping partial delimiters (#38992)
Signed-off-by: greg pereira <grpereir@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Greg Pereira committed
Apr 5, 2026 at 10:11 UTC
f53fa26e05c476a43f6db048a9e3b43bcb2b72fb
2 files changed
+33
-3
tests/tool_parsers/test_gemma4_tool_parser.py
+29
@@ -502,3 +502,32 @@ class TestStreamingExtraction:
502
results = self._simulate_streaming(parser, mock_request, chunks)
503
name = self._collect_function_name(results)
504
assert name == "get_status"
505
+
506
+ def test_streaming_split_delimiter_no_invalid_json(self, parser, mock_request):
507
+ """Partial <|"|> delimiter chars must not leak into streamed JSON.
508
+
509
+ Reproduces the bug from https://github.com/vllm-project/vllm/issues/38946
510
+ where a token boundary splits the string delimiter, leaving fragments
511
+ like '<|' at the end of a parsed value which then corrupt the JSON.
512
+ """
513
+ chunks = [
514
+ "<|tool_call>",
515
+ "call:todowrite{",
516
+ 'content:<|"|>Buy milk<|',
517
+ '"|>}',
518
+ "<tool_call|>",
519
+ ]
520
+
521
+ results = self._simulate_streaming(parser, mock_request, chunks)
522
+
523
+ args_text = self._collect_arguments(results)
524
+ assert args_text, "No arguments were streamed"
525
+
526
+ # Must be valid JSON — the original bug caused a JSON parse error
527
+ parsed_args = json.loads(args_text)
528
+ assert parsed_args["content"] == "Buy milk"
529
+
530
+ # Ensure no raw delimiter fragments leaked into the JSON
531
+ assert "<|" not in args_text, (
532
+ f"Partial delimiter leaked into JSON: {args_text!r}"
533
+ )
vllm/tool_parsers/gemma4_tool_parser.py
+4
-3
@@ -675,10 +675,11 @@ class Gemma4ToolParser(ToolParser):
675
current_args_json = json.dumps(current_args, ensure_ascii=False)
676
677
# Withhold trailing closing characters that may shift as more
678
- # tokens arrive. Strip trailing '}', '"', and ']' sequences
679
- # to get the "safe prefix".
678
+ # tokens arrive. Strip trailing '}', '"', ']' and partial
679
+ # STRING_DELIM fragments ('<', '|', '\\', '>') to get the
680
+ # "safe prefix".
681
safe_json = current_args_json
681
- while safe_json and safe_json[-1] in ("}", '"', "]"):
682
+ while safe_json and safe_json[-1] in ("}", '"', "]", "<", "|", "\\", ">"):
683
safe_json = safe_json[:-1]
684
685
prev_streamed = self.streamed_args_for_tool[self.current_tool_id]