A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[Bugfix][Frontend] Fix Gemma4 streaming HTML duplication after tool calls (#38909)
Signed-off-by: yoke233 <yoke2012@gmail.com>
yoke committed
Apr 8, 2026 at 11:03 UTC
d734445fcd79d31fbe039c072ba08128d71f43f9
2 files changed
+64
-2
tests/tool_parsers/test_gemma4_tool_parser.py
+60
@@ -531,3 +531,63 @@ class TestStreamingExtraction:
531
assert "<|" not in args_text, (
532
f"Partial delimiter leaked into JSON: {args_text!r}"
533
)
534
+
535
+ def test_streaming_does_not_duplicate_plain_text_after_tool_call(
536
+ self, parser, mock_request, monkeypatch
537
+ ):
538
+ """Buffered plain text after a tool call must not corrupt current_text."""
539
+ captured_current_texts: list[str] = []
540
+ original_extract_streaming = parser._extract_streaming
541
+
542
+ def wrapped_extract_streaming(previous_text, current_text, delta_text):
543
+ captured_current_texts.append(current_text)
544
+ return original_extract_streaming(previous_text, current_text, delta_text)
545
+
546
+ monkeypatch.setattr(parser, "_extract_streaming", wrapped_extract_streaming)
547
+
548
+ chunks = [
549
+ "<|tool_call>",
550
+ "call:get_weather{",
551
+ 'location:<|"|>Paris<|"|>}',
552
+ "<tool_call|><",
553
+ "div>",
554
+ ]
555
+
556
+ results = self._simulate_streaming(parser, mock_request, chunks)
557
+ content_parts = [
558
+ delta.content for delta, _ in results if delta is not None and delta.content
559
+ ]
560
+ assert "".join(content_parts) == "<div>"
561
+ assert captured_current_texts[-1].endswith("<tool_call|><div>")
562
+ assert not captured_current_texts[-1].endswith("<tool_call|><<div>")
563
+
564
+ def test_streaming_html_argument_does_not_duplicate_tag_prefixes(
565
+ self, parser, mock_request
566
+ ):
567
+ """HTML content inside tool arguments must not be duplicated."""
568
+ chunks = [
569
+ "<|tool_call>",
570
+ "call:write_file{",
571
+ 'path:<|"|>index.html<|"|>,',
572
+ 'content:<|"|><!DOCTYPE html>\n<',
573
+ 'html lang="zh-CN">\n<',
574
+ "head>\n <",
575
+ 'meta charset="UTF-8">\n <',
576
+ 'meta name="viewport" content="width=device-width">\n',
577
+ '<|"|>}',
578
+ "<tool_call|>",
579
+ ]
580
+
581
+ results = self._simulate_streaming(parser, mock_request, chunks)
582
+ args_text = self._collect_arguments(results)
583
+ assert args_text
584
+
585
+ parsed_args = json.loads(args_text)
586
+ assert parsed_args["path"] == "index.html"
587
+ assert (
588
+ parsed_args["content"] == "<!DOCTYPE html>\n"
589
+ '<html lang="zh-CN">\n'
590
+ "<head>\n"
591
+ ' <meta charset="UTF-8">\n'
592
+ ' <meta name="viewport" content="width=device-width">\n'
593
+ )
vllm/tool_parsers/gemma4_tool_parser.py
+4
-2
@@ -436,8 +436,10 @@ class Gemma4ToolParser(ToolParser):
436
) -> DeltaMessage | None:
437
# Buffer delta text to handle multi-token special sequences
438
delta_text = self._buffer_delta_text(delta_text)
439
- # Reconstruct current_text after buffering to stay in sync
440
- current_text = previous_text + delta_text
439
+ # Keep current_text from the upstream stream state. The buffered delta
440
+ # is only for emission, and must not be stitched back into the
441
+ # accumulated model text or normal content like "<div>" can be
442
+ # duplicated into "<<div>" when a tool call just ended.
443
444
# If no tool call token seen yet, emit as content
445
if self.tool_call_start_token not in current_text: