A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 15:00 UTC
[Bugfix] Fix the DSML token leakage in DSV4/3.2 (#40806)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Windswithyou 1694599440@qq.com
Chauncey committed
Apr 26, 2026 at 08:58 UTC
c798593f0d88cec583c599ea7ea40a2cc26c312b
2 files changed
+76
-23
tests/tool_parsers/test_deepseekv32_tool_parser.py
+52
@@ -484,6 +484,58 @@ class TestExtractToolCallsStreaming:
484
# Should have no tool call deltas yet
485
assert all(not d.tool_calls for d in deltas)
486
487
+ def test_no_marker_leak_chunked(self, parser):
488
+ """Chunked streaming must NOT leak DSML start-marker fragments
489
+ as content (GitHub #40801)."""
490
+ full_text = build_tool_call("fn", {"k": "v"})
491
+ deltas = self._stream_chunked(parser, full_text, chunk_size=5)
492
+ content = "".join(d.content for d in deltas if d.content is not None)
493
+ assert content == ""
494
+ args_str = self._reconstruct_args(deltas)
495
+ assert json.loads(args_str) == {"k": "v"}
496
+
497
+ def test_no_marker_leak_with_prefix_chunked(self, parser):
498
+ """Content before a tool call must not include start-marker
499
+ fragments when chunked (GitHub #40801)."""
500
+ full_text = "Hello!" + build_tool_call("fn", {"a": "b"})
501
+ deltas = self._stream_chunked(parser, full_text, chunk_size=5)
502
+ content = "".join(d.content for d in deltas if d.content is not None)
503
+ assert content == "Hello!"
504
+ assert "DSML" not in content
505
+ assert "<｜" not in content
506
+ args_str = self._reconstruct_args(deltas)
507
+ assert json.loads(args_str) == {"a": "b"}
508
+
509
+ def test_no_marker_leak_char_by_char(self, parser):
510
+ """Character-by-character streaming must not leak marker
511
+ fragments (GitHub #40801)."""
512
+ full_text = build_tool_call("fn", {"k": "v"})
513
+ deltas = self._stream_chunked(parser, full_text, chunk_size=1)
514
+ content = "".join(d.content for d in deltas if d.content is not None)
515
+ assert content == ""
516
+ args_str = self._reconstruct_args(deltas)
517
+ assert json.loads(args_str) == {"k": "v"}
518
+
519
+ def test_no_marker_leak_all_split_points(self, parser):
520
+ """Start token split at every possible boundary must not
521
+ leak (GitHub #40801)."""
522
+ for chunk_size in range(1, len(FC_START) + 2):
523
+ p = make_parser()
524
+ full_text = build_tool_call("fn", {"k": "v"})
525
+ deltas = self._stream_chunked(p, full_text, chunk_size=chunk_size)
526
+ content = "".join(d.content for d in deltas if d.content is not None)
527
+ assert content == "", (
528
+ f"Leaked content {content!r} at chunk_size={chunk_size}"
529
+ )
530
+
531
+ def test_false_partial_marker_emitted(self, parser):
532
+ """Text ending with a prefix of the start token that turns out
533
+ NOT to be a marker must still be emitted as content."""
534
+ full_text = "<｜DSM some regular text"
535
+ deltas = self._stream_chunked(parser, full_text, chunk_size=3)
536
+ content = "".join(d.content for d in deltas if d.content is not None)
537
+ assert content == full_text
538
+
539
540
class TestDelimiterPreservation:
541
"""Regression: fast detokenization skipping DSML delimiters (PR #33964)."""
vllm/tool_parsers/deepseekv32_tool_parser.py
+24
-23
@@ -26,6 +26,7 @@ from vllm.tool_parsers.abstract_tool_parser import (
26
Tool,
27
ToolParser,
28
)
29
+from vllm.tool_parsers.utils import partial_tag_overlap
30
31
logger = init_logger(__name__)
32
@@ -54,8 +55,8 @@ class DeepSeekV32ToolParser(ToolParser):
55
self.tool_call_start_token: str = "<｜DSML｜function_calls>"
56
57
# Streaming state
57
- self.is_tool_call_started: bool = False
58
self.current_tool_index: int = 0
59
+ self._sent_content_idx: int = 0
60
61
# Regex patterns for complete parsing
62
self.tool_call_complete_regex = re.compile(
@@ -219,7 +220,7 @@ class DeepSeekV32ToolParser(ToolParser):
220
def _reset_streaming_state(self):
221
"""Reset all streaming state."""
222
self.current_tool_index = 0
222
- self.is_tool_call_started = False
223
+ self._sent_content_idx = 0
224
self.prev_tool_call_arr.clear()
225
self.streamed_args_for_tool.clear()
226
@@ -264,6 +265,24 @@ class DeepSeekV32ToolParser(ToolParser):
265
266
return delta_tool_calls
267
268
+ def _extract_content(self, current_text: str) -> str | None:
269
+ """Return unsent non-tool-call text, or None.
270
+
271
+ Holds back any suffix that could be a partial start marker
272
+ so that split markers are never leaked as content.
273
+ """
274
+ if self.tool_call_start_token not in current_text:
275
+ overlap = partial_tag_overlap(current_text, self.tool_call_start_token)
276
+ sendable_idx = len(current_text) - overlap
277
+ else:
278
+ sendable_idx = current_text.index(self.tool_call_start_token)
279
+
280
+ if sendable_idx > self._sent_content_idx:
281
+ content = current_text[self._sent_content_idx : sendable_idx]
282
+ self._sent_content_idx = sendable_idx
283
+ return content
284
+ return None
285
+
286
def extract_tool_calls_streaming(
287
self,
288
previous_text: str,
@@ -285,29 +304,11 @@ class DeepSeekV32ToolParser(ToolParser):
304
if not previous_text:
305
self._reset_streaming_state()
306
288
- # Detect whether we've entered the tool-call region.
289
- # Use current_text (not delta_text) since the start token may
290
- # be split across chunks.
291
- content_before = None
292
- if self.is_tool_call_started:
293
- pass
294
- elif self.tool_call_start_token in current_text:
295
- # Tool-call region found, capture any plain text before it.
296
- self.is_tool_call_started = True
297
- start_idx = current_text.index(self.tool_call_start_token)
298
- content_before = current_text[len(previous_text) : start_idx] or None
299
- else:
300
- # Still in plain-text region, forward as content.
301
- return DeltaMessage(content=delta_text) if delta_text else None
302
-
303
- # Inside tool-call region: emit any newly completed invokes.
307
+ content = self._extract_content(current_text)
308
delta_tool_calls = self._extract_delta_tool_calls(current_text, request)
309
306
- if delta_tool_calls or content_before:
307
- return DeltaMessage(
308
- content=content_before,
309
- tool_calls=delta_tool_calls,
310
- )
310
+ if delta_tool_calls or content:
311
+ return DeltaMessage(content=content, tool_calls=delta_tool_calls)
312
313
# Empty delta with token ids means EOS or closing tag; return
314
# non-None so the serving framework can finalize finish_reason.