A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V1] support min_tokens for detokener (#22014)
Signed-off-by: calvin chen <wen.chen@dynamia.ai> Co-authored-by: Nick Hill <nhill@redhat.com>
Calvin Chen committed
Aug 16, 2025 at 10:28 UTC
e4e37ded563912d6d413cfc23cc1db098b3d2f09
2 files changed
+58
-3
tests/detokenizer/test_min_tokens.py
new
+50
@@ -0,0 +1,50 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+from transformers import AutoTokenizer
6
+
7
+from vllm import SamplingParams
8
+from vllm.v1.engine import EngineCoreRequest
9
+from vllm.v1.engine.detokenizer import FastIncrementalDetokenizer
10
+
11
+PROMPT = "Hello, my name is Lee, and I'm a student in the " + \
12
+ "college of engineering"
13
+
14
+
15
+@pytest.mark.parametrize("min_tokens,stop,truth", [
16
+ (0, None, " is Lee, and I'm a student in the college of engineering"),
17
+ (0, "e", " is L"),
18
+ (5, "e", " is Lee, and I'm a stud"),
19
+])
20
+def test_min_tokens_with_stop(min_tokens: int, stop: str, truth: str):
21
+ """Test for a specific min_tokens and stop.
22
+
23
+ See https://github.com/vllm-project/vllm/pull/22014
24
+ """
25
+ tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m")
26
+ all_prompt_ids = tokenizer(PROMPT, add_special_tokens=False).input_ids
27
+
28
+ # The prompt is "Hello, my name is"
29
+ prompt_token_ids = all_prompt_ids[:4]
30
+ params = SamplingParams(
31
+ stop=stop,
32
+ min_tokens=min_tokens,
33
+ )
34
+ request = EngineCoreRequest("",
35
+ prompt_token_ids,
36
+ None,
37
+ None,
38
+ None,
39
+ params,
40
+ None,
41
+ None,
42
+ 0.0,
43
+ None,
44
+ cache_salt=None,
45
+ data_parallel_rank=None)
46
+
47
+ detokenizer = FastIncrementalDetokenizer(tokenizer, request)
48
+
49
+ detokenizer.update(all_prompt_ids[4:], False)
50
+ assert detokenizer.output_text == truth
vllm/v1/engine/detokenizer.py
+8
-3
@@ -74,6 +74,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
74
params = request.sampling_params
75
assert params is not None
76
self.stop = stop = params.stop
77
+ self.min_tokens = params.min_tokens
78
self.include_stop_str_in_output = params.include_stop_str_in_output
79
80
# Number of chars to hold back when stop strings are to be excluded
@@ -111,10 +112,14 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
112
# 1) Detokenize the new token ids incrementally.
113
# TODO(woosuk): This method becomes very inefficient when the number of
114
# new_token_ids is more than 1. We need to optimize this.
114
- offset_before = len(self.output_text)
115
+ stop_check_offset = len(self.output_text)
116
for new_token_id in new_token_ids:
117
self.token_ids.append(new_token_id)
118
self.output_text += self.decode_next(new_token_id)
119
+ # Support min_tokens, see https://github.com/vllm-project/vllm/pull/22014
120
+ if self.min_tokens and len(
121
+ self.output_token_ids) <= self.min_tokens:
122
+ stop_check_offset = len(self.output_text)
123
124
if stop_terminated:
125
if skipped_stop_token_id is not None:
@@ -125,10 +130,10 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
130
131
# 2) Evaluate stop strings.
132
stop_string = None
128
- if self.stop:
133
+ if self.stop and len(self.output_token_ids) > self.min_tokens:
134
stop = StopChecker.check_stop_strings(
135
output_text=self.output_text,
131
- new_char_count=len(self.output_text) - offset_before,
136
+ new_char_count=len(self.output_text) - stop_check_offset,
137
stop=self.stop,
138
include_in_output=self.include_stop_str_in_output,
139
)