A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 12:30 UTC
[Frontend] Exploit tokenizers "new stream" in FastIncrementalDetokenizer (#34217)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Feb 11, 2026 at 02:03 UTC
e09546cf05f12c041083c289c24ecb48896f9620
1 file changed
+17
-31
vllm/v1/engine/detokenizer.py
+17
-31
@@ -19,9 +19,9 @@ from vllm.v1.engine import EngineCoreRequest
19
20
logger = init_logger(__name__)
21
22
-# Only tokenizers >= 0.21.1 supports DecodeStream used for
23
-# FastIncrementalDetokenizer.
24
-USE_FAST_DETOKENIZER = version.parse(tokenizers.__version__) >= version.parse("0.21.1")
22
+# Only tokenizers >= 0.22.0 supports DecodeStream with native prefill
23
+# (ids parameter) used for FastIncrementalDetokenizer.
24
+USE_FAST_DETOKENIZER = version.parse(tokenizers.__version__) >= version.parse("0.22.0")
25
26
# Error string from https://github.com/huggingface/tokenizers/blob/909fdde2a4ffedd9295206f705eb612be2a91b12/tokenizers/src/tokenizer/mod.rs#L1042
27
INVALID_PREFIX_ERR_MSG = "Invalid prefix encountered"
@@ -154,11 +154,10 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
154
# We return the full output text if the sequence is finished.
155
buffer_length = 0 if finished else self.stop_buffer_length
156
if not delta:
157
- return (
158
- self.output_text[:-buffer_length]
159
- if buffer_length
160
- else (self.output_text)
161
- )
157
+ if not buffer_length:
158
+ return self.output_text
159
+ return self.output_text[:-buffer_length]
160
+
161
length = len(self.output_text) - buffer_length
162
last_offset = self._last_output_text_offset
163
if last_offset < length:
@@ -176,24 +175,14 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
175
176
self.request_id = request.request_id
177
self.skip_special_tokens = sampling_params.skip_special_tokens
179
- self.stream = DecodeStream(skip_special_tokens=self.skip_special_tokens)
178
179
self.tokenizer: Tokenizer = tokenizer._tokenizer
180
183
- # Find a safe place to start.
184
- prompt_token_ids = request.prompt_token_ids or []
185
- prompt_suffix = prompt_token_ids
186
- prompt_len = len(prompt_suffix)
187
- if prompt_len > 4:
188
- for i in range(4, min(prompt_len + 1, 24)):
189
- suffix = prompt_token_ids[-i:]
190
- if "�" not in self.tokenizer.decode(suffix):
191
- prompt_suffix = suffix
192
- break
193
-
194
- # Prime the stream.
195
- for tid in prompt_suffix:
196
- self._protected_step(tid)
181
+ # Use native prefill to prime the decode stream with prompt tokens.
182
+ self.stream = DecodeStream(
183
+ ids=request.prompt_token_ids,
184
+ skip_special_tokens=self.skip_special_tokens,
185
+ )
186
187
self.spaces_between_special_tokens = (
188
sampling_params.skip_special_tokens
@@ -203,9 +192,8 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
192
if not self.spaces_between_special_tokens:
193
# Store dict of added token ids so that we can suppress
194
# the spaces between them.
206
- if (
207
- added_token_ids := getattr(self.tokenizer, "added_token_ids", None)
208
- ) is None:
195
+ added_token_ids = getattr(self.tokenizer, "added_token_ids", None)
196
+ if added_token_ids is None:
197
self.tokenizer.added_token_ids = added_token_ids = {
198
tid: tok.content
199
for tid, tok in self.tokenizer.get_added_tokens_decoder().items()
@@ -290,11 +278,9 @@ class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
278
279
@property
280
def output_token_ids(self) -> list[int]:
293
- return (
294
- self.token_ids
295
- if not self.prompt_len
296
- else (self.token_ids[self.prompt_len :])
297
- )
281
+ if self.prompt_len:
282
+ return self.token_ids[self.prompt_len :]
283
+ return self.token_ids
284
285
def num_output_tokens(self) -> int:
286
return len(self.token_ids) - self.prompt_len