A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 15:15 UTC
[Perf] Optimize detokenizer python logic (#32975)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Nick Hill <nhill@redhat.com>
Wentao Ye committed
Feb 10, 2026 at 02:54 UTC
e1060a71a1bb96103ce9ca98345184dcdc982467
2 files changed
+10
-6
vllm/v1/engine/detokenizer.py
+8
-4
@@ -35,6 +35,9 @@ class IncrementalDetokenizer:
35
def output_token_ids(self) -> list[int]:
36
return self.token_ids
37
38
+ def num_output_tokens(self) -> int:
39
+ return len(self.token_ids)
40
+
41
def update(self, new_token_ids: list[int], stop_terminated: bool) -> str | None:
42
self.token_ids.extend(new_token_ids)
43
return None
@@ -112,14 +115,12 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
115
skipped_stop_token_id = None
116
117
# 1) Detokenize the new token ids incrementally.
115
- # TODO(woosuk): This method becomes very inefficient when the number of
116
- # new_token_ids is more than 1. We need to optimize this.
118
stop_check_offset = len(self.output_text)
119
for new_token_id in new_token_ids:
120
self.token_ids.append(new_token_id)
121
self.output_text += self.decode_next(new_token_id)
122
# Support min_tokens, see https://github.com/vllm-project/vllm/pull/22014
122
- if self.min_tokens and len(self.output_token_ids) <= self.min_tokens:
123
+ if self.min_tokens and self.num_output_tokens() <= self.min_tokens:
124
stop_check_offset = len(self.output_text)
125
126
if skipped_stop_token_id is not None:
@@ -128,7 +129,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
129
130
# 2) Evaluate stop strings.
131
stop_string = None
131
- if self.stop and len(self.output_token_ids) > self.min_tokens:
132
+ if self.stop and self.num_output_tokens() > self.min_tokens:
133
stop = check_stop_strings(
134
output_text=self.output_text,
135
new_char_count=len(self.output_text) - stop_check_offset,
@@ -295,6 +296,9 @@ class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
296
else (self.token_ids[self.prompt_len :])
297
)
298
299
+ def num_output_tokens(self) -> int:
300
+ return len(self.token_ids) - self.prompt_len
301
+
302
def decode_next(self, next_token_id: int) -> str:
303
new_tokens, decoded_text, prefix_offset, read_offset = detokenize_incrementally(
304
tokenizer=self.tokenizer,
vllm/v1/engine/output_processor.py
+2
-2
@@ -292,7 +292,7 @@ class RequestState:
292
if not (
293
finished
294
or self.sent_tokens_offset == 0
295
- or len(self.detokenizer.output_token_ids) - self.sent_tokens_offset
295
+ or self.detokenizer.num_output_tokens() - self.sent_tokens_offset
296
>= self.stream_interval
297
):
298
return None
@@ -303,7 +303,7 @@ class RequestState:
303
new_token_ids = self.detokenizer.output_token_ids[
304
self.sent_tokens_offset :
305
]
306
- self.sent_tokens_offset = len(self.detokenizer.output_token_ids)
306
+ self.sent_tokens_offset = self.detokenizer.num_output_tokens()
307
308
external_req_id = self.external_req_id
309