A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[BugFix] Fix incremental detokenization perf issue (#16963)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Apr 22, 2025 at 01:16 UTC
e4d614423283d8beae4a92aeb537a49ad2662864
1 file changed
+1
-1
vllm/v1/engine/detokenizer.py
+1
-1
@@ -161,7 +161,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
161
prompt_suffix = request.prompt_token_ids
162
prompt_len = len(prompt_suffix)
163
if prompt_len > 4:
164
- for i in range(4, max(prompt_len + 1, 32)):
164
+ for i in range(4, min(prompt_len + 1, 24)):
165
suffix = request.prompt_token_ids[-i:]
166
if '�' not in self.tokenizer.decode(suffix):
167
prompt_suffix = suffix