A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
fix: don't skip first special token. (#1497)
Ricardo Lu committed
Oct 29, 2023 at 19:26 UTC
beac8dd4615566f153d2d8e8b74e703a4df18ce4
1 file changed
+5
-1
vllm/transformers_utils/tokenizer.py
+5
-1
@@ -120,7 +120,11 @@ def detokenize_incrementally(
120
# tokenizers (bigger = more conservative).
121
# Subtract 1 extra to account for the generated token.
122
prefix_offset = max(len(output_tokens) - 6, 0)
123
- read_offset = max(len(output_tokens) - 1, 0)
123
+ # If the first new token is a special token, we can't skip 1 extra token
124
+ if skip_special_tokens and new_token_id in tokenizer.all_special_ids:
125
+ read_offset = max(len(output_tokens), 0)
126
+ else:
127
+ read_offset = max(len(output_tokens) - 1, 0)
128
else:
129
# Put new_token_id in a list so skip_special_tokens is respected
130
new_tokens = tokenizer.convert_ids_to_tokens(