@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / beac8dd461
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

fix: don't skip first special token. (#1497)

Ricardo Lu committed Oct 29, 2023 at 19:26 UTC beac8dd4615566f153d2d8e8b74e703a4df18ce4
1 file changed +5 -1
vllm/transformers_utils/tokenizer.py
+5 -1
@@ -120,7 +120,11 @@ def detokenize_incrementally(
120 # tokenizers (bigger = more conservative).
121 # Subtract 1 extra to account for the generated token.
122 prefix_offset = max(len(output_tokens) - 6, 0)
123 - read_offset = max(len(output_tokens) - 1, 0)
123 + # If the first new token is a special token, we can't skip 1 extra token
124 + if skip_special_tokens and new_token_id in tokenizer.all_special_ids:
125 + read_offset = max(len(output_tokens), 0)
126 + else:
127 + read_offset = max(len(output_tokens) - 1, 0)
128 else:
129 # Put new_token_id in a list so skip_special_tokens is respected
130 new_tokens = tokenizer.convert_ids_to_tokens(