A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Fix handling of special tokens in decoding. (#418)
xcnick committed
Jul 12, 2023 at 23:14 UTC
c6dfc3cdbeb4687a8313b82d399177e293fee196
2 files changed
+6
-3
vllm/engine/llm_engine.py
+3
-2
@@ -276,8 +276,9 @@ class LLMEngine:
276
seq.get_last_token_id(),
277
skip_special_tokens=True,
278
)
279
- seq.output_tokens.append(new_token)
280
- seq.output_text = new_output_text
279
+ if new_token is not None:
280
+ seq.output_tokens.append(new_token)
281
+ seq.output_text = new_output_text
282
283
def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
284
"""Stop the finished sequences."""
vllm/transformers_utils/tokenizer.py
+3
-1
@@ -80,6 +80,8 @@ def detokenize_incrementally(
80
new_token: The new token as a string.
81
output_text: The new output text as a string.
82
"""
83
+ if skip_special_tokens and (new_token_id in tokenizer.all_special_ids):
84
+ return None, prev_output_tokens
85
new_token = tokenizer.convert_ids_to_tokens(
86
new_token_id, skip_special_tokens=skip_special_tokens)
87
output_tokens = prev_output_tokens + [new_token]
@@ -99,7 +101,7 @@ def detokenize_incrementally(
101
sub_texts = []
102
current_sub_text = []
103
for token in output_tokens:
102
- if skip_special_tokens and token in tokenizer.all_special_ids:
104
+ if skip_special_tokens and token in tokenizer.all_special_tokens:
105
continue
106
if token in tokenizer.added_tokens_encoder:
107
if current_sub_text: