@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c6dfc3cdbe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Fix handling of special tokens in decoding. (#418)

xcnick committed Jul 12, 2023 at 23:14 UTC c6dfc3cdbeb4687a8313b82d399177e293fee196
2 files changed +6 -3
vllm/engine/llm_engine.py
+3 -2
@@ -276,8 +276,9 @@ class LLMEngine:
276 seq.get_last_token_id(),
277 skip_special_tokens=True,
278 )
279 - seq.output_tokens.append(new_token)
280 - seq.output_text = new_output_text
279 + if new_token is not None:
280 + seq.output_tokens.append(new_token)
281 + seq.output_text = new_output_text
282
283 def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
284 """Stop the finished sequences."""
vllm/transformers_utils/tokenizer.py
+3 -1
@@ -80,6 +80,8 @@ def detokenize_incrementally(
80 new_token: The new token as a string.
81 output_text: The new output text as a string.
82 """
83 + if skip_special_tokens and (new_token_id in tokenizer.all_special_ids):
84 + return None, prev_output_tokens
85 new_token = tokenizer.convert_ids_to_tokens(
86 new_token_id, skip_special_tokens=skip_special_tokens)
87 output_tokens = prev_output_tokens + [new_token]
@@ -99,7 +101,7 @@ def detokenize_incrementally(
101 sub_texts = []
102 current_sub_text = []
103 for token in output_tokens:
102 - if skip_special_tokens and token in tokenizer.all_special_ids:
104 + if skip_special_tokens and token in tokenizer.all_special_tokens:
105 continue
106 if token in tokenizer.added_tokens_encoder:
107 if current_sub_text: