@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ec3b5ce9cc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Improve detokenization performance (#1338)

Antoni Baum committed Oct 13, 2023 at 09:59 UTC ec3b5ce9ccb4262194a16a8b1c31ffd6b3b824b9
1 file changed +4 -3
vllm/transformers_utils/tokenizer.py
+4 -3
@@ -81,10 +81,11 @@ def _convert_tokens_to_string_with_added_encoders(
81 # even when the loop body is very simple.
82 sub_texts = []
83 current_sub_text = []
84 + all_special_tokens = set(tokenizer.all_special_tokens)
85 for token in output_tokens:
85 - if skip_special_tokens and token in tokenizer.all_special_tokens:
86 + if skip_special_tokens and token in all_special_tokens:
87 continue
87 - if token in tokenizer.added_tokens_encoder:
88 + if token in tokenizer.get_added_vocab():
89 if current_sub_text:
90 sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
91 sub_texts.append(sub_text)
@@ -129,7 +130,7 @@ def detokenize_incrementally(
130 # The prefix text is necessary only to defeat cleanup algorithms in
131 # the decode which decide to add a space or not depending on the
132 # surrounding ids.
132 - if not getattr(tokenizer, "added_tokens_encoder", {}):
133 + if tokenizer.is_fast or not tokenizer.get_added_vocab():
134 prefix_text = tokenizer.convert_tokens_to_string(
135 output_tokens[prefix_offset:read_offset])
136 new_text = tokenizer.convert_tokens_to_string(