A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Improve detokenization performance (#1338)
Antoni Baum committed
Oct 13, 2023 at 09:59 UTC
ec3b5ce9ccb4262194a16a8b1c31ffd6b3b824b9
1 file changed
+4
-3
vllm/transformers_utils/tokenizer.py
+4
-3
@@ -81,10 +81,11 @@ def _convert_tokens_to_string_with_added_encoders(
81
# even when the loop body is very simple.
82
sub_texts = []
83
current_sub_text = []
84
+ all_special_tokens = set(tokenizer.all_special_tokens)
85
for token in output_tokens:
85
- if skip_special_tokens and token in tokenizer.all_special_tokens:
86
+ if skip_special_tokens and token in all_special_tokens:
87
continue
87
- if token in tokenizer.added_tokens_encoder:
88
+ if token in tokenizer.get_added_vocab():
89
if current_sub_text:
90
sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
91
sub_texts.append(sub_text)
@@ -129,7 +130,7 @@ def detokenize_incrementally(
130
# The prefix text is necessary only to defeat cleanup algorithms in
131
# the decode which decide to add a space or not depending on the
132
# surrounding ids.
132
- if not getattr(tokenizer, "added_tokens_encoder", {}):
133
+ if tokenizer.is_fast or not tokenizer.get_added_vocab():
134
prefix_text = tokenizer.convert_tokens_to_string(
135
output_tokens[prefix_offset:read_offset])
136
new_text = tokenizer.convert_tokens_to_string(