@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 49782fcb76
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Misc] Some minor simplifications to detokenization logic (#3670)

Some simplifications made for clarity. Also moves detokenization-related functions from tokenizer.py to detokenizer.py.

Nick Hill committed Apr 1, 2024 at 13:22 UTC 49782fcb769eb4f04a3cf5179c1e6c13ab633ce1
3 files changed +159 -165
tests/tokenization/test_detokenize.py
+2 -2
@@ -4,8 +4,8 @@ import pytest
4 from transformers import AutoTokenizer
5
6 from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
7 -from vllm.transformers_utils.detokenizer import Detokenizer
8 -from vllm.transformers_utils.tokenizer import detokenize_incrementally
7 +from vllm.transformers_utils.detokenizer import (Detokenizer,
8 + detokenize_incrementally)
9 from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
10
11 TRUTH = [
vllm/transformers_utils/detokenizer.py
+156 -8
@@ -1,10 +1,8 @@
1 -from typing import Dict, List, Optional
1 +from typing import Dict, List, Optional, Tuple, Union
2
3 -from transformers import PreTrainedTokenizer
3 +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
4
5 from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
6 -from vllm.transformers_utils.tokenizer import (convert_prompt_ids_to_tokens,
7 - detokenize_incrementally)
6 from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
7 BaseTokenizerGroup)
8
@@ -148,10 +146,160 @@ class Detokenizer:
146 )
147 sample_logprob.decoded_token = new_text
148
151 - if seq.tokens is None:
152 - seq.tokens = new_tokens
153 - else:
154 - seq.tokens.extend(new_tokens)
149 + seq.tokens.extend(new_tokens)
150 seq.prefix_offset = prefix_offset
151 seq.read_offset = read_offset
152 seq.output_text += new_decoded_token_text
153 +
154 +
155 +def _convert_tokens_to_string_with_added_encoders(
156 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
157 + output_tokens: List[str],
158 + skip_special_tokens: bool,
159 + spaces_between_special_tokens: bool,
160 +) -> str:
161 + # Adapted from
162 + # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
163 + # NOTE(woosuk): The following code is slow because it runs a for loop over
164 + # the output_tokens. In Python, running a for loop over a list can be slow
165 + # even when the loop body is very simple.
166 + sub_texts = []
167 + current_sub_text = []
168 + all_special_tokens = set(tokenizer.all_special_tokens)
169 + for token in output_tokens:
170 + if skip_special_tokens and token in all_special_tokens:
171 + continue
172 + if token in tokenizer.get_added_vocab():
173 + if current_sub_text:
174 + sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
175 + sub_texts.append(sub_text)
176 + current_sub_text = []
177 + sub_texts.append(token)
178 + else:
179 + current_sub_text.append(token)
180 + if current_sub_text:
181 + sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
182 + sub_texts.append(sub_text)
183 + if spaces_between_special_tokens:
184 + return " ".join(sub_texts)
185 + else:
186 + return "".join(sub_texts)
187 +
188 +
189 +# 5 is an arbitrary value that should work for all
190 +# tokenizers (bigger = more conservative).
191 +INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
192 +
193 +
194 +def convert_prompt_ids_to_tokens(
195 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
196 + prompt_ids: List[int],
197 + skip_special_tokens: bool = False,
198 +) -> Tuple[List[str], int, int]:
199 + """Converts the prompt ids to tokens and returns the tokens and offsets
200 + for incremental detokenization.
201 +
202 + Note that not all tokens are converted to strings. Only the tokens that
203 + are necessary for incremental detokenization are converted to strings.
204 + """
205 + # We do not need to convert the whole prompt to tokens.
206 + # Offset a little more in case we have special tokens.
207 + new_tokens = tokenizer.convert_ids_to_tokens(
208 + prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2:],
209 + skip_special_tokens=skip_special_tokens)
210 + read_offset = len(new_tokens)
211 + prefix_offset = max(
212 + read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
213 + return new_tokens, prefix_offset, read_offset
214 +
215 +
216 +# Based on
217 +# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
218 +# under Apache 2.0 license
219 +def detokenize_incrementally(
220 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
221 + all_input_ids: List[int],
222 + prev_tokens: Optional[List[str]],
223 + prefix_offset: int,
224 + read_offset: int,
225 + skip_special_tokens: bool = False,
226 + spaces_between_special_tokens: bool = True,
227 +) -> Tuple[List[str], str, int, int]:
228 + """Detokenizes the input ids incrementally and returns the new tokens
229 + and the new text.
230 +
231 + If `prev_tokens` is None, this function will convert the input ids to
232 + tokens and return the tokens and the new text. Otherwise, it will return the
233 + new tokens and the new text.
234 +
235 + This function will also return the new prefix offset and the new read
236 + offset to be used in the next iteration.
237 +
238 + The offsets are necessary to defeat cleanup algorithms in the decode which
239 + decide to add a space or not depending on the surrounding ids.
240 +
241 + Args:
242 + tokenizer: The tokenizer to use.
243 + all_input_ids: The input ids. The last id is the new token id.
244 + prev_tokens: The previous tokens. If None, this function will convert
245 + the input ids to tokens and return the tokens and the new text.
246 + prefix_offset: The prefix offset.
247 + read_offset: The read offset.
248 + skip_special_tokens: Whether to skip special tokens.
249 + spaces_between_special_tokens: Whether to add spaces between special
250 + tokens.
251 + """
252 + new_token_id = all_input_ids[-1]
253 + # This is the first iteration for this sequence
254 + is_first_iter = prev_tokens is None
255 + if is_first_iter:
256 + (prev_tokens, prefix_offset,
257 + read_offset) = convert_prompt_ids_to_tokens(
258 + tokenizer,
259 + all_input_ids[:-1],
260 + skip_special_tokens=skip_special_tokens)
261 +
262 + # If the new token id is out of bounds, return an empty string.
263 + if new_token_id >= len(tokenizer):
264 + new_tokens = [""]
265 + else:
266 + # Put new_token_id in a list so skip_special_tokens is respected
267 + new_tokens = tokenizer.convert_ids_to_tokens(
268 + [new_token_id], skip_special_tokens=skip_special_tokens)
269 + output_tokens = prev_tokens + new_tokens
270 +
271 + # If this is the first iteration, return all tokens.
272 + if is_first_iter:
273 + new_tokens = output_tokens
274 +
275 + # The prefix text is necessary only to defeat cleanup algorithms in
276 + # the decode which decide to add a space or not depending on the
277 + # surrounding ids.
278 + if tokenizer.is_fast or not tokenizer.get_added_vocab():
279 + prefix_text = tokenizer.convert_tokens_to_string(
280 + output_tokens[prefix_offset:read_offset])
281 + new_text = tokenizer.convert_tokens_to_string(
282 + output_tokens[prefix_offset:])
283 + else:
284 + prefix_text = _convert_tokens_to_string_with_added_encoders(
285 + tokenizer,
286 + output_tokens[prefix_offset:read_offset],
287 + skip_special_tokens=skip_special_tokens,
288 + spaces_between_special_tokens=spaces_between_special_tokens,
289 + )
290 + new_text = _convert_tokens_to_string_with_added_encoders(
291 + tokenizer,
292 + output_tokens[prefix_offset:],
293 + skip_special_tokens=skip_special_tokens,
294 + spaces_between_special_tokens=spaces_between_special_tokens,
295 + )
296 +
297 + if len(new_text) <= len(prefix_text) or new_text.endswith("�"):
298 + # utf-8 char at the end means it's a potential unfinished byte sequence
299 + # from byte fallback tokenization.
300 + # If it's in the middle, it's probably a real invalid id generated
301 + # by the model
302 + return new_tokens, "", prefix_offset, read_offset
303 +
304 + new_text = new_text[len(prefix_text):]
305 + return new_tokens, new_text, read_offset, len(output_tokens)
vllm/transformers_utils/tokenizer.py
+1 -155
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple, Union
1 +from typing import Optional, Union
2
3 from transformers import (AutoTokenizer, PreTrainedTokenizer,
4 PreTrainedTokenizerFast)
@@ -126,157 +126,3 @@ def get_lora_tokenizer(lora_request: LoRARequest, *args,
126
127
128 get_lora_tokenizer_async = make_async(get_lora_tokenizer)
129 -
130 -
131 -def _convert_tokens_to_string_with_added_encoders(
132 - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
133 - output_tokens: List[str],
134 - skip_special_tokens: bool,
135 - spaces_between_special_tokens: bool,
136 -) -> str:
137 - # Adapted from
138 - # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
139 - # NOTE(woosuk): The following code is slow because it runs a for loop over
140 - # the output_tokens. In Python, running a for loop over a list can be slow
141 - # even when the loop body is very simple.
142 - sub_texts = []
143 - current_sub_text = []
144 - all_special_tokens = set(tokenizer.all_special_tokens)
145 - for token in output_tokens:
146 - if skip_special_tokens and token in all_special_tokens:
147 - continue
148 - if token in tokenizer.get_added_vocab():
149 - if current_sub_text:
150 - sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
151 - sub_texts.append(sub_text)
152 - current_sub_text = []
153 - sub_texts.append(token)
154 - else:
155 - current_sub_text.append(token)
156 - if current_sub_text:
157 - sub_text = tokenizer.convert_tokens_to_string(current_sub_text)
158 - sub_texts.append(sub_text)
159 - if spaces_between_special_tokens:
160 - return " ".join(sub_texts)
161 - else:
162 - return "".join(sub_texts)
163 -
164 -
165 -# 5 is an arbitrary value that should work for all
166 -# tokenizers (bigger = more conservative).
167 -INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
168 -
169 -
170 -def convert_prompt_ids_to_tokens(
171 - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
172 - prompt_ids: List[int],
173 - skip_special_tokens: bool = False,
174 -) -> Tuple[List[str], int, int]:
175 - """Converts the prompt ids to tokens and returns the tokens and offsets
176 - for incremental detokenization.
177 -
178 - Note that not all tokens are converted to strings. Only the tokens that
179 - are necessary for incremental detokenization are converted to strings.
180 - """
181 - # Offset a little more in case we have special tokens.
182 - prefix_offset = max(
183 - len(prompt_ids) - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2, 0)
184 - # We do not need to convert the whole prompt to tokens.
185 - new_tokens = tokenizer.convert_ids_to_tokens(
186 - prompt_ids[prefix_offset:], skip_special_tokens=skip_special_tokens)
187 - prefix_offset = max(
188 - len(new_tokens) - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
189 - read_offset = len(new_tokens)
190 - return new_tokens, prefix_offset, read_offset
191 -
192 -
193 -# Based on
194 -# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
195 -# under Apache 2.0 license
196 -def detokenize_incrementally(
197 - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
198 - all_input_ids: List[int],
199 - prev_tokens: Optional[List[str]],
200 - prefix_offset: int,
201 - read_offset: int,
202 - skip_special_tokens: bool = False,
203 - spaces_between_special_tokens: bool = True,
204 -) -> Tuple[List[str], str, int, int]:
205 - """Detokenizes the input ids incrementally and returns the new tokens
206 - and the new text.
207 -
208 - If `prev_tokens` is None, this function will convert the input ids to
209 - tokens and return the tokens and the new text. Otherwise, it will return the
210 - new tokens and the new text.
211 -
212 - This function will also return the new prefix offset and the new read
213 - offset to be used in the next iteration.
214 -
215 - The offsets are necessary to defeat cleanup algorithms in the decode which
216 - decide to add a space or not depending on the surrounding ids.
217 -
218 - Args:
219 - tokenizer: The tokenizer to use.
220 - all_input_ids: The input ids. The last id is the new token id.
221 - prev_tokens: The previous tokens. If None, this function will convert
222 - the input ids to tokens and return the tokens and the new text.
223 - prefix_offset: The prefix offset.
224 - read_offset: The read offset.
225 - skip_special_tokens: Whether to skip special tokens.
226 - spaces_between_special_tokens: Whether to add spaces between special
227 - tokens.
228 - """
229 - new_token_id = all_input_ids[-1]
230 - # This is the first iteration for this sequence
231 - is_first_iter = prev_tokens is None
232 - if is_first_iter:
233 - (prev_tokens, prefix_offset,
234 - read_offset) = convert_prompt_ids_to_tokens(
235 - tokenizer,
236 - all_input_ids[:-1],
237 - skip_special_tokens=skip_special_tokens)
238 -
239 - # If the new token id is out of bounds, return an empty string.
240 - if new_token_id >= len(tokenizer):
241 - new_tokens = [""]
242 - else:
243 - # Put new_token_id in a list so skip_special_tokens is respected
244 - new_tokens = tokenizer.convert_ids_to_tokens(
245 - [new_token_id], skip_special_tokens=skip_special_tokens)
246 - output_tokens = prev_tokens + new_tokens
247 -
248 - # If this is the first iteration, return all tokens.
249 - if is_first_iter:
250 - new_tokens = output_tokens
251 -
252 - # The prefix text is necessary only to defeat cleanup algorithms in
253 - # the decode which decide to add a space or not depending on the
254 - # surrounding ids.
255 - if tokenizer.is_fast or not tokenizer.get_added_vocab():
256 - prefix_text = tokenizer.convert_tokens_to_string(
257 - output_tokens[prefix_offset:read_offset])
258 - new_text = tokenizer.convert_tokens_to_string(
259 - output_tokens[prefix_offset:])
260 - else:
261 - prefix_text = _convert_tokens_to_string_with_added_encoders(
262 - tokenizer,
263 - output_tokens[prefix_offset:read_offset],
264 - skip_special_tokens=skip_special_tokens,
265 - spaces_between_special_tokens=spaces_between_special_tokens,
266 - )
267 - new_text = _convert_tokens_to_string_with_added_encoders(
268 - tokenizer,
269 - output_tokens[prefix_offset:],
270 - skip_special_tokens=skip_special_tokens,
271 - spaces_between_special_tokens=spaces_between_special_tokens,
272 - )
273 -
274 - if len(new_text) > len(prefix_text) and not new_text.endswith("�"):
275 - # utf-8 char at the end means it's a potential unfinished byte sequence
276 - # from byte fallback tokenization.
277 - # If it's in the middle, it's probably a real invalid id generated
278 - # by the model
279 - new_text = new_text[len(prefix_text):]
280 - return new_tokens, new_text, read_offset, len(output_tokens)
281 - else:
282 - return new_tokens, "", prefix_offset, read_offset