| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import json |
| 7 | |
| 8 | from vllm.tokenizers import TokenizerLike |
| 9 | |
| 10 | |
| 11 | def _replace_none_with_empty(tokens: list[str | None]): |
| 12 | for i, token in enumerate(tokens): |
| 13 | if token is None: |
| 14 | tokens[i] = "" |
| 15 | |
| 16 | |
| 17 | def _convert_tokens_to_string_with_added_encoders( |
| 18 | tokenizer: TokenizerLike, |
| 19 | output_tokens: list[str], |
| 20 | skip_special_tokens: bool, |
| 21 | spaces_between_special_tokens: bool, |
| 22 | ) -> str: |
| 23 | # Adapted from |
| 24 | # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921 |
| 25 | # NOTE(woosuk): The following code is slow because it runs a for loop over |
| 26 | # the output_tokens. In Python, running a for loop over a list can be slow |
| 27 | # even when the loop body is very simple. |
| 28 | # Performance improvements: avoid repeated attribute and function lookups; |
| 29 | # localize frequently used objects; |
| 30 | |
| 31 | sub_texts: list[str] = [] |
| 32 | current_sub_text: list[str] = [] |
| 33 | convert_tokens_to_string = tokenizer.convert_tokens_to_string |
| 34 | added_vocab_set = set(tokenizer.get_added_vocab()) |
| 35 | all_special_tokens = ( |
| 36 | set(tokenizer.all_special_tokens) if skip_special_tokens else () |
| 37 | ) |
| 38 | |
| 39 | for token in output_tokens: |
| 40 | # Use precomputed set for skip-special check |
| 41 | if token in all_special_tokens: |
| 42 | continue |
| 43 | if token in added_vocab_set: |
| 44 | if current_sub_text: |
| 45 | sub_texts.append(convert_tokens_to_string(current_sub_text)) |
| 46 | current_sub_text.clear() |
| 47 | sub_texts.append(token) |
| 48 | else: |
| 49 | current_sub_text.append(token) |
| 50 | if current_sub_text: |
| 51 | sub_texts.append(convert_tokens_to_string(current_sub_text)) |
| 52 | if spaces_between_special_tokens: |
| 53 | return " ".join(sub_texts) |
| 54 | return "".join(sub_texts) |
| 55 | |
| 56 | |
| 57 | # 5 is an arbitrary value that should work for all |
| 58 | # tokenizers (bigger = more conservative). |
| 59 | INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5 |
| 60 | |
| 61 | |
| 62 | _CACHED_MARKER_KEY = "_vllm_space_marker_cache" |
| 63 | _NOT_CACHED = "__not_computed__" |
| 64 | |
| 65 | |
| 66 | def get_leading_space_marker(tokenizer: TokenizerLike) -> str | None: |
| 67 | """Read the space marker from the tokenizer's pre_tokenizer config. |
| 68 | |
| 69 | Only Metaspace pre_tokenizers (used by SentencePiece-based models like |
| 70 | Llama, Mistral, T5) have a replacement character whose leading instance |
| 71 | gets stripped by decode(). ByteLevel (GPT-2), BertPreTokenizer (BERT), |
| 72 | and others do not have this issue. |
| 73 | |
| 74 | Returns the marker character, or None if decode() is safe for single |
| 75 | tokens. |
| 76 | """ |
| 77 | cached = getattr(tokenizer, _CACHED_MARKER_KEY, _NOT_CACHED) |
| 78 | if cached is not _NOT_CACHED: |
| 79 | return cached # type: ignore[return-value] |
| 80 | |
| 81 | backend = getattr(tokenizer, "backend_tokenizer", None) |
| 82 | if backend is None: |
| 83 | result = None |
| 84 | else: |
| 85 | result = None |
| 86 | try: |
| 87 | config = json.loads(backend.to_str()) |
| 88 | except Exception: |
| 89 | pass |
| 90 | else: |
| 91 | pre = config.get("pre_tokenizer") or {} |
| 92 | pre_type = pre.get("type") |
| 93 | if pre_type == "Metaspace": |
| 94 | result = pre.get("replacement", "▁") |
| 95 | elif pre_type == "Sequence": |
| 96 | for sub in pre.get("pretokenizers", []): |
| 97 | if sub.get("type") == "Metaspace": |
| 98 | result = sub.get("replacement", "▁") |
| 99 | break |
| 100 | |
| 101 | setattr(tokenizer, _CACHED_MARKER_KEY, result) |
| 102 | return result |
| 103 | |
| 104 | |
| 105 | def _restore_leading_spaces(raw_token: str, token_str: str, marker: str) -> str: |
| 106 | """Restore leading spaces that decode() stripped from a raw vocab piece.""" |
| 107 | num_markers = 0 |
| 108 | for ch in raw_token: |
| 109 | if ch != marker: |
| 110 | break |
| 111 | num_markers += 1 |
| 112 | if num_markers == 0: |
| 113 | return token_str |
| 114 | existing = len(token_str) - len(token_str.lstrip(" ")) |
| 115 | missing = num_markers - existing |
| 116 | return " " * missing + token_str if missing > 0 else token_str |
| 117 | |
| 118 | |
| 119 | def convert_prompt_ids_to_tokens( |
| 120 | tokenizer: TokenizerLike, |
| 121 | prompt_ids: list[int], |
| 122 | skip_special_tokens: bool = False, |
| 123 | ) -> tuple[list[str], int, int]: |
| 124 | """Converts the prompt ids to tokens and returns the tokens and offsets |
| 125 | for incremental detokenization. |
| 126 | |
| 127 | Note that not all tokens are converted to strings. Only the tokens that |
| 128 | are necessary for incremental detokenization are converted to strings. |
| 129 | """ |
| 130 | # We do not need to convert the whole prompt to tokens. |
| 131 | # Offset a little more in case we have special tokens. |
| 132 | new_tokens = tokenizer.convert_ids_to_tokens( |
| 133 | prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2 :], |
| 134 | skip_special_tokens=skip_special_tokens, |
| 135 | ) |
| 136 | read_offset = len(new_tokens) |
| 137 | prefix_offset = max(read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0) |
| 138 | # This is required to guard against out-of-vocab prompt token ids |
| 139 | _replace_none_with_empty(new_tokens) # type: ignore[arg-type] |
| 140 | return new_tokens, prefix_offset, read_offset |
| 141 | |
| 142 | |
| 143 | def convert_ids_list_to_tokens( |
| 144 | tokenizer: TokenizerLike, |
| 145 | token_ids: list[int], |
| 146 | ) -> list[str]: |
| 147 | """Detokenize the input ids individually. |
| 148 | |
| 149 | Uses decode() for human-readable output, then checks the raw vocab |
| 150 | piece via convert_ids_to_tokens() to restore any leading spaces that |
| 151 | decode() stripped (SentencePiece add_dummy_prefix inverse). |
| 152 | |
| 153 | Args: |
| 154 | tokenizer: tokenizer used by model under test |
| 155 | token_ids: convert these tokens (Python list form) |
| 156 | |
| 157 | Returns: |
| 158 | Python list of token string representations |
| 159 | |
| 160 | """ |
| 161 | if not token_ids: |
| 162 | return [] |
| 163 | marker = get_leading_space_marker(tokenizer) |
| 164 | if marker is None: |
| 165 | return [tokenizer.decode([tid]) or "" for tid in token_ids] |
| 166 | raw_tokens = tokenizer.convert_ids_to_tokens(token_ids) |
| 167 | return [ |
| 168 | _restore_leading_spaces(raw, tokenizer.decode([tid]) or "", marker) |
| 169 | for tid, raw in zip(token_ids, raw_tokens) |
| 170 | ] |
| 171 | |
| 172 | |
| 173 | # Based on |
| 174 | # https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15 |
| 175 | # under Apache 2.0 license |
| 176 | def detokenize_incrementally( |
| 177 | tokenizer: TokenizerLike, |
| 178 | all_input_ids: list[int], |
| 179 | prev_tokens: list[str] | None, |
| 180 | prefix_offset: int, |
| 181 | read_offset: int, |
| 182 | skip_special_tokens: bool = False, |
| 183 | spaces_between_special_tokens: bool = True, |
| 184 | ) -> tuple[list[str], str, int, int]: |
| 185 | """Detokenizes the input ids incrementally and returns the new tokens |
| 186 | and the new text. |
| 187 | |
| 188 | If `prev_tokens` is None, this function will convert the input ids to |
| 189 | tokens and return the tokens and the new text. Otherwise, it will return the |
| 190 | new tokens and the new text. |
| 191 | |
| 192 | This function will also return the new prefix offset and the new read |
| 193 | offset to be used in the next iteration. |
| 194 | |
| 195 | The offsets are necessary to defeat cleanup algorithms in the decode which |
| 196 | decide to add a space or not depending on the surrounding ids. |
| 197 | |
| 198 | Args: |
| 199 | tokenizer: The tokenizer to use. |
| 200 | all_input_ids: The input ids. The last id is the new token id. |
| 201 | prev_tokens: The previous tokens. If None, this function will convert |
| 202 | the input ids to tokens and return the tokens and the new text. |
| 203 | prefix_offset: The prefix offset. |
| 204 | read_offset: The read offset. |
| 205 | skip_special_tokens: Whether to skip special tokens. |
| 206 | spaces_between_special_tokens: Whether to add spaces between special |
| 207 | tokens. |
| 208 | |
| 209 | """ |
| 210 | new_token_id = all_input_ids[-1] |
| 211 | # This is the first iteration for this sequence |
| 212 | is_first_iter = prev_tokens is None |
| 213 | if is_first_iter: |
| 214 | (prev_tokens, prefix_offset, read_offset) = convert_prompt_ids_to_tokens( |
| 215 | tokenizer, all_input_ids[:-1], skip_special_tokens=skip_special_tokens |
| 216 | ) |
| 217 | assert prev_tokens is not None |
| 218 | |
| 219 | # If the new token id is out of bounds, return an empty string. |
| 220 | if 0 <= new_token_id < len(tokenizer): |
| 221 | # Put new_token_id in a list so skip_special_tokens is respected |
| 222 | new_tokens = tokenizer.convert_ids_to_tokens( |
| 223 | [new_token_id], skip_special_tokens=skip_special_tokens |
| 224 | ) |
| 225 | if isinstance(new_tokens, str): |
| 226 | new_tokens = [new_tokens] |
| 227 | else: |
| 228 | # This is required to guard against out-of-vocab prompt token ids |
| 229 | # (for example when using dummy weights) |
| 230 | _replace_none_with_empty(new_tokens) # type: ignore[arg-type] |
| 231 | else: |
| 232 | new_tokens = [""] |
| 233 | output_tokens = prev_tokens + new_tokens |
| 234 | |
| 235 | # If this is the first iteration, return all tokens. |
| 236 | if is_first_iter: |
| 237 | new_tokens = output_tokens |
| 238 | |
| 239 | # The prefix text is necessary only to defeat cleanup algorithms in |
| 240 | # the decode which decide to add a space or not depending on the |
| 241 | # surrounding ids. |
| 242 | if tokenizer.is_fast or not tokenizer.get_added_vocab(): |
| 243 | prefix_text = tokenizer.convert_tokens_to_string( |
| 244 | output_tokens[prefix_offset:read_offset] |
| 245 | ) |
| 246 | new_text = tokenizer.convert_tokens_to_string(output_tokens[prefix_offset:]) |
| 247 | else: |
| 248 | prefix_text = _convert_tokens_to_string_with_added_encoders( |
| 249 | tokenizer, |
| 250 | output_tokens[prefix_offset:read_offset], |
| 251 | skip_special_tokens=skip_special_tokens, |
| 252 | spaces_between_special_tokens=spaces_between_special_tokens, |
| 253 | ) |
| 254 | new_text = _convert_tokens_to_string_with_added_encoders( |
| 255 | tokenizer, |
| 256 | output_tokens[prefix_offset:], |
| 257 | skip_special_tokens=skip_special_tokens, |
| 258 | spaces_between_special_tokens=spaces_between_special_tokens, |
| 259 | ) |
| 260 | |
| 261 | if len(new_text) <= len(prefix_text) or new_text.endswith("�"): |
| 262 | # utf-8 char at the end means it's a potential unfinished byte sequence |
| 263 | # from byte fallback tokenization. |
| 264 | # If it's in the middle, it's probably a real invalid id generated |
| 265 | # by the model |
| 266 | return new_tokens, "", prefix_offset, read_offset |
| 267 | |
| 268 | new_text = new_text[len(prefix_text) :] |
| 269 | return new_tokens, new_text, read_offset, len(output_tokens) |