| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Generator |
| 5 | from typing import Any, cast |
| 6 | |
| 7 | import pytest |
| 8 | from transformers import AutoTokenizer, PythonBackend, TokenizersBackend |
| 9 | |
| 10 | from vllm.sampling_params import SamplingParams |
| 11 | from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens |
| 12 | from vllm.tokenizers.mistral import MistralTokenizer |
| 13 | from vllm.tokenizers.protocol import TokenizerLike |
| 14 | from vllm.v1.engine import EngineCoreRequest |
| 15 | from vllm.v1.engine.detokenizer import ( |
| 16 | FastIncrementalDetokenizer, |
| 17 | IncrementalDetokenizer, |
| 18 | SlowIncrementalDetokenizer, |
| 19 | ) |
| 20 | |
| 21 | SPECIAL_TOKS_TRUTH = [ |
| 22 | "Some text with adjacent special tokens <|padding|><|padding|><fim_prefix><fim_middle><fim_suffix>other text<fim_pad>", # noqa |
| 23 | ] |
| 24 | |
| 25 | TRUTH = [ |
| 26 | "Hello here, this is a simple test", |
| 27 | "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving", # noqa |
| 28 | "我很感谢你的热情", |
| 29 | # Burmese text triggers an edge-case for Mistral's V3-Tekken tokenizer (eg. |
| 30 | # for mistralai/Pixtral-12B-2409) where tokens may map to bytes with |
| 31 | # incomplete UTF-8 characters |
| 32 | # see https://github.com/vllm-project/vllm/pull/9625 |
| 33 | "ပုံပြင်လေးပြောပြပါ်", |
| 34 | ] + SPECIAL_TOKS_TRUTH |
| 35 | |
| 36 | TOKENIZERS = [ |
| 37 | "facebook/opt-125m", |
| 38 | "openai-community/gpt2", |
| 39 | "bigcode/tiny_starcoder_py", |
| 40 | "EleutherAI/gpt-j-6b", |
| 41 | "EleutherAI/pythia-70m", |
| 42 | "bigscience/bloom-560m", |
| 43 | "tiiuae/falcon-7b", |
| 44 | "meta-llama/Llama-3.2-1B-Instruct", |
| 45 | "codellama/CodeLlama-7b-hf", |
| 46 | "mistralai/Pixtral-12B-2409", |
| 47 | ] |
| 48 | |
| 49 | |
| 50 | def _run_incremental_decode( |
| 51 | tokenizer, |
| 52 | all_input_ids, |
| 53 | skip_special_tokens: bool, |
| 54 | starting_index: int, |
| 55 | spaces_between_special_tokens: bool = True, |
| 56 | fast: bool | None = None, |
| 57 | ): |
| 58 | prompt_token_ids = all_input_ids[:starting_index] |
| 59 | |
| 60 | params = SamplingParams( |
| 61 | skip_special_tokens=skip_special_tokens, |
| 62 | spaces_between_special_tokens=spaces_between_special_tokens, |
| 63 | ) |
| 64 | request = EngineCoreRequest( |
| 65 | request_id="", |
| 66 | prompt_token_ids=prompt_token_ids, |
| 67 | mm_features=None, |
| 68 | sampling_params=params, |
| 69 | pooling_params=None, |
| 70 | arrival_time=0.0, |
| 71 | lora_request=None, |
| 72 | cache_salt=None, |
| 73 | data_parallel_rank=None, |
| 74 | ) |
| 75 | |
| 76 | if fast is None: |
| 77 | detokenizer = IncrementalDetokenizer.from_new_request(tokenizer, request) |
| 78 | elif fast: |
| 79 | detokenizer = FastIncrementalDetokenizer(tokenizer, request) |
| 80 | else: |
| 81 | detokenizer = SlowIncrementalDetokenizer(tokenizer, request) |
| 82 | |
| 83 | output_text = "" |
| 84 | for i, token_id in enumerate(all_input_ids[starting_index:]): |
| 85 | detokenizer.update([token_id], False) |
| 86 | finished = i == len(all_input_ids) - 1 |
| 87 | output_text += detokenizer.get_next_output_text(finished, delta=True) |
| 88 | |
| 89 | return output_text, detokenizer.output_token_ids |
| 90 | |
| 91 | |
| 92 | @pytest.fixture |
| 93 | def tokenizer(tokenizer_name): |
| 94 | return ( |
| 95 | MistralTokenizer.from_pretrained(tokenizer_name) |
| 96 | if "mistral" in tokenizer_name |
| 97 | else AutoTokenizer.from_pretrained(tokenizer_name) |
| 98 | ) |
| 99 | |
| 100 | |
| 101 | @pytest.mark.parametrize("tokenizer_name", ["mistralai/Pixtral-12B-2409"]) |
| 102 | @pytest.mark.parametrize( |
| 103 | "truth", |
| 104 | [ |
| 105 | # Burmese text triggers an edge-case where tokens may map to bytes with |
| 106 | # incomplete UTF-8 characters |
| 107 | "ပုံပြင်လေးပြောပြပါ", |
| 108 | # Using "URGENCY" since "CY" has token id 130282 |
| 109 | "URGENCY🌶️", |
| 110 | ], |
| 111 | ) |
| 112 | def test_mistral_edge_case(tokenizer, truth): |
| 113 | """Test for a specific edge cases with V3-Tekken MistralTokenizer. |
| 114 | |
| 115 | See https://github.com/vllm-project/vllm/pull/9625 |
| 116 | """ |
| 117 | starting_index = 0 |
| 118 | all_input_ids = tokenizer(truth, add_special_tokens=False).input_ids |
| 119 | |
| 120 | decoded_text, out_ids = _run_incremental_decode( |
| 121 | tokenizer, |
| 122 | all_input_ids, |
| 123 | skip_special_tokens=True, |
| 124 | starting_index=starting_index, |
| 125 | ) |
| 126 | assert decoded_text == truth |
| 127 | assert out_ids == all_input_ids[starting_index:] |
| 128 | |
| 129 | |
| 130 | @pytest.fixture |
| 131 | def skip_special_tokens(request, tokenizer_name) -> Generator[bool, Any, None]: |
| 132 | if "mistral" in tokenizer_name: |
| 133 | yield ( |
| 134 | True |
| 135 | if request.param |
| 136 | else pytest.skip("mistral doesn't support skip_special_tokens=False") |
| 137 | ) |
| 138 | else: |
| 139 | yield bool(request.param) |
| 140 | |
| 141 | |
| 142 | @pytest.mark.parametrize("truth", TRUTH) |
| 143 | @pytest.mark.parametrize("with_prompt", [True, False]) |
| 144 | @pytest.mark.parametrize("tokenizer_name", TOKENIZERS) |
| 145 | @pytest.mark.parametrize("skip_special_tokens", (True, False), indirect=True) |
| 146 | @pytest.mark.parametrize("spaces_between_special_tokens", (True, False)) |
| 147 | @pytest.mark.parametrize("fast", (True, False)) |
| 148 | def test_decode_streaming( |
| 149 | tokenizer, |
| 150 | truth, |
| 151 | with_prompt, |
| 152 | skip_special_tokens, |
| 153 | spaces_between_special_tokens, |
| 154 | fast, |
| 155 | ): |
| 156 | if fast and not isinstance(tokenizer, TokenizersBackend): |
| 157 | pytest.skip() |
| 158 | |
| 159 | if skip_special_tokens and not spaces_between_special_tokens: |
| 160 | pytest.skip() |
| 161 | |
| 162 | if not fast and isinstance(tokenizer, TokenizersBackend): |
| 163 | # Fix up inconsistency in fast/slow tokenizer behaviour. |
| 164 | tokenizer.add_special_tokens( |
| 165 | { |
| 166 | "additional_special_tokens": [ |
| 167 | at |
| 168 | for at in tokenizer._tokenizer.get_added_tokens_decoder().values() |
| 169 | if at.special |
| 170 | ] |
| 171 | } |
| 172 | ) |
| 173 | |
| 174 | extra_decode_args = ( |
| 175 | {} |
| 176 | if not isinstance(tokenizer, PythonBackend) |
| 177 | else {"spaces_between_special_tokens": spaces_between_special_tokens} |
| 178 | ) |
| 179 | |
| 180 | truth_tokens = tokenizer(truth, add_special_tokens=False).input_ids |
| 181 | if tokenizer.bos_token_id is not None: |
| 182 | truth_tokens.insert(0, tokenizer.bos_token_id) |
| 183 | truth_tokens.append(tokenizer.eos_token_id) |
| 184 | |
| 185 | new_truth = tokenizer.decode( |
| 186 | truth_tokens, skip_special_tokens=skip_special_tokens, **extra_decode_args |
| 187 | ) |
| 188 | |
| 189 | if with_prompt: |
| 190 | num_prompt_tokens = len( |
| 191 | tokenizer(truth[: len(truth) // 2], add_special_tokens=False).input_ids |
| 192 | ) |
| 193 | if tokenizer.bos_token_id is not None: |
| 194 | num_prompt_tokens += 1 |
| 195 | |
| 196 | prompt_input_ids = truth_tokens[:num_prompt_tokens] |
| 197 | generated_input_ids = truth_tokens[num_prompt_tokens:] |
| 198 | all_input_ids = prompt_input_ids + generated_input_ids |
| 199 | starting_index = len(prompt_input_ids) |
| 200 | prompt = tokenizer.decode( |
| 201 | prompt_input_ids, |
| 202 | skip_special_tokens=skip_special_tokens, |
| 203 | **extra_decode_args, |
| 204 | ) |
| 205 | |
| 206 | generated = new_truth[len(prompt) :] |
| 207 | else: |
| 208 | generated = new_truth |
| 209 | starting_index = 0 |
| 210 | all_input_ids = truth_tokens |
| 211 | |
| 212 | decoded_text, out_ids = _run_incremental_decode( |
| 213 | tokenizer, |
| 214 | all_input_ids, |
| 215 | skip_special_tokens=skip_special_tokens, |
| 216 | starting_index=starting_index, |
| 217 | spaces_between_special_tokens=spaces_between_special_tokens, |
| 218 | fast=fast, |
| 219 | ) |
| 220 | |
| 221 | assert decoded_text == generated |
| 222 | assert out_ids == all_input_ids[starting_index:] |
| 223 | |
| 224 | |
| 225 | @pytest.mark.parametrize("tokenizer_name", TOKENIZERS) |
| 226 | @pytest.mark.parametrize("fast", (True, False)) |
| 227 | def test_oov_decode(tokenizer, fast): |
| 228 | if fast and not isinstance(tokenizer, TokenizersBackend): |
| 229 | pytest.skip() |
| 230 | |
| 231 | decoded_text, out_ids = _run_incremental_decode( |
| 232 | tokenizer, |
| 233 | [len(tokenizer)], |
| 234 | skip_special_tokens=True, |
| 235 | starting_index=0, |
| 236 | spaces_between_special_tokens=True, |
| 237 | fast=fast, |
| 238 | ) |
| 239 | |
| 240 | assert decoded_text == "" |
| 241 | assert out_ids == [len(tokenizer)] |
| 242 | |
| 243 | |
| 244 | # ---------- convert_ids_list_to_tokens collision tests ---------- |
| 245 | |
| 246 | |
| 247 | class _MockBackend: |
| 248 | """Fake backend_tokenizer that exposes pre_tokenizer config.""" |
| 249 | |
| 250 | def __init__(self, pre_tokenizer_type, replacement=None): |
| 251 | import json |
| 252 | |
| 253 | pre: dict = {"type": pre_tokenizer_type} |
| 254 | if replacement is not None: |
| 255 | pre["replacement"] = replacement |
| 256 | self._config = json.dumps({"pre_tokenizer": pre}) |
| 257 | |
| 258 | def to_str(self): |
| 259 | return self._config |
| 260 | |
| 261 | |
| 262 | class _MockTokenizer: |
| 263 | """Minimal tokenizer mock for testing convert_ids_list_to_tokens.""" |
| 264 | |
| 265 | def __init__( |
| 266 | self, |
| 267 | raw_tokens: dict[int, str], |
| 268 | decoded_tokens: dict[int, str], |
| 269 | pre_tokenizer_type: str = "Metaspace", |
| 270 | replacement: str | None = "▁", |
| 271 | ): |
| 272 | self._raw = raw_tokens |
| 273 | self._decoded = decoded_tokens |
| 274 | self.backend_tokenizer = _MockBackend(pre_tokenizer_type, replacement) |
| 275 | |
| 276 | def convert_ids_to_tokens( |
| 277 | self, ids: list[int], skip_special_tokens: bool = False |
| 278 | ) -> list[str]: |
| 279 | return [self._raw[tid] for tid in ids] |
| 280 | |
| 281 | def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str: |
| 282 | return "".join(self._decoded[tid] for tid in ids) |
| 283 | |
| 284 | |
| 285 | def _mock_tokenizer( |
| 286 | raw_tokens: dict[int, str], |
| 287 | decoded_tokens: dict[int, str], |
| 288 | pre_tokenizer_type: str = "Metaspace", |
| 289 | replacement: str | None = "▁", |
| 290 | ) -> TokenizerLike: |
| 291 | return cast( |
| 292 | TokenizerLike, |
| 293 | _MockTokenizer( |
| 294 | raw_tokens=raw_tokens, |
| 295 | decoded_tokens=decoded_tokens, |
| 296 | pre_tokenizer_type=pre_tokenizer_type, |
| 297 | replacement=replacement, |
| 298 | ), |
| 299 | ) |
| 300 | |
| 301 | |
| 302 | def test_sentencepiece_leading_space_preserved(): |
| 303 | """▁true and true must produce distinct strings.""" |
| 304 | tok = _mock_tokenizer( |
| 305 | raw_tokens={0: "▁true", 1: "true", 2: "▁false", 3: "false"}, |
| 306 | decoded_tokens={0: "true", 1: "true", 2: "false", 3: "false"}, |
| 307 | ) |
| 308 | result = convert_ids_list_to_tokens(tok, [0, 1, 2, 3]) |
| 309 | assert result == [" true", "true", " false", "false"] |
| 310 | |
| 311 | # No dict collision when used as top_logprobs keys |
| 312 | logprobs = dict(zip(result, [-0.1, -0.2, -0.3, -0.4])) |
| 313 | assert len(logprobs) == 4 |
| 314 | |
| 315 | |
| 316 | def test_whitespace_run_tokens_stay_distinct(): |
| 317 | """▁, ▁▁, ▁▁▁ must produce different-length space strings.""" |
| 318 | tok = _mock_tokenizer( |
| 319 | raw_tokens={0: "▁", 1: "▁▁", 2: "▁▁▁"}, |
| 320 | decoded_tokens={0: "", 1: " ", 2: " "}, |
| 321 | ) |
| 322 | result = convert_ids_list_to_tokens(tok, [0, 1, 2]) |
| 323 | assert result == [" ", " ", " "] |
| 324 | |
| 325 | |
| 326 | def test_bpe_leading_space_already_preserved(): |
| 327 | """GPT-2 BPE: Ġtrue already decodes to ' true', no fix needed.""" |
| 328 | tok = _mock_tokenizer( |
| 329 | raw_tokens={0: "Ġtrue", 1: "true"}, |
| 330 | decoded_tokens={0: " true", 1: "true"}, |
| 331 | pre_tokenizer_type="ByteLevel", |
| 332 | replacement=None, |
| 333 | ) |
| 334 | result = convert_ids_list_to_tokens(tok, [0, 1]) |
| 335 | assert result == [" true", "true"] |
| 336 | |
| 337 | |
| 338 | def test_logprobs_count_stable_across_k(): |
| 339 | """logprobs=4 and logprobs=10 must return 4 and 10 entries.""" |
| 340 | tok = _mock_tokenizer( |
| 341 | raw_tokens={ |
| 342 | 0: "▁true", |
| 343 | 1: "a", |
| 344 | 2: "b", |
| 345 | 3: "c", |
| 346 | 4: "true", |
| 347 | 5: "d", |
| 348 | 6: "e", |
| 349 | 7: "f", |
| 350 | 8: "g", |
| 351 | 9: "h", |
| 352 | }, |
| 353 | decoded_tokens={ |
| 354 | 0: "true", |
| 355 | 1: "a", |
| 356 | 2: "b", |
| 357 | 3: "c", |
| 358 | 4: "true", |
| 359 | 5: "d", |
| 360 | 6: "e", |
| 361 | 7: "f", |
| 362 | 8: "g", |
| 363 | 9: "h", |
| 364 | }, |
| 365 | ) |
| 366 | ids = list(range(10)) |
| 367 | lps = [-0.1 * (i + 1) for i in range(10)] |
| 368 | |
| 369 | tokens4 = convert_ids_list_to_tokens(tok, ids[:4]) |
| 370 | top4 = dict(zip(tokens4, lps[:4])) |
| 371 | |
| 372 | tokens10 = convert_ids_list_to_tokens(tok, ids) |
| 373 | top10 = dict(zip(tokens10, lps)) |
| 374 | |
| 375 | assert len(top4) == 4 |
| 376 | assert len(top10) == 10 |
| 377 | assert top4[" true"] == top10[" true"] |