A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 09:45 UTC
[BUGFIX] MistralTokenizer._call__ adds an invalid EOS token (#29607)
Signed-off-by: Julien Denize <julien.denize@mistral.ai> Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Julien Denize committed
Nov 28, 2025 at 09:44 UTC
b2c1d294faca96643dbc2413d604ca160f458f0d
2 files changed
+87
-1
tests/tokenization/test_mistral_tokenizer.py
+68
@@ -331,6 +331,7 @@ class TestMistralTokenizer:
331
)
332
== token_ids
333
)
334
+ assert mistral_tokenizer.encode_one("") == []
335
336
def test_encode(self, mistral_tokenizer: MistralTokenizer):
337
token_ids = (
@@ -370,6 +371,51 @@ class TestMistralTokenizer:
371
mistral_tokenizer.encode("Hello world !", add_special_tokens=False)
372
== token_ids[1:]
373
)
374
+ assert mistral_tokenizer.encode("", add_special_tokens=False) == []
375
+
376
+ def test_call(self, mistral_tokenizer: MistralTokenizer):
377
+ token_ids = (
378
+ [1, 22177, 4304, 2662]
379
+ if mistral_tokenizer.is_tekken
380
+ else [1, 23325, 2294, 1686]
381
+ )
382
+ attn_mask = [1 for _ in range(len(token_ids))]
383
+
384
+ # Test 1: default
385
+ assert mistral_tokenizer("Hello world !") == {
386
+ "attention_mask": attn_mask[1:],
387
+ "input_ids": token_ids[1:],
388
+ }
389
+ # Test 2: special tokens
390
+ assert mistral_tokenizer("Hello world !", add_special_tokens=True) == {
391
+ "attention_mask": attn_mask,
392
+ "input_ids": token_ids,
393
+ }
394
+ # Test 3: special tokens + truncation
395
+ assert mistral_tokenizer(
396
+ "Hello world !", add_special_tokens=True, truncation=True, max_length=3
397
+ ) == {
398
+ "attention_mask": attn_mask[:-1],
399
+ "input_ids": token_ids[:-1],
400
+ }
401
+ # Test 4: special tokens + no truncation + max length
402
+ assert mistral_tokenizer(
403
+ "Hello world !", add_special_tokens=True, max_length=3
404
+ ) == {
405
+ "attention_mask": attn_mask,
406
+ "input_ids": token_ids,
407
+ }
408
+ # Test 5: empty string
409
+ assert mistral_tokenizer("") == {
410
+ "attention_mask": [],
411
+ "input_ids": [],
412
+ }
413
+
414
+ with pytest.raises(
415
+ ValueError,
416
+ match=(r"`text_pair` is not supported by `MistralTokenizer.__call__`."),
417
+ ):
418
+ mistral_tokenizer("Hello world !", "invalid pair")
419
420
@pytest.mark.parametrize(
421
"openai_request,add_generation_prompt,continue_final_message,expected_output,decoded_expected_output",
@@ -1087,6 +1133,24 @@ class TestMistralTokenizer:
1133
)
1134
== expected_tokens[mistral_tokenizer.is_tekken]
1135
)
1136
+ assert (
1137
+ mistral_tokenizer.decode(
1138
+ ids[mistral_tokenizer.is_tekken],
1139
+ skip_special_tokens=skip_special_tokens,
1140
+ )
1141
+ == expected_tokens[mistral_tokenizer.is_tekken]
1142
+ )
1143
+
1144
+ def test_decode_empty(
1145
+ self,
1146
+ mistral_tokenizer: MistralTokenizer,
1147
+ ):
1148
+ assert (
1149
+ mistral_tokenizer.decode(
1150
+ [],
1151
+ )
1152
+ == ""
1153
+ )
1154
1155
def test_decode_int(
1156
self,
@@ -1390,6 +1454,8 @@ class TestMistralTokenizer:
1454
== expected_strings[mistral_tokenizer.is_tekken]
1455
)
1456
1457
+ assert mistral_tokenizer.convert_tokens_to_string([]) == ""
1458
+
1459
@pytest.mark.parametrize(
1460
"skip_special_tokens,tuple_expected_tokens",
1461
(
@@ -2220,3 +2286,5 @@ class TestMistralTokenizer:
2286
ids, skip_special_tokens=skip_special_tokens
2287
)
2288
assert actual_tokens == expected_tokens
2289
+
2290
+ assert mistral_tokenizer.convert_ids_to_tokens([]) == []
vllm/transformers_utils/tokenizers/mistral.py
+19
-1
@@ -312,13 +312,27 @@ class MistralTokenizer(TokenizerBase):
312
truncation: bool = False,
313
max_length: int | None = None,
314
):
315
- return self.transformers_tokenizer(
315
+ if text_pair is not None:
316
+ raise ValueError(
317
+ "`text_pair` is not supported by `MistralTokenizer.__call__`."
318
+ )
319
+
320
+ encoded = self.transformers_tokenizer(
321
text=text,
322
text_pair=text_pair,
323
add_special_tokens=add_special_tokens,
324
truncation=truncation,
325
max_length=max_length,
326
)
327
+ # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
328
+ # is in, revert to only call self.transformers_tokenizer(...).
329
+ # Hack to fix wrongly added eos token, when fix will be supported the condition
330
+ # below will be False even before the revert is done.
331
+ if encoded["input_ids"] and encoded["input_ids"][-1] == self.eos_token_id:
332
+ encoded["input_ids"].pop(-1)
333
+ if attention_mask := encoded.get("attention_mask"):
334
+ attention_mask.pop(-1)
335
+ return encoded
336
337
@property
338
def vocab(self) -> list[str]:
@@ -349,6 +363,8 @@ class MistralTokenizer(TokenizerBase):
363
max_length: int | None = None,
364
add_special_tokens: bool | None = None,
365
) -> list[int]:
366
+ # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
367
+ # is in, directly call self.transformers_tokenizer.encode(...).
368
encoded = self.tokenizer.encode(
369
text, bos=add_special_tokens is not False, eos=False
370
)
@@ -387,6 +403,8 @@ class MistralTokenizer(TokenizerBase):
403
)
404
405
def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str:
406
+ # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
407
+ # is in, directly call self.transformers_tokenizer.decode(...).
408
if isinstance(ids, int):
409
ids = [ids]
410