A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 17:00 UTC
Patch Mistral Tokenizer (#28146)
Signed-off-by: Julien Denize <julien.denize@mistral.ai>
Julien Denize committed
Nov 6, 2025 at 07:43 UTC
a404e2c0f1bf100d28453a5a2ab7bd2a29d9aa31
2 files changed
+42
-22
tests/tokenization/test_mistral_tokenizer.py
+21
-8
@@ -334,20 +334,20 @@ class TestMistralTokenizer:
334
335
def test_encode(self, mistral_tokenizer: MistralTokenizer):
336
token_ids = (
337
- [1, 22177, 4304, 2662, 2]
337
+ [1, 22177, 4304, 2662]
338
if mistral_tokenizer.is_tekken
339
- else [1, 23325, 2294, 1686, 2]
339
+ else [1, 23325, 2294, 1686]
340
)
341
342
- assert mistral_tokenizer.encode("Hello world !") == token_ids[:-1]
343
- assert mistral_tokenizer.encode("Hello world !", max_length=3) == token_ids[:-2]
342
+ assert mistral_tokenizer.encode("Hello world !") == token_ids
343
+ assert mistral_tokenizer.encode("Hello world !", max_length=3) == token_ids[:-1]
344
assert (
345
mistral_tokenizer.encode("Hello world !", truncation=True, max_length=3)
346
- == token_ids[:-2]
346
+ == token_ids[:-1]
347
)
348
assert (
349
mistral_tokenizer.encode("Hello world !", truncation=False, max_length=3)
350
- == token_ids[:-1]
350
+ == token_ids
351
)
352
353
assert (
@@ -358,7 +358,7 @@ class TestMistralTokenizer:
358
mistral_tokenizer.encode(
359
"Hello world !", add_special_tokens=True, max_length=3
360
)
361
- == token_ids[:-2]
361
+ == token_ids[:-1]
362
)
363
assert (
364
mistral_tokenizer.encode(
@@ -368,7 +368,7 @@ class TestMistralTokenizer:
368
)
369
assert (
370
mistral_tokenizer.encode("Hello world !", add_special_tokens=False)
371
- == token_ids[1:-1]
371
+ == token_ids[1:]
372
)
373
374
@pytest.mark.parametrize(
@@ -1088,6 +1088,19 @@ class TestMistralTokenizer:
1088
== expected_tokens[mistral_tokenizer.is_tekken]
1089
)
1090
1091
+ def test_decode_int(
1092
+ self,
1093
+ mistral_tokenizer: MistralTokenizer,
1094
+ ):
1095
+ ids = 1
1096
+ assert (
1097
+ mistral_tokenizer.decode(
1098
+ ids,
1099
+ skip_special_tokens=False,
1100
+ )
1101
+ == "<s>"
1102
+ )
1103
+
1104
def test_convert_tokens_to_string(self, mistral_tokenizer: MistralTokenizer):
1105
tokens = (
1106
[
vllm/transformers_utils/tokenizers/mistral.py
+21
-14
@@ -165,6 +165,7 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:
165
166
class MistralTokenizer(TokenizerBase):
167
def __init__(self, tokenizer: "TransformersMistralTokenizer") -> None:
168
+ from mistral_common.protocol.instruct.validator import ValidationMode
169
from mistral_common.tokens.tokenizers.sentencepiece import (
170
SentencePieceTokenizer,
171
)
@@ -175,6 +176,14 @@ class MistralTokenizer(TokenizerBase):
176
self.instruct = self.mistral.instruct_tokenizer
177
self.tokenizer = self.instruct.tokenizer
178
179
+ mode = self.mistral._chat_completion_request_validator._mode
180
+ if mode != ValidationMode.test:
181
+ raise ValueError(
182
+ "Mistral tokenizer must be in test mode. Make sure to "
183
+ "set `mode='ValidationMode.test'` when creating the "
184
+ "Mistral tokenizer."
185
+ )
186
+
187
_mistral_version_str = str(self.tokenizer.version.value)
188
self.version: int = int(_mistral_version_str.split("v")[-1])
189
@@ -205,6 +214,7 @@ class MistralTokenizer(TokenizerBase):
214
def from_pretrained(
215
cls, path_or_repo_id: str, *, revision: str | None = None
216
) -> "MistralTokenizer":
217
+ from mistral_common.protocol.instruct.validator import ValidationMode
218
from transformers.tokenization_mistral_common import (
219
MistralCommonTokenizer as TransformersMistralTokenizer,
220
)
@@ -212,7 +222,7 @@ class MistralTokenizer(TokenizerBase):
222
str_revision = "main" if revision is None else revision
223
return cls(
224
TransformersMistralTokenizer.from_pretrained(
215
- path_or_repo_id, revision=str_revision
225
+ path_or_repo_id, revision=str_revision, mode=ValidationMode.test
226
)
227
)
228
@@ -339,20 +349,14 @@ class MistralTokenizer(TokenizerBase):
349
max_length: int | None = None,
350
add_special_tokens: bool | None = None,
351
) -> list[int]:
342
- if add_special_tokens is not None:
343
- return self.transformers_tokenizer.encode(
344
- text,
345
- truncation=truncation,
346
- max_length=max_length,
347
- add_special_tokens=add_special_tokens,
348
- )
349
- else:
350
- encoded = self.tokenizer.encode(text, bos=True, eos=False)
352
+ encoded = self.tokenizer.encode(
353
+ text, bos=add_special_tokens is not False, eos=False
354
+ )
355
352
- if truncation is not False and max_length is not None:
353
- return encoded[:max_length]
354
- else:
355
- return encoded
356
+ if truncation is not False and max_length is not None:
357
+ return encoded[:max_length]
358
+ else:
359
+ return encoded
360
361
def apply_chat_template(
362
self,
@@ -383,6 +387,9 @@ class MistralTokenizer(TokenizerBase):
387
)
388
389
def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str:
390
+ if isinstance(ids, int):
391
+ ids = [ids]
392
+
393
return self.transformers_tokenizer.decode(
394
ids, skip_special_tokens=skip_special_tokens
395
)