[Bugfix] Avoid MistralCommonBackend for HF tokenizers (#54192)
Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com>
Julien Denize committed
Sep 10, 2026 at 19:57 UTC
983b7e28c9f87c3a150081640e0fdeafec5d553a
7 files changed
+129
-157
tests/tokenizers_/test_registry.py
+84
@@ -10,6 +10,7 @@ from transformers import AutoConfig
10
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
11
12
from vllm.tokenizers import TokenizerLike
13
+from vllm.tokenizers.hf import CachedHfTokenizer
14
from vllm.tokenizers.registry import (
15
TokenizerRegistry,
16
cached_get_tokenizer,
@@ -68,6 +69,89 @@ def test_resolve_tokenizer_args_idempotent(runner_type):
69
)
70
71
72
+@pytest.mark.parametrize(
73
+ ("tokenizer_mode", "input_kwargs"),
74
+ [
75
+ ("hf", {}),
76
+ ("hf", {"mistral_format": False}),
77
+ ("slow", {}),
78
+ ("slow", {"mistral_format": False}),
79
+ ],
80
+)
81
+def test_resolve_tokenizer_args_forces_hf_mistral_format_false(
82
+ tokenizer_mode, input_kwargs
83
+):
84
+ resolved_mode, _, _, kwargs = resolve_tokenizer_args(
85
+ "mistralai/Mistral-Nemo-Instruct-2407",
86
+ tokenizer_mode=tokenizer_mode,
87
+ **input_kwargs,
88
+ )
89
+
90
+ assert resolved_mode == "hf"
91
+ assert kwargs["mistral_format"] is False
92
+
93
+
94
+@pytest.mark.parametrize("tokenizer_mode", ["hf", "slow"])
95
+def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(tokenizer_mode):
96
+ with pytest.raises(
97
+ ValueError,
98
+ match="mistral_format=True is not supported with tokenizer_mode='hf'",
99
+ ):
100
+ resolve_tokenizer_args(
101
+ "mistralai/Mistral-Nemo-Instruct-2407",
102
+ tokenizer_mode=tokenizer_mode,
103
+ mistral_format=True,
104
+ )
105
+
106
+
107
+@pytest.mark.parametrize(
108
+ ("transformers_version", "model_type", "native_filename", "expect_error"),
109
+ [
110
+ pytest.param("5.14.0", "mistral", "tekken.json", True),
111
+ pytest.param("5.15.0", "mistral", "tekken.json", False),
112
+ pytest.param("5.14.0", "mistral3", "tekken.json", True),
113
+ pytest.param("5.15.0", "mistral3", "tekken.json", False),
114
+ pytest.param("5.14.0", "mixtral", "tokenizer.model.v1", True),
115
+ pytest.param("5.15.0", "mixtral", "tokenizer.model.v1", False),
116
+ pytest.param("5.14.0", "mistral", None, False),
117
+ pytest.param("5.14.0", "qwen3_5_moe", "tekken.json", False),
118
+ ],
119
+)
120
+def test_get_tokenizer_handles_hf_mistral_transformers_compatibility(
121
+ tmp_path: Path,
122
+ transformers_version: str,
123
+ model_type: str,
124
+ native_filename: str | None,
125
+ expect_error: bool,
126
+):
127
+ if native_filename is not None:
128
+ (tmp_path / native_filename).touch()
129
+
130
+ with (
131
+ patch(
132
+ "vllm.tokenizers.registry.get_config",
133
+ return_value=SimpleNamespace(model_type=model_type),
134
+ ),
135
+ patch(
136
+ "vllm.tokenizers.registry.transformers.__version__",
137
+ transformers_version,
138
+ ),
139
+ patch.object(
140
+ CachedHfTokenizer,
141
+ "from_pretrained",
142
+ return_value=SimpleNamespace(is_fast=True),
143
+ ) as from_pretrained,
144
+ ):
145
+ if expect_error:
146
+ with pytest.raises(ValueError, match="requires transformers>=5.15.0"):
147
+ get_tokenizer(str(tmp_path), tokenizer_mode="hf")
148
+ else:
149
+ tokenizer = get_tokenizer(str(tmp_path), tokenizer_mode="hf")
150
+ assert tokenizer.is_fast is True
151
+
152
+ assert from_pretrained.called == (not expect_error)
153
+
154
+
155
def test_customized_tokenizer():
156
TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
157
tests/v1/structured_output/test_mistral_common_tokenizer.py
deleted
-102
@@ -1,102 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""Structured output with a `MistralCommonBackend` tokenizer."""
4
-
5
-import pytest
6
-from transformers import AutoTokenizer, MistralCommonBackend
7
-
8
-from vllm.config import StructuredOutputsConfig, VllmConfig
9
-from vllm.tokenizers import get_tokenizer
10
-from vllm.tokenizers.mistral import MistralTokenizer
11
-from vllm.v1.structured_output.backend_outlines import OutlinesBackend
12
-from vllm.v1.structured_output.backend_types import StructuredOutputOptions
13
-from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
14
-from vllm.v1.structured_output.utils import (
15
- _reduced_vocabulary,
16
- maybe_wrap_mistral_common_tokenizer,
17
-)
18
-
19
-TOKENIZER = "mistralai/Mistral-Nemo-Instruct-2407"
20
-JSON_SCHEMA = (
21
- '{"type": "object", "properties": {"x": {"type": "integer"}}, '
22
- '"required": ["x"], "additionalProperties": false}'
23
-)
24
-VALID_DOCUMENT = '{"x": 1}'
25
-
26
-BACKENDS = [("xgrammar", XgrammarBackend), ("outlines", OutlinesBackend)]
27
-
28
-
29
-@pytest.fixture(scope="module")
30
-def loaded_tokenizer():
31
- tokenizer = get_tokenizer(tokenizer_name=TOKENIZER, tokenizer_mode="hf")
32
- assert isinstance(tokenizer, MistralCommonBackend)
33
- return tokenizer
34
-
35
-
36
-@pytest.fixture(scope="module")
37
-def wrapped_tokenizer(loaded_tokenizer):
38
- return maybe_wrap_mistral_common_tokenizer(loaded_tokenizer)
39
-
40
-
41
-def _encode(tokenizer, text):
42
- special = set(tokenizer.all_special_ids)
43
- return [tid for tid in tokenizer.encode(text) if tid not in special]
44
-
45
-
46
-def _backend(cls, name, tokenizer):
47
- vllm_config = VllmConfig(
48
- structured_outputs_config=StructuredOutputsConfig(backend=name)
49
- )
50
- return cls(vllm_config, tokenizer=tokenizer, vocab_size=tokenizer.vocab_size)
51
-
52
-
53
-def test_wrapping_produces_a_mistral_tokenizer(loaded_tokenizer, wrapped_tokenizer):
54
- assert isinstance(wrapped_tokenizer, MistralTokenizer)
55
- assert len(wrapped_tokenizer.vocab) == loaded_tokenizer.vocab_size
56
-
57
-
58
-def test_other_tokenizers_are_left_alone():
59
- tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
60
- assert maybe_wrap_mistral_common_tokenizer(tokenizer) is tokenizer
61
-
62
-
63
-def test_wrapping_fixes_the_outlines_vocabulary(loaded_tokenizer, wrapped_tokenizer):
64
- unwrapped_vocab = _reduced_vocabulary(loaded_tokenizer)
65
- wrapped_vocab = _reduced_vocabulary(wrapped_tokenizer)
66
-
67
- assert unwrapped_vocab.keys() == wrapped_vocab.keys()
68
-
69
- mismapped = {
70
- token for token, ids in unwrapped_vocab.items() if ids != wrapped_vocab[token]
71
- }
72
- assert mismapped
73
-
74
- assert len({tuple(wrapped_vocab[t]) for t in mismapped}) == len(mismapped)
75
- assert len({tuple(unwrapped_vocab[t]) for t in mismapped}) < len(mismapped)
76
-
77
-
78
-@pytest.mark.parametrize("name,cls", BACKENDS)
79
-def test_backend_accepts_a_valid_document(wrapped_tokenizer, name, cls):
80
- backend = _backend(cls, name, wrapped_tokenizer)
81
- grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA)
82
-
83
- for token in _encode(wrapped_tokenizer, VALID_DOCUMENT):
84
- assert grammar.accept_tokens("req", [token])
85
-
86
-
87
-@pytest.mark.parametrize("name,cls", BACKENDS)
88
-def test_backend_rejects_an_invalid_token(wrapped_tokenizer, name, cls):
89
- backend = _backend(cls, name, wrapped_tokenizer)
90
- grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA)
91
-
92
- assert not grammar.accept_tokens("req", [_encode(wrapped_tokenizer, "]")[-1]])
93
-
94
-
95
-def test_xgrammar_vocab_size_matches_the_tokenizer(wrapped_tokenizer):
96
- backend = _backend(XgrammarBackend, "xgrammar", wrapped_tokenizer)
97
- assert backend.vocab_size == len(wrapped_tokenizer.vocab)
98
-
99
-
100
-def test_xgrammar_rejects_the_unwrapped_tokenizer(loaded_tokenizer):
101
- with pytest.raises(ValueError, match="Unsupported tokenizer type"):
102
- _backend(XgrammarBackend, "xgrammar", loaded_tokenizer)
vllm/tokenizers/hf.py
+2
-30
@@ -116,17 +116,10 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
116
tokenizer_all_special_tokens = tokenizer.all_special_tokens
117
tokenizer_vocab = tokenizer.get_vocab()
118
tokenizer_len = len(tokenizer)
119
- # The underlying tokenizer class could be MistralCommonBackend,
120
- # which does not implement is_fast in Transformers
119
+ # The underlying tokenizer class could be a specific backend,
120
+ # which does not always implement is_fast in Transformers
121
tokenizer_is_fast = getattr(tokenizer, "is_fast", True)
122
123
- # MistralCommonBackend is tekken-backed and needs byte-fallback-aware tokenization.
124
- mistral_tekkenizer = None
125
- if getattr(getattr(tokenizer, "tokenizer", None), "instruct_tokenizer", None):
126
- from vllm.tokenizers.mistral import mistral_common_tekkenizer
127
-
128
- mistral_tekkenizer = mistral_common_tekkenizer(tokenizer)
129
-
123
max_token_id = max(tokenizer_vocab.values())
124
max_chars_per_token = max(len(tok) for tok in tokenizer_vocab)
125
@@ -159,27 +152,6 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
152
def is_fast(self) -> bool:
153
return tokenizer_is_fast
154
162
- def convert_ids_to_tokens(self, ids, skip_special_tokens: bool = False):
163
- if mistral_tekkenizer is not None:
164
- from vllm.tokenizers.mistral import tekken_convert_ids_to_tokens
165
-
166
- return tekken_convert_ids_to_tokens(mistral_tekkenizer, ids)
167
- return super().convert_ids_to_tokens(
168
- ids, skip_special_tokens=skip_special_tokens
169
- )
170
-
171
- def convert_tokens_to_string(self, tokens: list[str]) -> str:
172
- if mistral_tekkenizer is not None:
173
- from vllm.tokenizers.mistral import tekken_convert_tokens_to_string
174
-
175
- return tekken_convert_tokens_to_string(mistral_tekkenizer, tokens)
176
- try:
177
- return super().convert_tokens_to_string(tokens)
178
- except NotImplementedError:
179
- # The underlying tokenizer class could be MistralCommonBackend,
180
- # which does not implement convert_tokens_to_string in Transformers
181
- return "".join(tokens)
182
-
155
def get_vocab(self) -> dict[str, int]:
156
return tokenizer_vocab
157
vllm/tokenizers/registry.py
+41
-4
@@ -7,9 +7,15 @@ from pathlib import Path
7
from typing import TYPE_CHECKING
8
9
import huggingface_hub
10
+import transformers
11
+from packaging.version import Version
12
+from transformers.models.auto.tokenization_auto import (
13
+ TOKENIZER_MAPPING_NAMES as _HF_TOKENIZER_MAPPING_NAMES,
14
+)
15
+from transformers.utils.import_utils import is_mistral_common_available
16
from typing_extensions import TypeVar, assert_never
17
12
-import vllm.envs as envs
18
+from vllm import envs
19
from vllm.logger import init_logger
20
from vllm.transformers_utils.config import _maybe_register_hf_config, get_config
21
from vllm.transformers_utils.repo_utils import (
@@ -26,6 +32,8 @@ if TYPE_CHECKING:
32
33
logger = init_logger(__name__)
34
35
+_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL = Version("5.15.0")
36
+
37
38
# Model types whose hub tokenizer_class is incorrect and should be overridden with
39
# TokenizersBackend (the generic fast tokenizer). Adding a model type here is always a
@@ -74,8 +82,6 @@ class _TokenizerRegistry:
82
83
self.tokenizers[tokenizer_mode] = (module, class_name)
84
77
- return None
78
-
85
def load_tokenizer_cls(self, tokenizer_mode: str) -> type[TokenizerLike]:
86
if tokenizer_mode not in self.tokenizers:
87
raise ValueError(f"No tokenizer registered for {tokenizer_mode=!r}.")
@@ -150,12 +156,15 @@ def resolve_tokenizer_args(
156
if (
157
tokenizer_mode == "auto"
158
and is_mistral_model_repo(
153
- model_name_or_path=str(tokenizer_name), revision=revision
159
+ model_name_or_path=str(tokenizer_name),
160
+ revision=revision,
161
+ token=kwargs.get("token"),
162
)
163
and any_pattern_in_repo_files(
164
model_name_or_path=str(tokenizer_name),
165
allow_patterns=["tekken.json", "tokenizer.model.v*"],
166
revision=revision,
167
+ token=kwargs.get("token"),
168
)
169
):
170
tokenizer_mode = "mistral"
@@ -164,6 +173,13 @@ def resolve_tokenizer_args(
173
if tokenizer_mode == "auto":
174
tokenizer_mode = "hf"
175
176
+ if tokenizer_mode == "hf":
177
+ if kwargs.pop("mistral_format", False):
178
+ raise ValueError(
179
+ "mistral_format=True is not supported with tokenizer_mode='hf'"
180
+ )
181
+ kwargs["mistral_format"] = False
182
+
183
return tokenizer_mode, tokenizer_name, args, kwargs
184
185
@@ -231,6 +247,27 @@ def get_tokenizer(
247
trust_remote_code=trust_remote_code,
248
revision=revision,
249
config_format=config_format,
250
+ token=kwargs.get("token"),
251
+ )
252
+
253
+ # TODO: delete when Transformers version dependency is bumped >= 5.15.0
254
+ if (
255
+ tokenizer_mode == "hf"
256
+ and Version(transformers.__version__) < _MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL
257
+ and is_mistral_common_available()
258
+ and _HF_TOKENIZER_MAPPING_NAMES.get(getattr(config, "model_type", None))
259
+ == "MistralCommonBackend"
260
+ and any_pattern_in_repo_files(
261
+ model_name_or_path=str(tokenizer_name),
262
+ allow_patterns=["tekken.json", "tokenizer.model.v*"],
263
+ revision=revision,
264
+ token=kwargs.get("token"),
265
+ )
266
+ ):
267
+ raise ValueError(
268
+ "Loading Mistral models with tokenizer_mode='hf' requires "
269
+ f"transformers>={_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL}. "
270
+ "Please upgrade transformers or delete Mistral tokenizer files."
271
)
272
273
# Some models have an incorrect tokenizer_class on the hub.
vllm/v1/structured_output/__init__.py
+2
-3
@@ -18,7 +18,6 @@ from vllm.v1.structured_output.backend_types import (
18
StructuredOutputGrammar,
19
)
20
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
21
-from vllm.v1.structured_output.utils import maybe_wrap_mistral_common_tokenizer
21
22
if TYPE_CHECKING:
23
import numpy as np
@@ -77,8 +76,8 @@ class StructuredOutputManager:
76
# of CPUs.
77
max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
78
self.executor = ThreadPoolExecutor(max_workers=max_workers)
80
- self.tokenizer = maybe_wrap_mistral_common_tokenizer(
81
- cached_tokenizer_from_config(model_config=self.vllm_config.model_config)
79
+ self.tokenizer = cached_tokenizer_from_config(
80
+ model_config=self.vllm_config.model_config
81
)
82
reasoning_parser_plugin = (
83
self.vllm_config.structured_outputs_config.reasoning_parser_plugin
vllm/v1/structured_output/backend_guidance.py
-5
@@ -8,7 +8,6 @@ from dataclasses import dataclass
8
from typing import TYPE_CHECKING, Any
9
10
import torch
11
-from transformers import MistralCommonBackend
11
12
from vllm.exceptions import VLLMValidationError
13
from vllm.logger import init_logger
@@ -98,10 +97,6 @@ class GuidanceBackend(StructuredOutputBackend):
97
98
if is_mistral_tokenizer(self.tokenizer):
99
self.ll_tokenizer = self.tokenizer.llg_tokenizer
101
- elif isinstance(self.tokenizer, MistralCommonBackend):
102
- from mistral_common.guidance.tokenizer import from_mistral_tokenizer
103
-
104
- self.ll_tokenizer = from_mistral_tokenizer(self.tokenizer.tokenizer)
100
else:
101
self.ll_tokenizer = llguidance_hf.from_tokenizer(
102
self.tokenizer, max(self.vocab_size, len(self.tokenizer))
vllm/v1/structured_output/utils.py
-13
@@ -14,7 +14,6 @@ from typing import TYPE_CHECKING, TypeVar
14
import regex as re
15
import torch
16
from cachetools import LRUCache
17
-from transformers import MistralCommonBackend
17
18
import vllm.envs as envs
19
from vllm.logger import init_logger
@@ -320,18 +319,6 @@ re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$")
319
re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$")
320
321
323
-def maybe_wrap_mistral_common_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike:
324
- """The grammar backends cannot consume a `MistralCommonBackend` directly."""
325
- if not isinstance(tokenizer, MistralCommonBackend):
326
- return tokenizer
327
-
328
- # Deferred: `vllm.tokenizers.mistral` pulls in a large dependency tree, and
329
- # this module is imported by `vllm.v1.worker.gpu_model_runner`.
330
- from vllm.tokenizers.mistral import MistralTokenizer
331
-
332
- return MistralTokenizer(tokenizer)
333
-
334
-
322
def _reduced_vocabulary(tokenizer: TokenizerLike) -> dict[bytes, list[int]]:
323
"""Create a map from vocabulary tokens to lists of equivalent token ids.
324