[Tokenizer] Use HF config for HF tokenizers (#49907)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jul 27, 2026 at 02:49 UTC
cbc3a872005d595493b546acf46f94a9c9f68cbb
1 file changed
+10
-4
vllm/tokenizers/registry.py
+10
-4
@@ -205,17 +205,27 @@ def get_tokenizer(
205
**kwargs,
206
)
207
208
+ if tokenizer_cls == TokenizerLike:
209
+ tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode)
210
+ else:
211
+ tokenizer_cls_ = tokenizer_cls
212
+
213
# Ensure that, if the config were to come from vllm.transformers_utils.config, it is
214
# registered with AutoConfig before the tokenizer is loaded. This is necessary since
215
# tokenizer_cls_.from_pretrained will call AutoConfig.from_pretrained internally.
216
# This may fail for paths that don't have a model config (e.g. LoRA adapters),
217
# which is fine — those don't need custom config registration.
218
+ # HF-backed tokenizers must receive the HF config. In a dual-format Mistral
219
+ # repository, auto detection intentionally prefers params.json, but passing
220
+ # that generic config to AutoTokenizer can select the wrong tokenizer class.
221
+ config_format = "hf" if tokenizer_cls_ is CachedHfTokenizer else "auto"
222
config = None
223
with contextlib.suppress(ValueError, OSError):
224
config = get_config(
225
tokenizer_name,
226
trust_remote_code=trust_remote_code,
227
revision=revision,
228
+ config_format=config_format,
229
)
230
231
# Some models have an incorrect tokenizer_class on the hub.
@@ -229,10 +239,6 @@ def get_tokenizer(
239
model_type,
240
)
241
tokenizer_cls_ = TokenizersBackend
232
- elif tokenizer_cls == TokenizerLike:
233
- tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode)
234
- else:
235
- tokenizer_cls_ = tokenizer_cls
242
243
if config is not None and tokenizer_cls_ is CachedHfTokenizer:
244
# AutoTokenizer otherwise reloads config.json internally. Reuse the