A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 07:30 UTC
Improve Mistral format checks. (#33253)
Signed-off-by: Julien Denize <julien.denize@mistral.ai> Signed-off-by: juliendenize <julien.denize@mistral.ai> Co-authored-by: Michael Goin <mgoin64@gmail.com>
Julien Denize committed
Jan 30, 2026 at 15:23 UTC
ae5b7aff2bbca60b335190f76f4c637119035db2
8 files changed
+193
-24
tests/config/test_model_arch_config.py
+4
-1
@@ -83,7 +83,10 @@ def _assert_model_arch_config(
83
assert model_arch_config.is_deepseek_mla == expected["is_deepseek_mla"]
84
85
torch_dtype = ModelArchConfigConvertorBase.get_torch_dtype(
86
- model_config.hf_config, model_config.model, revision=model_config.revision
86
+ model_config.hf_config,
87
+ model_config.model,
88
+ revision=model_config.revision,
89
+ config_format="hf",
90
)
91
assert str(torch_dtype) == expected["dtype"]
92
tests/conftest.py
+1
@@ -365,6 +365,7 @@ class HfRunner:
365
self.config,
366
dtype=dtype,
367
is_pooling_model=is_sentence_transformer or is_cross_encoder,
368
+ config_format="hf",
369
)
370
371
model_kwargs = model_kwargs if model_kwargs is not None else {}
tests/transformers_utils/test_repo_utils.py
+97
-1
@@ -8,7 +8,11 @@ from unittest.mock import MagicMock, call, patch
8
9
import pytest
10
11
-from vllm.transformers_utils.repo_utils import list_filtered_repo_files
11
+from vllm.transformers_utils.repo_utils import (
12
+ any_pattern_in_repo_files,
13
+ is_mistral_model_repo,
14
+ list_filtered_repo_files,
15
+)
16
17
18
@pytest.mark.parametrize(
@@ -60,3 +64,95 @@ def test_list_filtered_repo_files(
64
repo_type="model",
65
token="token",
66
)
67
+
68
+
69
+@pytest.mark.parametrize(
70
+ ("allow_patterns", "expected_bool"),
71
+ [
72
+ (["*.json", "correct*.txt"], True),
73
+ (
74
+ ["*.jpeg"],
75
+ True,
76
+ ),
77
+ (
78
+ ["not_found.jpeg"],
79
+ False,
80
+ ),
81
+ ],
82
+)
83
+def test_one_filtered_repo_files(allow_patterns: list[str], expected_bool: bool):
84
+ with tempfile.TemporaryDirectory() as tmp_dir:
85
+ # Prep folder and files
86
+ path_tmp_dir = Path(tmp_dir)
87
+ subfolder = path_tmp_dir / "subfolder"
88
+ subfolder.mkdir()
89
+ (path_tmp_dir / "uncorrect.jpeg").touch()
90
+ (subfolder / "correct.txt").touch()
91
+
92
+ def _glob_path() -> list[str]:
93
+ return [
94
+ str(file.relative_to(path_tmp_dir))
95
+ for file in path_tmp_dir.glob("**/*")
96
+ if file.is_file()
97
+ ]
98
+
99
+ # Patch list_repo_files called by fn
100
+ with patch(
101
+ "vllm.transformers_utils.repo_utils.list_repo_files",
102
+ MagicMock(return_value=_glob_path()),
103
+ ) as mock_list_repo_files:
104
+ assert (
105
+ any_pattern_in_repo_files(
106
+ tmp_dir, allow_patterns, "revision", "model", "token"
107
+ )
108
+ ) is expected_bool
109
+ assert mock_list_repo_files.call_count == 1
110
+ assert mock_list_repo_files.call_args_list[0] == call(
111
+ repo_id=tmp_dir,
112
+ revision="revision",
113
+ repo_type="model",
114
+ token="token",
115
+ )
116
+
117
+
118
+@pytest.mark.parametrize(
119
+ ("files", "expected_bool"),
120
+ [
121
+ (["consolidated.safetensors", "incorrect.txt"], True),
122
+ (["consolidated-1.safetensors", "incorrect.txt"], True),
123
+ (
124
+ ["consolidated-1.json"],
125
+ False,
126
+ ),
127
+ ],
128
+)
129
+def test_is_mistral_model_repo(files: list[str], expected_bool: bool):
130
+ with tempfile.TemporaryDirectory() as tmp_dir:
131
+ # Prep folder and files
132
+ path_tmp_dir = Path(tmp_dir)
133
+ for file in files:
134
+ (path_tmp_dir / file).touch()
135
+
136
+ def _glob_path() -> list[str]:
137
+ return [
138
+ str(file.relative_to(path_tmp_dir))
139
+ for file in path_tmp_dir.glob("**/*")
140
+ if file.is_file()
141
+ ]
142
+
143
+ # Patch list_repo_files called by fn
144
+ with patch(
145
+ "vllm.transformers_utils.repo_utils.list_repo_files",
146
+ MagicMock(return_value=_glob_path()),
147
+ ) as mock_list_repo_files:
148
+ assert (
149
+ is_mistral_model_repo(tmp_dir, "revision", "model", "token")
150
+ is expected_bool
151
+ )
152
+ assert mock_list_repo_files.call_count == 1
153
+ assert mock_list_repo_files.call_args_list[0] == call(
154
+ repo_id=tmp_dir,
155
+ revision="revision",
156
+ repo_type="model",
157
+ token="token",
158
+ )
vllm/config/model.py
+3
-1
@@ -565,6 +565,7 @@ class ModelConfig:
565
self.dtype,
566
is_pooling_model=self.runner_type == "pooling",
567
revision=self.revision,
568
+ config_format=self.config_format,
569
)
570
571
self.original_max_model_len = self.max_model_len
@@ -1844,9 +1845,10 @@ def _get_and_verify_dtype(
1845
*,
1846
is_pooling_model: bool,
1847
revision: str | None = None,
1848
+ config_format: ConfigFormat = "hf",
1849
) -> torch.dtype:
1850
config_dtype = ModelArchConfigConvertorBase.get_torch_dtype(
1849
- config, model_id, revision=revision
1851
+ config, model_id, revision=revision, config_format=config_format
1852
)
1853
model_type = config.model_type
1854
vllm/tokenizers/registry.py
+19
-17
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import importlib.util
3
from dataclasses import dataclass, field
4
from functools import lru_cache
5
from pathlib import Path
@@ -18,7 +17,10 @@ from vllm.transformers_utils.gguf_utils import (
17
is_remote_gguf,
18
split_remote_gguf,
19
)
21
-from vllm.transformers_utils.repo_utils import list_filtered_repo_files
20
+from vllm.transformers_utils.repo_utils import (
21
+ any_pattern_in_repo_files,
22
+ is_mistral_model_repo,
23
+)
24
from vllm.utils.import_utils import resolve_obj_by_qualname
25
26
from .protocol import TokenizerLike
@@ -142,26 +144,26 @@ def resolve_tokenizer_args(
144
kwargs["use_fast"] = False
145
146
# Try to use official Mistral tokenizer if possible
145
- if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"):
146
- allow_patterns = ["tekken.json", "tokenizer.model.v*"]
147
- files_list = list_filtered_repo_files(
147
+ if (
148
+ tokenizer_mode == "auto"
149
+ and is_mistral_model_repo(
150
+ model_name_or_path=str(tokenizer_name), revision=revision
151
+ )
152
+ and any_pattern_in_repo_files(
153
model_name_or_path=str(tokenizer_name),
149
- allow_patterns=allow_patterns,
154
+ allow_patterns=["tekken.json", "tokenizer.model.v*"],
155
revision=revision,
156
)
152
- if len(files_list) > 0:
153
- tokenizer_mode = "mistral"
157
+ ):
158
+ tokenizer_mode = "mistral"
159
160
# Try to use Grok2 tiktoken tokenizer if possible
156
- if tokenizer_mode == "auto":
157
- allow_patterns = ["tokenizer.tok.json"]
158
- files_list = list_filtered_repo_files(
159
- model_name_or_path=str(tokenizer_name),
160
- allow_patterns=allow_patterns,
161
- revision=revision,
162
- )
163
- if len(files_list) > 0:
164
- tokenizer_mode = "grok2"
161
+ if tokenizer_mode == "auto" and any_pattern_in_repo_files(
162
+ model_name_or_path=str(tokenizer_name),
163
+ allow_patterns=["tokenizer.tok.json"],
164
+ revision=revision,
165
+ ):
166
+ tokenizer_mode = "grok2"
167
168
# Fallback to HF tokenizer
169
if tokenizer_mode == "auto":
vllm/transformers_utils/config.py
+6
-2
@@ -23,6 +23,7 @@ from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
23
24
from vllm import envs
25
from vllm.logger import init_logger
26
+from vllm.transformers_utils.repo_utils import is_mistral_model_repo
27
from vllm.transformers_utils.utils import parse_safetensors_file_metadata
28
29
from .config_parser_base import ConfigParserBase
@@ -49,7 +50,6 @@ except ImportError:
50
ALLOWED_LAYER_TYPES as ALLOWED_ATTENTION_LAYER_TYPES,
51
)
52
52
-
53
if envs.VLLM_USE_MODELSCOPE:
54
from modelscope import AutoConfig
55
else:
@@ -581,7 +581,11 @@ def get_config(
581
try:
582
# First check for Mistral to avoid defaulting to
583
# Transformers implementation.
584
- if file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
584
+ if is_mistral_model_repo(
585
+ model_name_or_path=str(model), revision=revision
586
+ ) and file_or_path_exists(
587
+ model=model, config_name=MISTRAL_CONFIG_NAME, revision=revision
588
+ ):
589
config_format = "mistral"
590
elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
591
model, HF_CONFIG_NAME, revision=revision
vllm/transformers_utils/model_arch_config_convertor.py
+27
-2
@@ -1,9 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from collections.abc import Iterator
5
+from contextlib import contextmanager
6
from typing import final
7
8
import torch
9
+from huggingface_hub import constants
10
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
11
from transformers import PretrainedConfig
12
@@ -14,6 +17,7 @@ from vllm.config.model_arch import (
17
from vllm.config.utils import getattr_iter
18
from vllm.logger import init_logger
19
from vllm.transformers_utils.config import (
20
+ ConfigFormat,
21
try_get_safetensors_metadata,
22
)
23
from vllm.utils.torch_utils import common_broadcastable_dtype
@@ -21,6 +25,22 @@ from vllm.utils.torch_utils import common_broadcastable_dtype
25
logger = init_logger(__name__)
26
27
28
+@contextmanager
29
+def _maybe_patch_hf_hub_constants(config_format: ConfigFormat) -> Iterator[None]:
30
+ if config_format == "mistral":
31
+ hf_safetensors_single_file = constants.SAFETENSORS_SINGLE_FILE
32
+ hf_safetensors_index_file = constants.SAFETENSORS_INDEX_FILE
33
+ constants.SAFETENSORS_SINGLE_FILE = "consolidated.safetensors"
34
+ constants.SAFETENSORS_INDEX_FILE = "consolidated.safetensors.index.json"
35
+ try:
36
+ yield
37
+ finally:
38
+ constants.SAFETENSORS_SINGLE_FILE = hf_safetensors_single_file
39
+ constants.SAFETENSORS_INDEX_FILE = hf_safetensors_index_file
40
+ else:
41
+ yield
42
+
43
+
44
class ModelArchConfigConvertorBase:
45
def __init__(self, hf_config: PretrainedConfig, hf_text_config: PretrainedConfig):
46
self.hf_config = hf_config
@@ -123,7 +143,11 @@ class ModelArchConfigConvertorBase:
143
@final
144
@classmethod
145
def get_torch_dtype(
126
- cls, hf_config: PretrainedConfig, model_id: str, revision: str | None
146
+ cls,
147
+ hf_config: PretrainedConfig,
148
+ model_id: str,
149
+ revision: str | None,
150
+ config_format: ConfigFormat,
151
):
152
# NOTE: getattr(config, "dtype", torch.float32) is not correct
153
# because config.dtype can be None.
@@ -140,7 +164,8 @@ class ModelArchConfigConvertorBase:
164
165
# Try to read the dtype of the weights if they are in safetensors format
166
if config_dtype is None:
143
- repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
167
+ with _maybe_patch_hf_hub_constants(config_format):
168
+ repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
169
170
if repo_mt and (files_mt := repo_mt.files_metadata):
171
param_dtypes: set[torch.dtype] = {
vllm/transformers_utils/repo_utils.py
+36
@@ -127,6 +127,42 @@ def list_filtered_repo_files(
127
return file_list
128
129
130
+def any_pattern_in_repo_files(
131
+ model_name_or_path: str,
132
+ allow_patterns: list[str],
133
+ revision: str | None = None,
134
+ repo_type: str | None = None,
135
+ token: str | bool | None = None,
136
+):
137
+ return (
138
+ len(
139
+ list_filtered_repo_files(
140
+ model_name_or_path=model_name_or_path,
141
+ allow_patterns=allow_patterns,
142
+ revision=revision,
143
+ repo_type=repo_type,
144
+ token=token,
145
+ )
146
+ )
147
+ > 0
148
+ )
149
+
150
+
151
+def is_mistral_model_repo(
152
+ model_name_or_path: str,
153
+ revision: str | None = None,
154
+ repo_type: str | None = None,
155
+ token: str | bool | None = None,
156
+) -> bool:
157
+ return any_pattern_in_repo_files(
158
+ model_name_or_path=model_name_or_path,
159
+ allow_patterns=["consolidated*.safetensors"],
160
+ revision=revision,
161
+ repo_type=repo_type,
162
+ token=token,
163
+ )
164
+
165
+
166
def file_exists(
167
repo_id: str,
168
file_name: str,