A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 09:45 UTC
[Chore]: Reorganize model repo operating functions in `transformers_utils` (#29680)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Nov 29, 2025 at 00:46 UTC
f946a8d74308f4a47bc1624c9307c1bce45874eb
8 files changed
+304
-280
tests/transformers_utils/test_config.py
+2
-2
@@ -8,7 +8,7 @@ from unittest.mock import MagicMock, call, patch
8
9
import pytest
10
11
-from vllm.transformers_utils.config import list_filtered_repo_files
11
+from vllm.transformers_utils.repo_utils import list_filtered_repo_files
12
13
14
@pytest.mark.parametrize(
@@ -44,7 +44,7 @@ def test_list_filtered_repo_files(
44
45
# Patch list_repo_files called by fn
46
with patch(
47
- "vllm.transformers_utils.config.list_repo_files",
47
+ "vllm.transformers_utils.repo_utils.list_repo_files",
48
MagicMock(return_value=_glob_path()),
49
) as mock_list_repo_files:
50
out_files = sorted(
vllm/engine/arg_utils.py
+1
-1
@@ -83,10 +83,10 @@ from vllm.platforms import CpuArchEnum, current_platform
83
from vllm.plugins import load_general_plugins
84
from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized
85
from vllm.transformers_utils.config import (
86
- get_model_path,
86
is_interleaved,
87
maybe_override_with_speculators,
88
)
89
+from vllm.transformers_utils.repo_utils import get_model_path
90
from vllm.transformers_utils.utils import is_cloud_storage, is_gguf
91
from vllm.utils.argparse_utils import FlexibleArgumentParser
92
from vllm.utils.mem_constants import GiB_bytes
vllm/model_executor/model_loader/default_loader.py
+1
-1
@@ -31,7 +31,7 @@ from vllm.model_executor.model_loader.weight_utils import (
31
safetensors_weights_iterator,
32
)
33
from vllm.platforms import current_platform
34
-from vllm.transformers_utils.config import list_filtered_repo_files
34
+from vllm.transformers_utils.repo_utils import list_filtered_repo_files
35
36
logger = init_logger(__name__)
37
vllm/model_executor/models/adapters.py
+1
-1
@@ -14,9 +14,9 @@ from vllm.logger import init_logger
14
from vllm.model_executor.layers.activation import get_act_fn
15
from vllm.model_executor.models.config import VerifyAndUpdateConfig
16
from vllm.transformers_utils.config import (
17
- get_hf_file_bytes,
17
try_get_dense_modules,
18
)
19
+from vllm.transformers_utils.repo_utils import get_hf_file_bytes
20
21
from .interfaces_base import VllmModelForPooling, is_pooling_model
22
vllm/transformers_utils/config.py
+9
-270
@@ -1,30 +1,17 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import fnmatch
5
-import json
4
import os
7
-import time
5
from collections.abc import Callable
6
from dataclasses import asdict
7
from functools import cache, partial
8
from importlib.metadata import version
9
from pathlib import Path
13
-from typing import Any, Literal, TypeAlias, TypeVar
10
+from typing import Any, Literal, TypeAlias
11
12
import huggingface_hub
13
from huggingface_hub import (
14
get_safetensors_metadata,
18
- hf_hub_download,
19
- try_to_load_from_cache,
20
-)
21
-from huggingface_hub import list_repo_files as hf_list_repo_files
22
-from huggingface_hub.utils import (
23
- EntryNotFoundError,
24
- HfHubHTTPError,
25
- LocalEntryNotFoundError,
26
- RepositoryNotFoundError,
27
- RevisionNotFoundError,
15
)
16
from packaging.version import Version
17
from transformers import GenerationConfig, PretrainedConfig
@@ -40,6 +27,14 @@ from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
27
from vllm import envs
28
from vllm.logger import init_logger
29
from vllm.transformers_utils.config_parser_base import ConfigParserBase
30
+from vllm.transformers_utils.repo_utils import (
31
+ _get_hf_token,
32
+ file_or_path_exists,
33
+ get_hf_file_to_dict,
34
+ list_repo_files,
35
+ try_get_local_file,
36
+ with_retry,
37
+)
38
from vllm.transformers_utils.utils import (
39
check_gguf_file,
40
is_gguf,
@@ -58,21 +53,6 @@ MISTRAL_CONFIG_NAME = "params.json"
53
logger = init_logger(__name__)
54
55
61
-def _get_hf_token() -> str | None:
62
- """
63
- Get the HuggingFace token from environment variable.
64
-
65
- Returns None if the token is not set, is an empty string,
66
- or contains only whitespace.
67
- This follows the same pattern as huggingface_hub library which
68
- treats empty string tokens as None to avoid authentication errors.
69
- """
70
- token = os.getenv("HF_TOKEN")
71
- if token and token.strip():
72
- return token
73
- return None
74
-
75
-
56
class LazyConfigDict(dict):
57
def __getitem__(self, key):
58
if isinstance(value := super().__getitem__(key), type):
@@ -308,143 +288,6 @@ def register_config_parser(config_format: str):
288
return _wrapper
289
290
311
-_R = TypeVar("_R")
312
-
313
-
314
-def with_retry(
315
- func: Callable[[], _R],
316
- log_msg: str,
317
- max_retries: int = 2,
318
- retry_delay: int = 2,
319
-) -> _R:
320
- for attempt in range(max_retries):
321
- try:
322
- return func()
323
- except Exception as e:
324
- if attempt == max_retries - 1:
325
- logger.error("%s: %s", log_msg, e)
326
- raise
327
- logger.error(
328
- "%s: %s, retrying %d of %d", log_msg, e, attempt + 1, max_retries
329
- )
330
- time.sleep(retry_delay)
331
- retry_delay *= 2
332
-
333
- raise AssertionError("Should not be reached")
334
-
335
-
336
-# @cache doesn't cache exceptions
337
-@cache
338
-def list_repo_files(
339
- repo_id: str,
340
- *,
341
- revision: str | None = None,
342
- repo_type: str | None = None,
343
- token: str | bool | None = None,
344
-) -> list[str]:
345
- def lookup_files() -> list[str]:
346
- # directly list files if model is local
347
- if (local_path := Path(repo_id)).exists():
348
- return [
349
- str(file.relative_to(local_path))
350
- for file in local_path.rglob("*")
351
- if file.is_file()
352
- ]
353
- # if model is remote, use hf_hub api to list files
354
- try:
355
- if envs.VLLM_USE_MODELSCOPE:
356
- from vllm.transformers_utils.utils import modelscope_list_repo_files
357
-
358
- return modelscope_list_repo_files(
359
- repo_id,
360
- revision=revision,
361
- token=os.getenv("MODELSCOPE_API_TOKEN", None),
362
- )
363
- return hf_list_repo_files(
364
- repo_id, revision=revision, repo_type=repo_type, token=token
365
- )
366
- except huggingface_hub.errors.OfflineModeIsEnabled:
367
- # Don't raise in offline mode,
368
- # all we know is that we don't have this
369
- # file cached.
370
- return []
371
-
372
- return with_retry(lookup_files, "Error retrieving file list")
373
-
374
-
375
-def list_filtered_repo_files(
376
- model_name_or_path: str,
377
- allow_patterns: list[str],
378
- revision: str | None = None,
379
- repo_type: str | None = None,
380
- token: str | bool | None = None,
381
-) -> list[str]:
382
- try:
383
- all_files = list_repo_files(
384
- repo_id=model_name_or_path,
385
- revision=revision,
386
- token=token,
387
- repo_type=repo_type,
388
- )
389
- except Exception:
390
- logger.error(
391
- "Error retrieving file list. Please ensure your `model_name_or_path`"
392
- "`repo_type`, `token` and `revision` arguments are correctly set. "
393
- "Returning an empty list."
394
- )
395
- return []
396
-
397
- file_list = []
398
- # Filter patterns on filenames
399
- for pattern in allow_patterns:
400
- file_list.extend(
401
- [
402
- file
403
- for file in all_files
404
- if fnmatch.fnmatch(os.path.basename(file), pattern)
405
- ]
406
- )
407
- return file_list
408
-
409
-
410
-def file_exists(
411
- repo_id: str,
412
- file_name: str,
413
- *,
414
- repo_type: str | None = None,
415
- revision: str | None = None,
416
- token: str | bool | None = None,
417
-) -> bool:
418
- file_list = list_repo_files(
419
- repo_id, repo_type=repo_type, revision=revision, token=token
420
- )
421
- return file_name in file_list
422
-
423
-
424
-# In offline mode the result can be a false negative
425
-def file_or_path_exists(
426
- model: str | Path, config_name: str, revision: str | None
427
-) -> bool:
428
- if (local_path := Path(model)).exists():
429
- return (local_path / config_name).is_file()
430
-
431
- # Offline mode support: Check if config file is cached already
432
- cached_filepath = try_to_load_from_cache(
433
- repo_id=model, filename=config_name, revision=revision
434
- )
435
- if isinstance(cached_filepath, str):
436
- # The config file exists in cache- we can continue trying to load
437
- return True
438
-
439
- # NB: file_exists will only check for the existence of the config file on
440
- # hf_hub. This will fail in offline mode.
441
-
442
- # Call HF to check if the file exists
443
- return file_exists(
444
- str(model), config_name, revision=revision, token=_get_hf_token()
445
- )
446
-
447
-
291
def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> None:
292
"""Some models may have no rope_theta in their config but still use RoPE.
293
This function sets a default rope_theta if it's missing."""
@@ -836,72 +679,6 @@ def get_config(
679
return config
680
681
839
-def try_get_local_file(
840
- model: str | Path, file_name: str, revision: str | None = "main"
841
-) -> Path | None:
842
- file_path = Path(model) / file_name
843
- if file_path.is_file():
844
- return file_path
845
- else:
846
- try:
847
- cached_filepath = try_to_load_from_cache(
848
- repo_id=model, filename=file_name, revision=revision
849
- )
850
- if isinstance(cached_filepath, str):
851
- return Path(cached_filepath)
852
- except ValueError:
853
- ...
854
- return None
855
-
856
-
857
-def get_hf_file_to_dict(
858
- file_name: str, model: str | Path, revision: str | None = "main"
859
-):
860
- """
861
- Downloads a file from the Hugging Face Hub and returns
862
- its contents as a dictionary.
863
-
864
- Parameters:
865
- - file_name (str): The name of the file to download.
866
- - model (str): The name of the model on the Hugging Face Hub.
867
- - revision (str): The specific version of the model.
868
-
869
- Returns:
870
- - config_dict (dict): A dictionary containing
871
- the contents of the downloaded file.
872
- """
873
-
874
- file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
875
-
876
- if file_path is None:
877
- try:
878
- hf_hub_file = hf_hub_download(model, file_name, revision=revision)
879
- except huggingface_hub.errors.OfflineModeIsEnabled:
880
- return None
881
- except (
882
- RepositoryNotFoundError,
883
- RevisionNotFoundError,
884
- EntryNotFoundError,
885
- LocalEntryNotFoundError,
886
- ) as e:
887
- logger.debug("File or repository not found in hf_hub_download", e)
888
- return None
889
- except HfHubHTTPError as e:
890
- logger.warning(
891
- "Cannot connect to Hugging Face Hub. Skipping file download for '%s':",
892
- file_name,
893
- exc_info=e,
894
- )
895
- return None
896
- file_path = Path(hf_hub_file)
897
-
898
- if file_path is not None and file_path.is_file():
899
- with open(file_path) as file:
900
- return json.load(file)
901
-
902
- return None
903
-
904
-
682
@cache
683
def get_pooling_config(model: str, revision: str | None = "main") -> dict | None:
684
"""
@@ -1316,41 +1093,3 @@ def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int:
1093
)
1094
1095
return max_position_embeddings
1319
-
1320
-
1321
-def get_model_path(model: str | Path, revision: str | None = None):
1322
- if os.path.exists(model):
1323
- return model
1324
- assert huggingface_hub.constants.HF_HUB_OFFLINE
1325
- common_kwargs = {
1326
- "local_files_only": huggingface_hub.constants.HF_HUB_OFFLINE,
1327
- "revision": revision,
1328
- }
1329
-
1330
- if envs.VLLM_USE_MODELSCOPE:
1331
- from modelscope.hub.snapshot_download import snapshot_download
1332
-
1333
- return snapshot_download(model_id=model, **common_kwargs)
1334
-
1335
- from huggingface_hub import snapshot_download
1336
-
1337
- return snapshot_download(repo_id=model, **common_kwargs)
1338
-
1339
-
1340
-def get_hf_file_bytes(
1341
- file_name: str, model: str | Path, revision: str | None = "main"
1342
-) -> bytes | None:
1343
- """Get file contents from HuggingFace repository as bytes."""
1344
- file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
1345
-
1346
- if file_path is None:
1347
- hf_hub_file = hf_hub_download(
1348
- model, file_name, revision=revision, token=_get_hf_token()
1349
- )
1350
- file_path = Path(hf_hub_file)
1351
-
1352
- if file_path is not None and file_path.is_file():
1353
- with open(file_path, "rb") as file:
1354
- return file.read()
1355
-
1356
- return None
vllm/transformers_utils/gguf_utils.py
+1
-1
@@ -9,7 +9,7 @@ from gguf.constants import Keys, VisionProjectorType
9
from transformers import Gemma3Config, PretrainedConfig, SiglipVisionConfig
10
11
from vllm.logger import init_logger
12
-from vllm.transformers_utils.config import list_filtered_repo_files
12
+from vllm.transformers_utils.repo_utils import list_filtered_repo_files
13
14
logger = init_logger(__name__)
15
vllm/transformers_utils/repo_utils.py
new
+287
@@ -0,0 +1,287 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Utilities for model repo interaction."""
4
+
5
+import fnmatch
6
+import json
7
+import os
8
+import time
9
+from collections.abc import Callable
10
+from functools import cache
11
+from pathlib import Path
12
+from typing import TypeVar
13
+
14
+import huggingface_hub
15
+from huggingface_hub import (
16
+ hf_hub_download,
17
+ try_to_load_from_cache,
18
+)
19
+from huggingface_hub import list_repo_files as hf_list_repo_files
20
+from huggingface_hub.utils import (
21
+ EntryNotFoundError,
22
+ HfHubHTTPError,
23
+ LocalEntryNotFoundError,
24
+ RepositoryNotFoundError,
25
+ RevisionNotFoundError,
26
+)
27
+
28
+from vllm import envs
29
+from vllm.logger import init_logger
30
+
31
+logger = init_logger(__name__)
32
+
33
+
34
+def _get_hf_token() -> str | None:
35
+ """
36
+ Get the HuggingFace token from environment variable.
37
+
38
+ Returns None if the token is not set, is an empty string,
39
+ or contains only whitespace.
40
+ This follows the same pattern as huggingface_hub library which
41
+ treats empty string tokens as None to avoid authentication errors.
42
+ """
43
+ token = os.getenv("HF_TOKEN")
44
+ if token and token.strip():
45
+ return token
46
+ return None
47
+
48
+
49
+_R = TypeVar("_R")
50
+
51
+
52
+def with_retry(
53
+ func: Callable[[], _R],
54
+ log_msg: str,
55
+ max_retries: int = 2,
56
+ retry_delay: int = 2,
57
+) -> _R:
58
+ for attempt in range(max_retries):
59
+ try:
60
+ return func()
61
+ except Exception as e:
62
+ if attempt == max_retries - 1:
63
+ logger.error("%s: %s", log_msg, e)
64
+ raise
65
+ logger.error(
66
+ "%s: %s, retrying %d of %d", log_msg, e, attempt + 1, max_retries
67
+ )
68
+ time.sleep(retry_delay)
69
+ retry_delay *= 2
70
+
71
+ raise AssertionError("Should not be reached")
72
+
73
+
74
+# @cache doesn't cache exceptions
75
+@cache
76
+def list_repo_files(
77
+ repo_id: str,
78
+ *,
79
+ revision: str | None = None,
80
+ repo_type: str | None = None,
81
+ token: str | bool | None = None,
82
+) -> list[str]:
83
+ def lookup_files() -> list[str]:
84
+ # directly list files if model is local
85
+ if (local_path := Path(repo_id)).exists():
86
+ return [
87
+ str(file.relative_to(local_path))
88
+ for file in local_path.rglob("*")
89
+ if file.is_file()
90
+ ]
91
+ # if model is remote, use hf_hub api to list files
92
+ try:
93
+ if envs.VLLM_USE_MODELSCOPE:
94
+ from vllm.transformers_utils.utils import modelscope_list_repo_files
95
+
96
+ return modelscope_list_repo_files(
97
+ repo_id,
98
+ revision=revision,
99
+ token=os.getenv("MODELSCOPE_API_TOKEN", None),
100
+ )
101
+ return hf_list_repo_files(
102
+ repo_id, revision=revision, repo_type=repo_type, token=token
103
+ )
104
+ except huggingface_hub.errors.OfflineModeIsEnabled:
105
+ # Don't raise in offline mode,
106
+ # all we know is that we don't have this
107
+ # file cached.
108
+ return []
109
+
110
+ return with_retry(lookup_files, "Error retrieving file list")
111
+
112
+
113
+def list_filtered_repo_files(
114
+ model_name_or_path: str,
115
+ allow_patterns: list[str],
116
+ revision: str | None = None,
117
+ repo_type: str | None = None,
118
+ token: str | bool | None = None,
119
+) -> list[str]:
120
+ try:
121
+ all_files = list_repo_files(
122
+ repo_id=model_name_or_path,
123
+ revision=revision,
124
+ token=token,
125
+ repo_type=repo_type,
126
+ )
127
+ except Exception:
128
+ logger.error(
129
+ "Error retrieving file list. Please ensure your `model_name_or_path`"
130
+ "`repo_type`, `token` and `revision` arguments are correctly set. "
131
+ "Returning an empty list."
132
+ )
133
+ return []
134
+
135
+ file_list = []
136
+ # Filter patterns on filenames
137
+ for pattern in allow_patterns:
138
+ file_list.extend(
139
+ [
140
+ file
141
+ for file in all_files
142
+ if fnmatch.fnmatch(os.path.basename(file), pattern)
143
+ ]
144
+ )
145
+ return file_list
146
+
147
+
148
+def file_exists(
149
+ repo_id: str,
150
+ file_name: str,
151
+ *,
152
+ repo_type: str | None = None,
153
+ revision: str | None = None,
154
+ token: str | bool | None = None,
155
+) -> bool:
156
+ file_list = list_repo_files(
157
+ repo_id, repo_type=repo_type, revision=revision, token=token
158
+ )
159
+ return file_name in file_list
160
+
161
+
162
+# In offline mode the result can be a false negative
163
+def file_or_path_exists(
164
+ model: str | Path, config_name: str, revision: str | None
165
+) -> bool:
166
+ if (local_path := Path(model)).exists():
167
+ return (local_path / config_name).is_file()
168
+
169
+ # Offline mode support: Check if config file is cached already
170
+ cached_filepath = try_to_load_from_cache(
171
+ repo_id=model, filename=config_name, revision=revision
172
+ )
173
+ if isinstance(cached_filepath, str):
174
+ # The config file exists in cache- we can continue trying to load
175
+ return True
176
+
177
+ # NB: file_exists will only check for the existence of the config file on
178
+ # hf_hub. This will fail in offline mode.
179
+
180
+ # Call HF to check if the file exists
181
+ return file_exists(
182
+ str(model), config_name, revision=revision, token=_get_hf_token()
183
+ )
184
+
185
+
186
+def get_model_path(model: str | Path, revision: str | None = None):
187
+ if os.path.exists(model):
188
+ return model
189
+ assert huggingface_hub.constants.HF_HUB_OFFLINE
190
+ common_kwargs = {
191
+ "local_files_only": huggingface_hub.constants.HF_HUB_OFFLINE,
192
+ "revision": revision,
193
+ }
194
+
195
+ if envs.VLLM_USE_MODELSCOPE:
196
+ from modelscope.hub.snapshot_download import snapshot_download
197
+
198
+ return snapshot_download(model_id=model, **common_kwargs)
199
+
200
+ from huggingface_hub import snapshot_download
201
+
202
+ return snapshot_download(repo_id=model, **common_kwargs)
203
+
204
+
205
+def get_hf_file_bytes(
206
+ file_name: str, model: str | Path, revision: str | None = "main"
207
+) -> bytes | None:
208
+ """Get file contents from HuggingFace repository as bytes."""
209
+ file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
210
+
211
+ if file_path is None:
212
+ hf_hub_file = hf_hub_download(
213
+ model, file_name, revision=revision, token=_get_hf_token()
214
+ )
215
+ file_path = Path(hf_hub_file)
216
+
217
+ if file_path is not None and file_path.is_file():
218
+ with open(file_path, "rb") as file:
219
+ return file.read()
220
+
221
+ return None
222
+
223
+
224
+def try_get_local_file(
225
+ model: str | Path, file_name: str, revision: str | None = "main"
226
+) -> Path | None:
227
+ file_path = Path(model) / file_name
228
+ if file_path.is_file():
229
+ return file_path
230
+ else:
231
+ try:
232
+ cached_filepath = try_to_load_from_cache(
233
+ repo_id=model, filename=file_name, revision=revision
234
+ )
235
+ if isinstance(cached_filepath, str):
236
+ return Path(cached_filepath)
237
+ except ValueError:
238
+ ...
239
+ return None
240
+
241
+
242
+def get_hf_file_to_dict(
243
+ file_name: str, model: str | Path, revision: str | None = "main"
244
+):
245
+ """
246
+ Downloads a file from the Hugging Face Hub and returns
247
+ its contents as a dictionary.
248
+
249
+ Parameters:
250
+ - file_name (str): The name of the file to download.
251
+ - model (str): The name of the model on the Hugging Face Hub.
252
+ - revision (str): The specific version of the model.
253
+
254
+ Returns:
255
+ - config_dict (dict): A dictionary containing
256
+ the contents of the downloaded file.
257
+ """
258
+
259
+ file_path = try_get_local_file(model=model, file_name=file_name, revision=revision)
260
+
261
+ if file_path is None:
262
+ try:
263
+ hf_hub_file = hf_hub_download(model, file_name, revision=revision)
264
+ except huggingface_hub.errors.OfflineModeIsEnabled:
265
+ return None
266
+ except (
267
+ RepositoryNotFoundError,
268
+ RevisionNotFoundError,
269
+ EntryNotFoundError,
270
+ LocalEntryNotFoundError,
271
+ ) as e:
272
+ logger.debug("File or repository not found in hf_hub_download", e)
273
+ return None
274
+ except HfHubHTTPError as e:
275
+ logger.warning(
276
+ "Cannot connect to Hugging Face Hub. Skipping file download for '%s':",
277
+ file_name,
278
+ exc_info=e,
279
+ )
280
+ return None
281
+ file_path = Path(hf_hub_file)
282
+
283
+ if file_path is not None and file_path.is_file():
284
+ with open(file_path) as file:
285
+ return json.load(file)
286
+
287
+ return None
vllm/transformers_utils/tokenizer.py
+2
-4
@@ -15,11 +15,9 @@ from typing_extensions import assert_never
15
16
from vllm import envs
17
from vllm.logger import init_logger
18
-from vllm.transformers_utils.config import (
19
- get_sentence_transformer_tokenizer_config,
20
- list_filtered_repo_files,
21
-)
18
+from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config
19
from vllm.transformers_utils.gguf_utils import get_gguf_file_path_from_hf
20
+from vllm.transformers_utils.repo_utils import list_filtered_repo_files
21
from vllm.transformers_utils.tokenizers import MistralTokenizer
22
from vllm.transformers_utils.utils import (
23
check_gguf_file,