A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Misc] Use model_redirect to redirect the model name to a local folder. (#14116)
wang.yuqi committed
Mar 27, 2025 at 17:21 UTC
3f532cb6a69e51a6578b85642fcba34ac348f7a4
3 files changed
+50
-3
vllm/config.py
+7
-3
@@ -38,7 +38,7 @@ from vllm.transformers_utils.config import (
38
get_sentence_transformer_tokenizer_config, is_encoder_decoder,
39
try_get_generation_config, uses_mrope)
40
from vllm.transformers_utils.s3_utils import S3Model
41
-from vllm.transformers_utils.utils import is_s3
41
+from vllm.transformers_utils.utils import is_s3, maybe_model_redirect
42
from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless,
43
get_cpu_memory, random_uuid, resolve_obj_by_qualname)
44
@@ -266,9 +266,13 @@ class ModelConfig:
266
override_generation_config: Optional[dict[str, Any]] = None,
267
model_impl: Union[str, ModelImpl] = ModelImpl.AUTO,
268
) -> None:
269
- self.model = model
269
+ self.model = maybe_model_redirect(model)
270
+ self.tokenizer = maybe_model_redirect(tokenizer)
271
+
272
self.hf_config_path = hf_config_path
271
- self.tokenizer = tokenizer
273
+ if isinstance(hf_config_path, str):
274
+ self.hf_config_path = maybe_model_redirect(hf_config_path)
275
+
276
self.tokenizer_mode = tokenizer_mode
277
self.trust_remote_code = trust_remote_code
278
self.allowed_local_media_path = allowed_local_media_path
vllm/envs.py
+5
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
22
S3_ACCESS_KEY_ID: Optional[str] = None
23
S3_SECRET_ACCESS_KEY: Optional[str] = None
24
S3_ENDPOINT_URL: Optional[str] = None
25
+ VLLM_MODEL_REDIRECT_PATH: Optional[str] = None
26
VLLM_CACHE_ROOT: str = os.path.expanduser("~/.cache/vllm")
27
VLLM_CONFIG_ROOT: str = os.path.expanduser("~/.config/vllm")
28
VLLM_USAGE_STATS_SERVER: str = "https://stats.vllm.ai"
@@ -635,6 +636,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
636
"VLLM_CI_USE_S3":
637
lambda: os.environ.get("VLLM_CI_USE_S3", "0") == "1",
638
639
+ # Use model_redirect to redirect the model name to a local folder.
640
+ "VLLM_MODEL_REDIRECT_PATH":
641
+ lambda: os.environ.get("VLLM_MODEL_REDIRECT_PATH", None),
642
+
643
# Whether to use atomicAdd reduce in gptq/awq marlin kernel.
644
"VLLM_MARLIN_USE_ATOMIC_ADD":
645
lambda: os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1",
vllm/transformers_utils/utils.py
+38
@@ -1,9 +1,15 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+from functools import cache
4
from os import PathLike
5
from pathlib import Path
6
from typing import List, Optional, Union
7
8
+from vllm.envs import VLLM_MODEL_REDIRECT_PATH
9
+from vllm.logger import init_logger
10
+
11
+logger = init_logger(__name__)
12
+
13
14
def is_s3(model_or_path: str) -> bool:
15
return model_or_path.lower().startswith('s3://')
@@ -38,3 +44,35 @@ def modelscope_list_repo_files(
44
if file['Type'] == 'blob'
45
]
46
return files
47
+
48
+
49
+@cache
50
+def maybe_model_redirect(model: str) -> str:
51
+ """
52
+ Use model_redirect to redirect the model name to a local folder.
53
+
54
+ :param model: hf model name
55
+ :return: maybe redirect to a local folder
56
+ """
57
+
58
+ model_redirect_path = VLLM_MODEL_REDIRECT_PATH
59
+
60
+ if not model_redirect_path:
61
+ return model
62
+
63
+ if not Path(model_redirect_path).exists():
64
+ return model
65
+
66
+ with open(model_redirect_path) as f:
67
+ for line in f.readlines():
68
+ try:
69
+ model_name, redirect_name = line.split("\t")
70
+ if model == model_name:
71
+ redirect_name = redirect_name.strip()
72
+ logger.info("model redirect: [ %s ] -> [ %s ]", model,
73
+ redirect_name)
74
+ return redirect_name
75
+ except Exception:
76
+ pass
77
+
78
+ return model