A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Misc][ModelScope] Change to use runtime VLLM_USE_MODELSCOPE (#18655)
Signed-off-by: Mengqing Cao <cmq0113@163.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
Mengqing Cao committed
May 25, 2025 at 12:51 UTC
6ab681bcbe9ba8318cf3aafd318f375ef8fd7de3
5 files changed
+19
-17
tests/test_regression.py
+3
@@ -60,6 +60,9 @@ def test_model_from_modelscope(monkeypatch: pytest.MonkeyPatch):
60
# model: https://modelscope.cn/models/qwen/Qwen1.5-0.5B-Chat/summary
61
with monkeypatch.context() as m:
62
m.setenv("VLLM_USE_MODELSCOPE", "True")
63
+ # Don't use HF_TOKEN for ModelScope repos, otherwise it will fail
64
+ # with 400 Client Error: Bad Request.
65
+ m.setenv("HF_TOKEN", "")
66
llm = LLM(model="qwen/Qwen1.5-0.5B-Chat")
67
68
prompts = [
vllm/model_executor/model_loader/default_loader.py
+2
-2
@@ -11,8 +11,8 @@ import torch
11
from torch import nn
12
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
13
14
+from vllm import envs
15
from vllm.config import LoadConfig, LoadFormat, ModelConfig, VllmConfig
15
-from vllm.envs import VLLM_USE_MODELSCOPE
16
from vllm.logger import init_logger
17
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
18
from vllm.model_executor.model_loader.utils import (
@@ -64,7 +64,7 @@ class DefaultModelLoader(BaseModelLoader):
64
65
Returns the path to the downloaded model, or None if the model is not
66
downloaded from ModelScope."""
67
- if VLLM_USE_MODELSCOPE:
67
+ if envs.VLLM_USE_MODELSCOPE:
68
# download model from ModelScope hub,
69
# lazy import so that modelscope is not required for normal use.
70
# pylint: disable=C.
vllm/transformers_utils/__init__.py
+2
-2
@@ -1,8 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from vllm.envs import VLLM_USE_MODELSCOPE
3
+from vllm import envs
4
5
-if VLLM_USE_MODELSCOPE:
5
+if envs.VLLM_USE_MODELSCOPE:
6
try:
7
# Patch here, before each import happens
8
import modelscope
vllm/transformers_utils/config.py
+10
-11
@@ -24,7 +24,7 @@ from transformers.models.auto.modeling_auto import (
24
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
25
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
26
27
-from vllm.envs import VLLM_USE_MODELSCOPE
27
+from vllm import envs
28
from vllm.logger import init_logger
29
# yapf conflicts with isort for this block
30
# yapf: disable
@@ -45,13 +45,12 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
45
from vllm.transformers_utils.utils import check_gguf_file
46
from vllm.utils import resolve_obj_by_qualname
47
48
-if VLLM_USE_MODELSCOPE:
48
+if envs.VLLM_USE_MODELSCOPE:
49
from modelscope import AutoConfig
50
else:
51
from transformers import AutoConfig
52
53
MISTRAL_CONFIG_NAME = "params.json"
54
-HF_TOKEN = os.getenv('HF_TOKEN', None)
54
55
logger = init_logger(__name__)
56
@@ -130,7 +129,7 @@ def list_repo_files(
129
]
130
# if model is remote, use hf_hub api to list files
131
try:
133
- if VLLM_USE_MODELSCOPE:
132
+ if envs.VLLM_USE_MODELSCOPE:
133
from vllm.transformers_utils.utils import (
134
modelscope_list_repo_files)
135
return modelscope_list_repo_files(repo_id,
@@ -185,7 +184,7 @@ def file_or_path_exists(model: Union[str, Path], config_name: str,
184
return file_exists(str(model),
185
config_name,
186
revision=revision,
188
- token=HF_TOKEN)
187
+ token=os.getenv('HF_TOKEN', None))
188
189
190
def patch_rope_scaling(config: PretrainedConfig) -> None:
@@ -312,7 +311,7 @@ def get_config(
311
model,
312
revision=revision,
313
code_revision=code_revision,
315
- token=HF_TOKEN,
314
+ token=os.getenv('HF_TOKEN', None),
315
**kwargs,
316
)
317
@@ -324,7 +323,7 @@ def get_config(
323
model,
324
revision=revision,
325
code_revision=code_revision,
327
- token=HF_TOKEN,
326
+ token=os.getenv('HF_TOKEN', None),
327
**kwargs,
328
)
329
else:
@@ -334,7 +333,7 @@ def get_config(
333
trust_remote_code=trust_remote_code,
334
revision=revision,
335
code_revision=code_revision,
337
- token=HF_TOKEN,
336
+ token=os.getenv('HF_TOKEN', None),
337
**kwargs,
338
)
339
except ValueError as e:
@@ -352,7 +351,7 @@ def get_config(
351
raise e
352
353
elif config_format == ConfigFormat.MISTRAL:
355
- config = load_params_config(model, revision, token=HF_TOKEN, **kwargs)
354
+ config = load_params_config(model, revision, **kwargs)
355
else:
356
supported_formats = [
357
fmt.value for fmt in ConfigFormat if fmt != ConfigFormat.AUTO
@@ -561,7 +560,7 @@ def get_sentence_transformer_tokenizer_config(model: str,
560
# If model is on HuggingfaceHub, get the repo files
561
repo_files = list_repo_files(model,
562
revision=revision,
564
- token=HF_TOKEN)
563
+ token=os.getenv('HF_TOKEN', None))
564
except Exception:
565
repo_files = []
566
@@ -768,7 +767,7 @@ def get_hf_image_processor_config(
767
**kwargs,
768
) -> dict[str, Any]:
769
# ModelScope does not provide an interface for image_processor
771
- if VLLM_USE_MODELSCOPE:
770
+ if envs.VLLM_USE_MODELSCOPE:
771
return dict()
772
# Separate model folder from file path for GGUF models
773
if check_gguf_file(model):
vllm/transformers_utils/tokenizer.py
+2
-2
@@ -13,7 +13,7 @@ import huggingface_hub
13
from transformers import (AutoTokenizer, PreTrainedTokenizer,
14
PreTrainedTokenizerFast)
15
16
-from vllm.envs import VLLM_USE_MODELSCOPE
16
+from vllm import envs
17
from vllm.logger import init_logger
18
from vllm.lora.request import LoRARequest
19
from vllm.transformers_utils.tokenizer_base import (TokenizerBase,
@@ -168,7 +168,7 @@ def get_tokenizer(
168
) -> AnyTokenizer:
169
"""Gets a tokenizer for the given model name via HuggingFace or ModelScope.
170
"""
171
- if VLLM_USE_MODELSCOPE:
171
+ if envs.VLLM_USE_MODELSCOPE:
172
# download model from ModelScope hub,
173
# lazy import so that modelscope is not required for normal use.
174
# pylint: disable=C.