A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
Bugfix: fix broken of download models from modelscope (#5233)
Co-authored-by: mulin.lyh <mulin.lyh@taobao.com>
liuyhwangyh committed
Jun 7, 2024 at 00:28 UTC
4efff036f0dfeee21e82044e9b6e63b861b817a3
3 files changed
+32
-2
tests/test_regression.py
+21
@@ -53,6 +53,27 @@ def test_gc():
53
assert allocated < 50 * 1024 * 1024
54
55
56
+def test_model_from_modelscope(monkeypatch):
57
+ # model: https://modelscope.cn/models/qwen/Qwen1.5-0.5B-Chat/summary
58
+ MODELSCOPE_MODEL_NAME = "qwen/Qwen1.5-0.5B-Chat"
59
+ monkeypatch.setenv("VLLM_USE_MODELSCOPE", "True")
60
+ try:
61
+ llm = LLM(model=MODELSCOPE_MODEL_NAME)
62
+
63
+ prompts = [
64
+ "Hello, my name is",
65
+ "The president of the United States is",
66
+ "The capital of France is",
67
+ "The future of AI is",
68
+ ]
69
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
70
+
71
+ outputs = llm.generate(prompts, sampling_params)
72
+ assert len(outputs) == 4
73
+ finally:
74
+ monkeypatch.delenv("VLLM_USE_MODELSCOPE", raising=False)
75
+
76
+
77
if __name__ == "__main__":
78
import pytest
79
pytest.main([__file__])
vllm/config.py
+5
-1
@@ -113,7 +113,11 @@ class ModelConfig:
113
self.revision = revision
114
self.code_revision = code_revision
115
self.rope_scaling = rope_scaling
116
- self.tokenizer_revision = tokenizer_revision
116
+ # The tokenizer version is consistent with the model version by default.
117
+ if tokenizer_revision is None:
118
+ self.tokenizer_revision = revision
119
+ else:
120
+ self.tokenizer_revision = tokenizer_revision
121
self.quantization = quantization
122
self.quantization_param_path = quantization_param_path
123
self.enforce_eager = enforce_eager
vllm/transformers_utils/config.py
+6
-1
@@ -1,7 +1,8 @@
1
from typing import Dict, Optional
2
3
-from transformers import AutoConfig, PretrainedConfig
3
+from transformers import PretrainedConfig
4
5
+from vllm.envs import VLLM_USE_MODELSCOPE
6
from vllm.logger import init_logger
7
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
8
JAISConfig, MPTConfig, RWConfig)
@@ -24,6 +25,10 @@ def get_config(model: str,
25
code_revision: Optional[str] = None,
26
rope_scaling: Optional[dict] = None) -> PretrainedConfig:
27
try:
28
+ if VLLM_USE_MODELSCOPE:
29
+ from modelscope import AutoConfig
30
+ else:
31
+ from transformers import AutoConfig
32
config = AutoConfig.from_pretrained(
33
model,
34
trust_remote_code=trust_remote_code,