@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4efff036f0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

Bugfix: fix broken of download models from modelscope (#5233)

Co-authored-by: mulin.lyh <mulin.lyh@taobao.com>

liuyhwangyh committed Jun 7, 2024 at 00:28 UTC 4efff036f0dfeee21e82044e9b6e63b861b817a3
3 files changed +32 -2
tests/test_regression.py
+21
@@ -53,6 +53,27 @@ def test_gc():
53 assert allocated < 50 * 1024 * 1024
54
55
56 +def test_model_from_modelscope(monkeypatch):
57 + # model: https://modelscope.cn/models/qwen/Qwen1.5-0.5B-Chat/summary
58 + MODELSCOPE_MODEL_NAME = "qwen/Qwen1.5-0.5B-Chat"
59 + monkeypatch.setenv("VLLM_USE_MODELSCOPE", "True")
60 + try:
61 + llm = LLM(model=MODELSCOPE_MODEL_NAME)
62 +
63 + prompts = [
64 + "Hello, my name is",
65 + "The president of the United States is",
66 + "The capital of France is",
67 + "The future of AI is",
68 + ]
69 + sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
70 +
71 + outputs = llm.generate(prompts, sampling_params)
72 + assert len(outputs) == 4
73 + finally:
74 + monkeypatch.delenv("VLLM_USE_MODELSCOPE", raising=False)
75 +
76 +
77 if __name__ == "__main__":
78 import pytest
79 pytest.main([__file__])
vllm/config.py
+5 -1
@@ -113,7 +113,11 @@ class ModelConfig:
113 self.revision = revision
114 self.code_revision = code_revision
115 self.rope_scaling = rope_scaling
116 - self.tokenizer_revision = tokenizer_revision
116 + # The tokenizer version is consistent with the model version by default.
117 + if tokenizer_revision is None:
118 + self.tokenizer_revision = revision
119 + else:
120 + self.tokenizer_revision = tokenizer_revision
121 self.quantization = quantization
122 self.quantization_param_path = quantization_param_path
123 self.enforce_eager = enforce_eager
vllm/transformers_utils/config.py
+6 -1
@@ -1,7 +1,8 @@
1 from typing import Dict, Optional
2
3 -from transformers import AutoConfig, PretrainedConfig
3 +from transformers import PretrainedConfig
4
5 +from vllm.envs import VLLM_USE_MODELSCOPE
6 from vllm.logger import init_logger
7 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
8 JAISConfig, MPTConfig, RWConfig)
@@ -24,6 +25,10 @@ def get_config(model: str,
25 code_revision: Optional[str] = None,
26 rope_scaling: Optional[dict] = None) -> PretrainedConfig:
27 try:
28 + if VLLM_USE_MODELSCOPE:
29 + from modelscope import AutoConfig
30 + else:
31 + from transformers import AutoConfig
32 config = AutoConfig.from_pretrained(
33 model,
34 trust_remote_code=trust_remote_code,