A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Fix code for downloading models from modelscope (#8443)
tastelikefeet committed
Sep 28, 2024 at 23:24 UTC
39d3f8d94fd2691b70ee809e7565402f8a061c6b
1 file changed
+17
vllm/transformers_utils/__init__.py
+17
@@ -0,0 +1,17 @@
1
+from vllm.envs import VLLM_USE_MODELSCOPE
2
+
3
+if VLLM_USE_MODELSCOPE:
4
+ # Patch here, before each import happens
5
+ import modelscope
6
+ from packaging import version
7
+
8
+ # patch_hub begins from modelscope>=1.18.1
9
+ if version.parse(modelscope.__version__) <= version.parse('1.18.0'):
10
+ raise ImportError(
11
+ 'Using vLLM with ModelScope needs modelscope>=1.18.1, please '
12
+ 'install by `pip install modelscope>=1.18.1`')
13
+
14
+ from modelscope.utils.hf_util import patch_hub
15
+
16
+ # Patch hub to download models from modelscope to speed up.
17
+ patch_hub()