@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 39d3f8d94f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Fix code for downloading models from modelscope (#8443)

tastelikefeet committed Sep 28, 2024 at 23:24 UTC 39d3f8d94fd2691b70ee809e7565402f8a061c6b
1 file changed +17
vllm/transformers_utils/__init__.py
+17
@@ -0,0 +1,17 @@
1 +from vllm.envs import VLLM_USE_MODELSCOPE
2 +
3 +if VLLM_USE_MODELSCOPE:
4 + # Patch here, before each import happens
5 + import modelscope
6 + from packaging import version
7 +
8 + # patch_hub begins from modelscope>=1.18.1
9 + if version.parse(modelscope.__version__) <= version.parse('1.18.0'):
10 + raise ImportError(
11 + 'Using vLLM with ModelScope needs modelscope>=1.18.1, please '
12 + 'install by `pip install modelscope>=1.18.1`')
13 +
14 + from modelscope.utils.hf_util import patch_hub
15 +
16 + # Patch hub to download models from modelscope to speed up.
17 + patch_hub()