A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Support download models from www.modelscope.cn (#1588)
liuyhwangyh committed
Nov 18, 2023 at 12:38 UTC
edb305584bda536ee2b35db9c3e0615b43a5ee02
4 files changed
+58
-4
docs/source/getting_started/quickstart.rst
+27
@@ -40,6 +40,16 @@ Initialize vLLM's engine for offline inference with the ``LLM`` class and the `O
40
41
llm = LLM(model="facebook/opt-125m")
42
43
+Use model from www.modelscope.cn
44
+
45
+.. code-block:: shell
46
+
47
+ export VLLM_USE_MODELSCOPE=True
48
+
49
+.. code-block:: python
50
+
51
+ llm = LLM(model="qwen/Qwen-7B-Chat", revision="v1.1.8", trust_remote_code=True)
52
+
53
Call ``llm.generate`` to generate the outputs. It adds the input prompts to vLLM engine's waiting queue and executes the vLLM engine to generate the outputs with high throughput. The outputs are returned as a list of ``RequestOutput`` objects, which include all the output tokens.
54
55
.. code-block:: python
@@ -67,6 +77,16 @@ Start the server:
77
78
$ python -m vllm.entrypoints.api_server
79
80
+Use model from www.modelscope.cn
81
+
82
+.. code-block:: console
83
+
84
+ $ VLLM_USE_MODELSCOPE=True python -m vllm.entrypoints.api_server \
85
+ $ --model="qwen/Qwen-7B-Chat" \
86
+ $ --revision="v1.1.8" \
87
+ $ --trust-remote-code
88
+
89
+
90
By default, this command starts the server at ``http://localhost:8000`` with the OPT-125M model.
91
92
Query the model in shell:
@@ -95,6 +115,13 @@ Start the server:
115
$ python -m vllm.entrypoints.openai.api_server \
116
$ --model facebook/opt-125m
117
118
+Use model from www.modelscope.cn
119
+
120
+.. code-block:: console
121
+
122
+ $ VLLM_USE_MODELSCOPE=True python -m vllm.entrypoints.openai.api_server \
123
+ $ --model="qwen/Qwen-7B-Chat" --revision="v1.1.8" --trust-remote-code
124
+
125
By default, it starts the server at ``http://localhost:8000``. You can specify the address with ``--host`` and ``--port`` arguments. The server currently hosts one model at a time (OPT-125M in the above command) and implements `list models <https://platform.openai.com/docs/api-reference/models/list>`_ and `create completion <https://platform.openai.com/docs/api-reference/completions/create>`_ endpoints. We are actively adding support for more endpoints.
126
127
This server can be queried in the same format as OpenAI API. For example, list the models:
docs/source/models/supported_models.rst
+14
@@ -81,4 +81,18 @@ Alternatively, you can raise an issue on our `GitHub <https://github.com/vllm-pr
81
output = llm.generate("Hello, my name is")
82
print(output)
83
84
+ To use model from www.modelscope.cn
85
+
86
+ .. code-block:: shell
87
+
88
+ $ export VLLM_USE_MODELSCOPE=True
89
+
90
+ .. code-block:: python
91
+
92
+ from vllm import LLM
93
+
94
+ llm = LLM(model=..., revision=..., trust_remote_code=True) # Name or path of your model
95
+ output = llm.generate("Hello, my name is")
96
+ print(output)
97
+
98
If vLLM successfully generates text, it indicates that your model is supported.
vllm/config.py
+13
-1
@@ -1,4 +1,5 @@
1
from typing import Optional, Union
2
+import os
3
4
import torch
5
from transformers import PretrainedConfig
@@ -76,7 +77,18 @@ class ModelConfig:
77
self.tokenizer_revision = tokenizer_revision
78
self.quantization = quantization
79
79
- self.hf_config = get_config(model, trust_remote_code, revision)
80
+ if os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true":
81
+ # download model from ModelScope hub,
82
+ # lazy import so that modelscope is not required for normal use.
83
+ from modelscope.hub.snapshot_download import snapshot_download # pylint: disable=C
84
+ model_path = snapshot_download(model_id=model,
85
+ cache_dir=download_dir,
86
+ revision=revision)
87
+ self.model = model_path
88
+ self.download_dir = model_path
89
+ self.tokenizer = model_path
90
+
91
+ self.hf_config = get_config(self.model, trust_remote_code, revision)
92
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
93
self.max_model_len = _get_and_verify_max_len(self.hf_config,
94
max_model_len)
vllm/entrypoints/openai/api_server.py
+4
-3
@@ -648,9 +648,10 @@ if __name__ == "__main__":
648
max_model_len = engine_model_config.max_model_len
649
650
# A separate tokenizer to map token IDs to strings.
651
- tokenizer = get_tokenizer(engine_args.tokenizer,
652
- tokenizer_mode=engine_args.tokenizer_mode,
653
- trust_remote_code=engine_args.trust_remote_code)
651
+ tokenizer = get_tokenizer(
652
+ engine_model_config.tokenizer,
653
+ tokenizer_mode=engine_model_config.tokenizer_mode,
654
+ trust_remote_code=engine_model_config.trust_remote_code)
655
656
uvicorn.run(app,
657
host=args.host,