A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] enable out-of-tree model register (#3871)
youkaichao committed
Apr 6, 2024 at 17:11 UTC
95baec828f3ee046074dace1d88202a920b7dc15
6 files changed
+148
-2
.buildkite/test-pipeline.yaml
+4
-1
@@ -34,7 +34,10 @@ steps:
34
command: pytest -v -s engine tokenization test_sequence.py test_config.py
35
36
- label: Entrypoints Test
37
- command: pytest -v -s entrypoints
37
+ commands:
38
+ # these tests have to be separated, because each one will allocate all posible GPU memory
39
+ - pytest -v -s entrypoints --ignore=entrypoints/test_server_oot_registration.py
40
+ - pytest -v -s entrypoints/test_server_oot_registration.py
41
42
- label: Examples Test
43
working_dir: "/vllm-workspace/examples"
docs/source/models/adding_model.rst
+27
@@ -21,6 +21,8 @@ This document provides a high-level guide on integrating a `HuggingFace Transfor
21
Start by forking our `GitHub`_ repository and then :ref:`build it from source <build_from_source>`.
22
This gives you the ability to modify the codebase and test your model.
23
24
+.. tip::
25
+ If you don't want to fork the repository and modify vLLM's codebase, please refer to the "Out-of-Tree Model Integration" section below.
26
27
1. Bring your model code
28
------------------------
@@ -94,3 +96,28 @@ This method should load the weights from the HuggingFace's checkpoint file and a
96
----------------------
97
98
Finally, include your :code:`*ForCausalLM` class in `vllm/model_executor/models/__init__.py <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/__init__.py>`_ and register it to the :code:`_MODEL_REGISTRY` in `vllm/model_executor/model_loader.py <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/model_loader.py>`_.
99
+
100
+6. Out-of-Tree Model Integration
101
+--------------------------------------------
102
+
103
+We also provide a way to integrate a model without modifying the vLLM codebase. Step 2, 3, 4 are still required, but you can skip step 1 and 5.
104
+
105
+Just add the following lines in your code:
106
+
107
+.. code-block:: python
108
+
109
+ from vllm import ModelRegistry
110
+ from your_code import YourModelForCausalLM
111
+ ModelRegistry.register_model("YourModelForCausalLM", YourModelForCausalLM)
112
+
113
+If you are running api server with `python -m vllm.entrypoints.openai.api_server args`, you can wrap the entrypoint with the following code:
114
+
115
+.. code-block:: python
116
+
117
+ from vllm import ModelRegistry
118
+ from your_code import YourModelForCausalLM
119
+ ModelRegistry.register_model("YourModelForCausalLM", YourModelForCausalLM)
120
+ import runpy
121
+ runpy.run_module('vllm.entrypoints.openai.api_server', run_name='__main__')
122
+
123
+Save the above code in a file and run it with `python your_file.py args`.
tests/entrypoints/test_server_oot_registration.py
new
+66
@@ -0,0 +1,66 @@
1
+import multiprocessing
2
+import sys
3
+import time
4
+
5
+import torch
6
+from openai import OpenAI, OpenAIError
7
+
8
+from vllm import ModelRegistry
9
+from vllm.model_executor.models.opt import OPTForCausalLM
10
+from vllm.model_executor.sampling_metadata import SamplingMetadata
11
+from vllm.utils import get_open_port
12
+
13
+
14
+class MyOPTForCausalLM(OPTForCausalLM):
15
+
16
+ def compute_logits(self, hidden_states: torch.Tensor,
17
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
18
+ # this dummy model always predicts the first token
19
+ logits = super().compute_logits(hidden_states, sampling_metadata)
20
+ logits.zero_()
21
+ logits[:, 0] += 1.0
22
+ return logits
23
+
24
+
25
+def server_function(port):
26
+ # register our dummy model
27
+ ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
28
+ sys.argv = ["placeholder.py"] + \
29
+ ("--model facebook/opt-125m --dtype"
30
+ f" float32 --api-key token-abc123 --port {port}").split()
31
+ import runpy
32
+ runpy.run_module('vllm.entrypoints.openai.api_server', run_name='__main__')
33
+
34
+
35
+def test_oot_registration_for_api_server():
36
+ port = get_open_port()
37
+ server = multiprocessing.Process(target=server_function, args=(port, ))
38
+ server.start()
39
+ client = OpenAI(
40
+ base_url=f"http://localhost:{port}/v1",
41
+ api_key="token-abc123",
42
+ )
43
+ while True:
44
+ try:
45
+ completion = client.chat.completions.create(
46
+ model="facebook/opt-125m",
47
+ messages=[{
48
+ "role": "system",
49
+ "content": "You are a helpful assistant."
50
+ }, {
51
+ "role": "user",
52
+ "content": "Hello!"
53
+ }],
54
+ temperature=0,
55
+ )
56
+ break
57
+ except OpenAIError as e:
58
+ if "Connection error" in str(e):
59
+ time.sleep(3)
60
+ else:
61
+ raise e
62
+ server.kill()
63
+ generated_text = completion.choices[0].message.content
64
+ # make sure only the first token is generated
65
+ rest = generated_text.replace("<s>", "")
66
+ assert rest == ""
tests/models/test_oot_registration.py
new
+32
@@ -0,0 +1,32 @@
1
+import torch
2
+
3
+from vllm import LLM, ModelRegistry, SamplingParams
4
+from vllm.model_executor.models.opt import OPTForCausalLM
5
+from vllm.model_executor.sampling_metadata import SamplingMetadata
6
+
7
+
8
+class MyOPTForCausalLM(OPTForCausalLM):
9
+
10
+ def compute_logits(self, hidden_states: torch.Tensor,
11
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
12
+ # this dummy model always predicts the first token
13
+ logits = super().compute_logits(hidden_states, sampling_metadata)
14
+ logits.zero_()
15
+ logits[:, 0] += 1.0
16
+ return logits
17
+
18
+
19
+def test_oot_registration():
20
+ # register our dummy model
21
+ ModelRegistry.register_model("OPTForCausalLM", MyOPTForCausalLM)
22
+ prompts = ["Hello, my name is", "The text does not matter"]
23
+ sampling_params = SamplingParams(temperature=0)
24
+ llm = LLM(model="facebook/opt-125m")
25
+ first_token = llm.get_tokenizer().decode(0)
26
+ outputs = llm.generate(prompts, sampling_params)
27
+
28
+ for output in outputs:
29
+ generated_text = output.outputs[0].text
30
+ # make sure only the first token is generated
31
+ rest = generated_text.replace(first_token, "")
32
+ assert rest == ""
vllm/__init__.py
+2
@@ -5,6 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5
from vllm.engine.llm_engine import LLMEngine
6
from vllm.engine.ray_utils import initialize_ray_cluster
7
from vllm.entrypoints.llm import LLM
8
+from vllm.model_executor.models import ModelRegistry
9
from vllm.outputs import CompletionOutput, RequestOutput
10
from vllm.sampling_params import SamplingParams
11
@@ -12,6 +13,7 @@ __version__ = "0.4.0.post1"
13
14
__all__ = [
15
"LLM",
16
+ "ModelRegistry",
17
"SamplingParams",
18
"RequestOutput",
19
"CompletionOutput",
vllm/model_executor/models/__init__.py
+17
-1
@@ -1,5 +1,5 @@
1
import importlib
2
-from typing import List, Optional, Type
2
+from typing import Dict, List, Optional, Type
3
4
import torch.nn as nn
5
@@ -55,6 +55,10 @@ _MODELS = {
55
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
56
}
57
58
+# Architecture -> type.
59
+# out of tree models
60
+_OOT_MODELS: Dict[str, Type[nn.Module]] = {}
61
+
62
# Models not supported by ROCm.
63
_ROCM_UNSUPPORTED_MODELS = []
64
@@ -74,6 +78,8 @@ class ModelRegistry:
78
79
@staticmethod
80
def load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
81
+ if model_arch in _OOT_MODELS:
82
+ return _OOT_MODELS[model_arch]
83
if model_arch not in _MODELS:
84
return None
85
if is_hip():
@@ -95,6 +101,16 @@ class ModelRegistry:
101
def get_supported_archs() -> List[str]:
102
return list(_MODELS.keys())
103
104
+ @staticmethod
105
+ def register_model(model_arch: str, model_cls: Type[nn.Module]):
106
+ if model_arch in _MODELS:
107
+ logger.warning(
108
+ f"Model architecture {model_arch} is already registered, "
109
+ "and will be overwritten by the new model "
110
+ f"class {model_cls.__name__}.")
111
+ global _OOT_MODELS
112
+ _OOT_MODELS[model_arch] = model_cls
113
+
114
115
__all__ = [
116
"ModelRegistry",