A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Core][VLM] Test registration for OOT multimodal models (#8717)
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Roger Wang committed
Oct 4, 2024 at 10:38 UTC
26aa325f4ffe8bf1d9b921535cc02fb31d80a96d
12 files changed
+227
-49
docs/source/models/adding_model.rst
+15
-3
@@ -85,16 +85,16 @@ When it comes to the linear layers, we provide the following options to parallel
85
* :code:`ReplicatedLinear`: Replicates the inputs and weights across multiple GPUs. No memory saving.
86
* :code:`RowParallelLinear`: The input tensor is partitioned along the hidden dimension. The weight matrix is partitioned along the rows (input dimension). An *all-reduce* operation is performed after the matrix multiplication to reduce the results. Typically used for the second FFN layer and the output linear transformation of the attention layer.
87
* :code:`ColumnParallelLinear`: The input tensor is replicated. The weight matrix is partitioned along the columns (output dimension). The result is partitioned along the column dimension. Typically used for the first FFN layer and the separated QKV transformation of the attention layer in the original Transformer.
88
-* :code:`MergedColumnParallelLinear`: Column-parallel linear that merges multiple `ColumnParallelLinear` operators. Typically used for the first FFN layer with weighted activation functions (e.g., SiLU). This class handles the sharded weight loading logic of multiple weight matrices.
88
+* :code:`MergedColumnParallelLinear`: Column-parallel linear that merges multiple :code:`ColumnParallelLinear` operators. Typically used for the first FFN layer with weighted activation functions (e.g., SiLU). This class handles the sharded weight loading logic of multiple weight matrices.
89
* :code:`QKVParallelLinear`: Parallel linear layer for the query, key, and value projections of the multi-head and grouped-query attention mechanisms. When number of key/value heads are less than the world size, this class replicates the key/value heads properly. This class handles the weight loading and replication of the weight matrices.
90
91
-Note that all the linear layers above take `linear_method` as an input. vLLM will set this parameter according to different quantization schemes to support weight quantization.
91
+Note that all the linear layers above take :code:`linear_method` as an input. vLLM will set this parameter according to different quantization schemes to support weight quantization.
92
93
4. Implement the weight loading logic
94
-------------------------------------
95
96
You now need to implement the :code:`load_weights` method in your :code:`*ForCausalLM` class.
97
-This method should load the weights from the HuggingFace's checkpoint file and assign them to the corresponding layers in your model. Specifically, for `MergedColumnParallelLinear` and `QKVParallelLinear` layers, if the original model has separated weight matrices, you need to load the different parts separately.
97
+This method should load the weights from the HuggingFace's checkpoint file and assign them to the corresponding layers in your model. Specifically, for :code:`MergedColumnParallelLinear` and :code:`QKVParallelLinear` layers, if the original model has separated weight matrices, you need to load the different parts separately.
98
99
5. Register your model
100
----------------------
@@ -114,6 +114,18 @@ Just add the following lines in your code:
114
from your_code import YourModelForCausalLM
115
ModelRegistry.register_model("YourModelForCausalLM", YourModelForCausalLM)
116
117
+If your model imports modules that initialize CUDA, consider instead lazy-importing it to avoid an error like :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`:
118
+
119
+.. code-block:: python
120
+
121
+ from vllm import ModelRegistry
122
+
123
+ ModelRegistry.register_model("YourModelForCausalLM", "your_code:YourModelForCausalLM")
124
+
125
+.. important::
126
+ If your model is a multimodal model, make sure the model class implements the :class:`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.
127
+ Read more about that :ref:`here <enabling_multimodal_inputs>`.
128
+
129
If you are running api server with :code:`vllm serve <args>`, you can wrap the entrypoint with the following code:
130
131
.. code-block:: python
find_cuda_init.py
new
+33
@@ -0,0 +1,33 @@
1
+import importlib
2
+import traceback
3
+from typing import Callable
4
+from unittest.mock import patch
5
+
6
+
7
+def find_cuda_init(fn: Callable[[], object]) -> None:
8
+ """
9
+ Helper function to debug CUDA re-initialization errors.
10
+
11
+ If `fn` initializes CUDA, prints the stack trace of how this happens.
12
+ """
13
+ from torch.cuda import _lazy_init
14
+
15
+ stack = None
16
+
17
+ def wrapper():
18
+ nonlocal stack
19
+ stack = traceback.extract_stack()
20
+ return _lazy_init()
21
+
22
+ with patch("torch.cuda._lazy_init", wrapper):
23
+ fn()
24
+
25
+ if stack is not None:
26
+ print("==== CUDA Initialized ====")
27
+ print("".join(traceback.format_list(stack)).strip())
28
+ print("==========================")
29
+
30
+
31
+if __name__ == "__main__":
32
+ find_cuda_init(
33
+ lambda: importlib.import_module("vllm.model_executor.models.llava"))
tests/conftest.py
+25
-5
@@ -879,15 +879,16 @@ def num_gpus_available():
879
880
881
temp_dir = tempfile.gettempdir()
882
-_dummy_path = os.path.join(temp_dir, "dummy_opt")
882
+_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
883
+_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
884
885
886
@pytest.fixture
887
def dummy_opt_path():
887
- json_path = os.path.join(_dummy_path, "config.json")
888
- if not os.path.exists(_dummy_path):
888
+ json_path = os.path.join(_dummy_opt_path, "config.json")
889
+ if not os.path.exists(_dummy_opt_path):
890
snapshot_download(repo_id="facebook/opt-125m",
890
- local_dir=_dummy_path,
891
+ local_dir=_dummy_opt_path,
892
ignore_patterns=[
893
"*.bin", "*.bin.index.json", "*.pt", "*.h5",
894
"*.msgpack"
@@ -898,4 +899,23 @@ def dummy_opt_path():
899
config["architectures"] = ["MyOPTForCausalLM"]
900
with open(json_path, "w") as f:
901
json.dump(config, f)
901
- return _dummy_path
902
+ return _dummy_opt_path
903
+
904
+
905
+@pytest.fixture
906
+def dummy_llava_path():
907
+ json_path = os.path.join(_dummy_llava_path, "config.json")
908
+ if not os.path.exists(_dummy_llava_path):
909
+ snapshot_download(repo_id="llava-hf/llava-1.5-7b-hf",
910
+ local_dir=_dummy_llava_path,
911
+ ignore_patterns=[
912
+ "*.bin", "*.bin.index.json", "*.pt", "*.h5",
913
+ "*.msgpack"
914
+ ])
915
+ assert os.path.exists(json_path)
916
+ with open(json_path, "r") as f:
917
+ config = json.load(f)
918
+ config["architectures"] = ["MyLlava"]
919
+ with open(json_path, "w") as f:
920
+ json.dump(config, f)
921
+ return _dummy_llava_path
tests/entrypoints/openai/test_audio.py
+3
-1
@@ -21,7 +21,9 @@ def server():
21
"--dtype",
22
"bfloat16",
23
"--max-model-len",
24
- "4096",
24
+ "2048",
25
+ "--max-num-seqs",
26
+ "5",
27
"--enforce-eager",
28
]
29
tests/entrypoints/openai/test_vision.py
+10
-3
@@ -23,9 +23,16 @@ TEST_IMAGE_URLS = [
23
@pytest.fixture(scope="module")
24
def server():
25
args = [
26
- "--dtype", "bfloat16", "--max-model-len", "4096", "--max-num-seqs",
27
- "5", "--enforce-eager", "--trust-remote-code", "--limit-mm-per-prompt",
28
- f"image={MAXIMUM_IMAGES}"
26
+ "--dtype",
27
+ "bfloat16",
28
+ "--max-model-len",
29
+ "2048",
30
+ "--max-num-seqs",
31
+ "5",
32
+ "--enforce-eager",
33
+ "--trust-remote-code",
34
+ "--limit-mm-per-prompt",
35
+ f"image={MAXIMUM_IMAGES}",
36
]
37
38
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
tests/models/test_oot_registration.py
+38
@@ -3,6 +3,7 @@ import os
3
import pytest
4
5
from vllm import LLM, SamplingParams
6
+from vllm.assets.image import ImageAsset
7
8
from ..utils import fork_new_process_for_each_test
9
@@ -29,3 +30,40 @@ def test_oot_registration(dummy_opt_path):
30
# make sure only the first token is generated
31
rest = generated_text.replace(first_token, "")
32
assert rest == ""
33
+
34
+
35
+image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
36
+
37
+
38
+@fork_new_process_for_each_test
39
+def test_oot_multimodal_registration(dummy_llava_path):
40
+ os.environ["VLLM_PLUGINS"] = "register_dummy_model"
41
+ prompts = [{
42
+ "prompt": "What's in the image?<image>",
43
+ "multi_modal_data": {
44
+ "image": image
45
+ },
46
+ }, {
47
+ "prompt": "Describe the image<image>",
48
+ "multi_modal_data": {
49
+ "image": image
50
+ },
51
+ }]
52
+
53
+ sampling_params = SamplingParams(temperature=0)
54
+ llm = LLM(model=dummy_llava_path,
55
+ load_format="dummy",
56
+ max_num_seqs=1,
57
+ trust_remote_code=True,
58
+ gpu_memory_utilization=0.98,
59
+ max_model_len=4096,
60
+ enforce_eager=True,
61
+ limit_mm_per_prompt={"image": 1})
62
+ first_token = llm.get_tokenizer().decode(0)
63
+ outputs = llm.generate(prompts, sampling_params)
64
+
65
+ for output in outputs:
66
+ generated_text = output.outputs[0].text
67
+ # make sure only the first token is generated
68
+ rest = generated_text.replace(first_token, "")
69
+ assert rest == ""
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/__init__.py
+8
-20
@@ -1,26 +1,14 @@
1
-from typing import Optional
2
-
3
-import torch
4
-
1
from vllm import ModelRegistry
6
-from vllm.model_executor.models.opt import OPTForCausalLM
7
-from vllm.model_executor.sampling_metadata import SamplingMetadata
8
-
9
-
10
-class MyOPTForCausalLM(OPTForCausalLM):
11
-
12
- def compute_logits(
13
- self, hidden_states: torch.Tensor,
14
- sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
15
- # this dummy model always predicts the first token
16
- logits = super().compute_logits(hidden_states, sampling_metadata)
17
- if logits is not None:
18
- logits.zero_()
19
- logits[:, 0] += 1.0
20
- return logits
2
3
4
def register():
24
- # register our dummy model
5
+ # Test directly passing the model
6
+ from .my_opt import MyOPTForCausalLM
7
+
8
if "MyOPTForCausalLM" not in ModelRegistry.get_supported_archs():
9
ModelRegistry.register_model("MyOPTForCausalLM", MyOPTForCausalLM)
10
+
11
+ # Test passing lazy model
12
+ if "MyLlava" not in ModelRegistry.get_supported_archs():
13
+ ModelRegistry.register_model("MyLlava",
14
+ "vllm_add_dummy_model.my_llava:MyLlava")
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
new
+28
@@ -0,0 +1,28 @@
1
+from typing import Optional
2
+
3
+import torch
4
+
5
+from vllm.inputs import INPUT_REGISTRY
6
+from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
7
+ dummy_data_for_llava,
8
+ get_max_llava_image_tokens,
9
+ input_processor_for_llava)
10
+from vllm.model_executor.sampling_metadata import SamplingMetadata
11
+from vllm.multimodal import MULTIMODAL_REGISTRY
12
+
13
+
14
+@MULTIMODAL_REGISTRY.register_image_input_mapper()
15
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
16
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
17
+@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
18
+class MyLlava(LlavaForConditionalGeneration):
19
+
20
+ def compute_logits(
21
+ self, hidden_states: torch.Tensor,
22
+ sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
23
+ # this dummy model always predicts the first token
24
+ logits = super().compute_logits(hidden_states, sampling_metadata)
25
+ if logits is not None:
26
+ logits.zero_()
27
+ logits[:, 0] += 1.0
28
+ return logits
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_opt.py
new
+19
@@ -0,0 +1,19 @@
1
+from typing import Optional
2
+
3
+import torch
4
+
5
+from vllm.model_executor.models.opt import OPTForCausalLM
6
+from vllm.model_executor.sampling_metadata import SamplingMetadata
7
+
8
+
9
+class MyOPTForCausalLM(OPTForCausalLM):
10
+
11
+ def compute_logits(
12
+ self, hidden_states: torch.Tensor,
13
+ sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
14
+ # this dummy model always predicts the first token
15
+ logits = super().compute_logits(hidden_states, sampling_metadata)
16
+ if logits is not None:
17
+ logits.zero_()
18
+ logits[:, 0] += 1.0
19
+ return logits
vllm/engine/arg_utils.py
+2
@@ -183,6 +183,8 @@ class EngineArgs:
183
def __post_init__(self):
184
if self.tokenizer is None:
185
self.tokenizer = self.model
186
+ from vllm.plugins import load_general_plugins
187
+ load_general_plugins()
188
189
@staticmethod
190
def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
vllm/engine/llm_engine.py
-3
@@ -290,9 +290,6 @@ class LLMEngine:
290
model_config.mm_processor_kwargs,
291
)
292
# TODO(woosuk): Print more configs in debug mode.
293
- from vllm.plugins import load_general_plugins
294
- load_general_plugins()
295
-
293
self.model_config = model_config
294
self.cache_config = cache_config
295
self.lora_config = lora_config
vllm/model_executor/models/registry.py
+46
-14
@@ -125,9 +125,10 @@ _MODELS = {
125
**_CONDITIONAL_GENERATION_MODELS,
126
}
127
128
-# Architecture -> type.
128
+# Architecture -> type or (module, class).
129
# out of tree models
130
_OOT_MODELS: Dict[str, Type[nn.Module]] = {}
131
+_OOT_MODELS_LAZY: Dict[str, Tuple[str, str]] = {}
132
133
# Models not supported by ROCm.
134
_ROCM_UNSUPPORTED_MODELS: List[str] = []
@@ -159,17 +160,24 @@ class ModelRegistry:
160
161
@staticmethod
162
def _get_module_cls_name(model_arch: str) -> Tuple[str, str]:
162
- module_relname, cls_name = _MODELS[model_arch]
163
- return f"vllm.model_executor.models.{module_relname}", cls_name
163
+ if model_arch in _MODELS:
164
+ module_relname, cls_name = _MODELS[model_arch]
165
+ return f"vllm.model_executor.models.{module_relname}", cls_name
166
+
167
+ if model_arch in _OOT_MODELS_LAZY:
168
+ return _OOT_MODELS_LAZY[model_arch]
169
+
170
+ raise KeyError(model_arch)
171
172
@staticmethod
173
@lru_cache(maxsize=128)
174
def _try_get_model_stateful(model_arch: str) -> Optional[Type[nn.Module]]:
168
- if model_arch not in _MODELS:
175
+ try:
176
+ mod_name, cls_name = ModelRegistry._get_module_cls_name(model_arch)
177
+ except KeyError:
178
return None
179
171
- module_name, cls_name = ModelRegistry._get_module_cls_name(model_arch)
172
- module = importlib.import_module(module_name)
180
+ module = importlib.import_module(mod_name)
181
return getattr(module, cls_name, None)
182
183
@staticmethod
@@ -219,14 +227,35 @@ class ModelRegistry:
227
return list(_MODELS.keys()) + list(_OOT_MODELS.keys())
228
229
@staticmethod
222
- def register_model(model_arch: str, model_cls: Type[nn.Module]):
230
+ def register_model(model_arch: str, model_cls: Union[Type[nn.Module],
231
+ str]):
232
+ """
233
+ Register an external model to be used in vLLM.
234
+
235
+ :code:`model_cls` can be either:
236
+
237
+ - A :class:`torch.nn.Module` class directly referencing the model.
238
+ - A string in the format :code:`<module>:<class>` which can be used to
239
+ lazily import the model. This is useful to avoid initializing CUDA
240
+ when importing the model and thus the related error
241
+ :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
242
+ """
243
if model_arch in _MODELS:
244
logger.warning(
245
"Model architecture %s is already registered, and will be "
246
"overwritten by the new model class %s.", model_arch,
227
- model_cls.__name__)
247
+ model_cls)
248
+
249
+ if isinstance(model_cls, str):
250
+ split_str = model_cls.split(":")
251
+ if len(split_str) != 2:
252
+ msg = "Expected a string in the format `<module>:<class>`"
253
+ raise ValueError(msg)
254
229
- _OOT_MODELS[model_arch] = model_cls
255
+ module_name, cls_name = split_str
256
+ _OOT_MODELS_LAZY[model_arch] = module_name, cls_name
257
+ else:
258
+ _OOT_MODELS[model_arch] = model_cls
259
260
@staticmethod
261
@lru_cache(maxsize=128)
@@ -248,13 +277,16 @@ class ModelRegistry:
277
if model is not None:
278
return func(model)
279
251
- if model_arch not in _MODELS and default is not None:
252
- return default
280
+ try:
281
+ mod_name, cls_name = ModelRegistry._get_module_cls_name(model_arch)
282
+ except KeyError:
283
+ if default is not None:
284
+ return default
285
254
- module_name, cls_name = ModelRegistry._get_module_cls_name(model_arch)
286
+ raise
287
288
valid_name_characters = string.ascii_letters + string.digits + "._"
257
- if any(s not in valid_name_characters for s in module_name):
289
+ if any(s not in valid_name_characters for s in mod_name):
290
raise ValueError(f"Unsafe module name detected for {model_arch}")
291
if any(s not in valid_name_characters for s in cls_name):
292
raise ValueError(f"Unsafe class name detected for {model_arch}")
@@ -266,7 +298,7 @@ class ModelRegistry:
298
err_id = uuid.uuid4()
299
300
stmts = ";".join([
269
- f"from {module_name} import {cls_name}",
301
+ f"from {mod_name} import {cls_name}",
302
f"from {func.__module__} import {func.__name__}",
303
f"assert {func.__name__}({cls_name}), '{err_id}'",
304
])