A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] Explicit interface for vLLM models and support OOT embedding models (#9108)
Cyrus Leung committed
Oct 7, 2024 at 14:10 UTC
8c6de96ea1e6e51e49a170c28ad3efc16db9413e
10 files changed
+342
-37
tests/conftest.py
+20
@@ -871,6 +871,7 @@ def num_gpus_available():
871
temp_dir = tempfile.gettempdir()
872
_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
873
_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
874
+_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")
875
876
877
@pytest.fixture
@@ -909,3 +910,22 @@ def dummy_llava_path():
910
with open(json_path, "w") as f:
911
json.dump(config, f)
912
return _dummy_llava_path
913
+
914
+
915
+@pytest.fixture
916
+def dummy_gemma2_embedding_path():
917
+ json_path = os.path.join(_dummy_gemma2_embedding_path, "config.json")
918
+ if not os.path.exists(_dummy_gemma2_embedding_path):
919
+ snapshot_download(repo_id="BAAI/bge-multilingual-gemma2",
920
+ local_dir=_dummy_gemma2_embedding_path,
921
+ ignore_patterns=[
922
+ "*.bin", "*.bin.index.json", "*.pt", "*.h5",
923
+ "*.msgpack"
924
+ ])
925
+ assert os.path.exists(json_path)
926
+ with open(json_path, "r") as f:
927
+ config = json.load(f)
928
+ config["architectures"] = ["MyGemma2Embedding"]
929
+ with open(json_path, "w") as f:
930
+ json.dump(config, f)
931
+ return _dummy_gemma2_embedding_path
tests/models/test_oot_registration.py
+15
-3
@@ -2,7 +2,7 @@ import os
2
3
import pytest
4
5
-from vllm import LLM, SamplingParams
5
+from vllm import LLM, PoolingParams, SamplingParams
6
from vllm.assets.image import ImageAsset
7
8
from ..utils import fork_new_process_for_each_test
@@ -17,7 +17,7 @@ def test_plugin(dummy_opt_path):
17
18
19
@fork_new_process_for_each_test
20
-def test_oot_registration(dummy_opt_path):
20
+def test_oot_registration_text_generation(dummy_opt_path):
21
os.environ["VLLM_PLUGINS"] = "register_dummy_model"
22
prompts = ["Hello, my name is", "The text does not matter"]
23
sampling_params = SamplingParams(temperature=0)
@@ -32,11 +32,23 @@ def test_oot_registration(dummy_opt_path):
32
assert rest == ""
33
34
35
+@fork_new_process_for_each_test
36
+def test_oot_registration_embedding(dummy_gemma2_embedding_path):
37
+ os.environ["VLLM_PLUGINS"] = "register_dummy_model"
38
+ prompts = ["Hello, my name is", "The text does not matter"]
39
+ sampling_params = PoolingParams()
40
+ llm = LLM(model=dummy_gemma2_embedding_path, load_format="dummy")
41
+ outputs = llm.encode(prompts, sampling_params)
42
+
43
+ for output in outputs:
44
+ assert all(v == 0 for v in output.outputs.embedding)
45
+
46
+
47
image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
48
49
50
@fork_new_process_for_each_test
39
-def test_oot_multimodal_registration(dummy_llava_path):
51
+def test_oot_registration_multimodal(dummy_llava_path):
52
os.environ["VLLM_PLUGINS"] = "register_dummy_model"
53
prompts = [{
54
"prompt": "What's in the image?<image>",
tests/models/test_registry.py
+22
-2
@@ -3,7 +3,14 @@ import warnings
3
import pytest
4
import torch.cuda
5
6
-from vllm.model_executor.models import ModelRegistry
6
+from vllm.model_executor.models import (is_embedding_model,
7
+ is_text_generation_model,
8
+ supports_multimodal)
9
+from vllm.model_executor.models.registry import (_EMBEDDING_MODELS,
10
+ _MULTIMODAL_MODELS,
11
+ _SPECULATIVE_DECODING_MODELS,
12
+ _TEXT_GENERATION_MODELS,
13
+ ModelRegistry)
14
from vllm.platforms import current_platform
15
16
from ..utils import fork_new_process_for_each_test
@@ -12,7 +19,20 @@ from ..utils import fork_new_process_for_each_test
19
@pytest.mark.parametrize("model_arch", ModelRegistry.get_supported_archs())
20
def test_registry_imports(model_arch):
21
# Ensure all model classes can be imported successfully
15
- ModelRegistry.resolve_model_cls(model_arch)
22
+ model_cls, _ = ModelRegistry.resolve_model_cls(model_arch)
23
+
24
+ if model_arch in _SPECULATIVE_DECODING_MODELS:
25
+ pass # Ignore these models which do not have a unified format
26
+ else:
27
+ assert is_text_generation_model(model_cls) is (
28
+ model_arch in _TEXT_GENERATION_MODELS
29
+ or model_arch in _MULTIMODAL_MODELS)
30
+
31
+ assert is_embedding_model(model_cls) is (model_arch
32
+ in _EMBEDDING_MODELS)
33
+
34
+ assert supports_multimodal(model_cls) is (model_arch
35
+ in _MULTIMODAL_MODELS)
36
37
38
@fork_new_process_for_each_test
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/__init__.py
+6
@@ -9,6 +9,12 @@ def register():
9
ModelRegistry.register_model("MyOPTForCausalLM", MyOPTForCausalLM)
10
11
# Test passing lazy model
12
+ if "MyGemma2Embedding" not in ModelRegistry.get_supported_archs():
13
+ ModelRegistry.register_model(
14
+ "MyGemma2Embedding",
15
+ "vllm_add_dummy_model.my_gemma_embedding:MyGemma2Embedding",
16
+ )
17
+
18
if "MyLlava" not in ModelRegistry.get_supported_archs():
19
ModelRegistry.register_model("MyLlava",
20
"vllm_add_dummy_model.my_llava:MyLlava")
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_gemma_embedding.py
new
+34
@@ -0,0 +1,34 @@
1
+from typing import List, Optional, Union
2
+
3
+import torch
4
+
5
+from vllm.attention import AttentionMetadata
6
+from vllm.model_executor.models.gemma2_embedding import Gemma2EmbeddingModel
7
+from vllm.sequence import IntermediateTensors
8
+
9
+
10
+class MyGemma2Embedding(Gemma2EmbeddingModel):
11
+
12
+ def forward(
13
+ self,
14
+ input_ids: torch.Tensor,
15
+ positions: torch.Tensor,
16
+ kv_caches: List[torch.Tensor],
17
+ attn_metadata: AttentionMetadata,
18
+ intermediate_tensors: Optional[IntermediateTensors] = None,
19
+ inputs_embeds: Optional[torch.Tensor] = None,
20
+ ) -> Union[torch.Tensor, IntermediateTensors]:
21
+ hidden_states = super().forward(
22
+ input_ids,
23
+ positions,
24
+ kv_caches,
25
+ attn_metadata,
26
+ intermediate_tensors=intermediate_tensors,
27
+ inputs_embeds=inputs_embeds,
28
+ )
29
+
30
+ if isinstance(hidden_states, IntermediateTensors):
31
+ return hidden_states
32
+
33
+ # Return all-zero embeddings
34
+ return torch.zeros_like(hidden_states)
vllm/model_executor/models/__init__.py
+7
@@ -1,10 +1,17 @@
1
from .interfaces import (HasInnerState, SupportsLoRA, SupportsMultiModal,
2
SupportsPP, has_inner_state, supports_lora,
3
supports_multimodal, supports_pp)
4
+from .interfaces_base import (VllmModelForEmbedding,
5
+ VllmModelForTextGeneration, is_embedding_model,
6
+ is_text_generation_model)
7
from .registry import ModelRegistry
8
9
__all__ = [
10
"ModelRegistry",
11
+ "VllmModelForEmbedding",
12
+ "is_embedding_model",
13
+ "VllmModelForTextGeneration",
14
+ "is_text_generation_model",
15
"HasInnerState",
16
"has_inner_state",
17
"SupportsLoRA",
vllm/model_executor/models/interfaces.py
+7
-21
@@ -1,4 +1,3 @@
1
-import inspect
1
from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,
2
Protocol, Type, Union, overload, runtime_checkable)
3
@@ -6,9 +5,9 @@ import torch
5
from typing_extensions import TypeIs
6
7
from vllm.logger import init_logger
8
+from vllm.utils import supports_kw
9
10
if TYPE_CHECKING:
11
- from vllm.attention import AttentionMetadata
11
from vllm.config import LoRAConfig, MultiModalConfig, SchedulerConfig
12
from vllm.sequence import IntermediateTensors
13
@@ -142,9 +141,7 @@ def supports_lora(
141
return result
142
143
145
-def _supports_lora(
146
- model: Union[Type[object], object],
147
-) -> Union[TypeIs[Type[SupportsLoRA]], TypeIs[SupportsLoRA]]:
144
+def _supports_lora(model: Union[Type[object], object]) -> bool:
145
if isinstance(model, type):
146
return isinstance(model, _SupportsLoRAType)
147
@@ -175,10 +172,7 @@ class SupportsPP(Protocol):
172
173
def forward(
174
self,
178
- input_ids: torch.Tensor,
179
- position_ids: torch.Tensor,
180
- kv_caches: List[torch.Tensor],
181
- attn_metadata: "AttentionMetadata",
175
+ *,
176
intermediate_tensors: Optional["IntermediateTensors"],
177
) -> Union[torch.Tensor, "IntermediateTensors"]:
178
"""
@@ -205,10 +199,7 @@ class _SupportsPPType(Protocol):
199
200
def forward(
201
self,
208
- input_ids: torch.Tensor,
209
- position_ids: torch.Tensor,
210
- kv_caches: List[torch.Tensor],
211
- attn_metadata: "AttentionMetadata",
202
+ *,
203
intermediate_tensors: Optional["IntermediateTensors"],
204
) -> Union[torch.Tensor, "IntermediateTensors"]:
205
...
@@ -257,24 +248,19 @@ def supports_pp(
248
return supports_attributes and supports_inspect
249
250
260
-def _supports_pp_attributes(
261
- model: Union[Type[object], object],
262
-) -> Union[bool, TypeIs[Type[SupportsPP]], TypeIs[SupportsPP]]:
251
+def _supports_pp_attributes(model: Union[Type[object], object]) -> bool:
252
if isinstance(model, type):
253
return isinstance(model, _SupportsPPType)
254
255
return isinstance(model, SupportsPP)
256
257
269
-def _supports_pp_inspect(
270
- model: Union[Type[object], object],
271
-) -> Union[bool, TypeIs[Type[SupportsPP]], TypeIs[SupportsPP]]:
258
+def _supports_pp_inspect(model: Union[Type[object], object]) -> bool:
259
model_forward = getattr(model, "forward", None)
260
if not callable(model_forward):
261
return False
262
276
- forward_params = inspect.signature(model_forward).parameters
277
- return "intermediate_tensors" in forward_params
263
+ return supports_kw(model_forward, "intermediate_tensors")
264
265
266
@runtime_checkable
vllm/model_executor/models/interfaces_base.py
new
+191
@@ -0,0 +1,191 @@
1
+from typing import (TYPE_CHECKING, List, Optional, Protocol, Type, Union,
2
+ overload, runtime_checkable)
3
+
4
+import torch
5
+import torch.nn as nn
6
+from transformers import PretrainedConfig
7
+from typing_extensions import TypeIs, TypeVar
8
+
9
+from vllm.logger import init_logger
10
+from vllm.utils import supports_kw
11
+
12
+if TYPE_CHECKING:
13
+ from vllm.attention import AttentionMetadata
14
+ from vllm.config import CacheConfig
15
+ from vllm.model_executor.layers.pooler import PoolerOutput
16
+ from vllm.model_executor.layers.quantization import QuantizationConfig
17
+ from vllm.model_executor.layers.sampler import SamplerOutput
18
+ from vllm.model_executor.pooling_metadata import PoolingMetadata
19
+ from vllm.model_executor.sampling_metadata import SamplingMetadata
20
+
21
+logger = init_logger(__name__)
22
+
23
+# The type of HF config
24
+C_co = TypeVar("C_co", bound=PretrainedConfig, covariant=True)
25
+
26
+# The type of hidden states
27
+# Currently, T = torch.Tensor for all models except for Medusa
28
+# which has T = List[torch.Tensor]
29
+T = TypeVar("T", default=torch.Tensor)
30
+T_co = TypeVar("T_co", default=torch.Tensor, covariant=True)
31
+
32
+# NOTE: Unlike those in `interfaces.py`, we don't define `ClassVar` tags
33
+# for the base interfaces to avoid breaking OOT registration for existing models
34
+# that don't inherit from the base interface classes
35
+
36
+
37
+@runtime_checkable
38
+class VllmModel(Protocol[C_co, T_co]):
39
+
40
+ def __init__(
41
+ self,
42
+ config: C_co,
43
+ *,
44
+ cache_config: Optional["CacheConfig"],
45
+ quant_config: Optional["QuantizationConfig"],
46
+ ) -> None:
47
+ ...
48
+
49
+ def forward(
50
+ self,
51
+ input_ids: torch.Tensor,
52
+ positions: torch.Tensor,
53
+ kv_caches: List[torch.Tensor],
54
+ attn_metadata: "AttentionMetadata",
55
+ ) -> T_co:
56
+ ...
57
+
58
+
59
+def _check_vllm_model_init(model: Union[Type[object], object]) -> bool:
60
+ model_init = model.__init__
61
+ vllm_kws = ("cache_config", "quant_config")
62
+ missing_kws = tuple(kw for kw in vllm_kws
63
+ if not supports_kw(model_init, kw))
64
+
65
+ if missing_kws and (isinstance(model, type)
66
+ and issubclass(model, nn.Module)):
67
+ logger.warning(
68
+ "The model (%s) is missing "
69
+ "vLLM-specific keywords from its initializer: %s",
70
+ model,
71
+ missing_kws,
72
+ )
73
+
74
+ return len(missing_kws) == 0
75
+
76
+
77
+def _check_vllm_model_forward(model: Union[Type[object], object]) -> bool:
78
+ model_forward = getattr(model, "forward", None)
79
+ if not callable(model_forward):
80
+ return False
81
+
82
+ vllm_kws = ("input_ids", "positions", "kv_caches", "attn_metadata")
83
+ missing_kws = tuple(kw for kw in vllm_kws
84
+ if not supports_kw(model_forward, kw))
85
+
86
+ if missing_kws and (isinstance(model, type)
87
+ and issubclass(model, nn.Module)):
88
+ logger.warning(
89
+ "The model (%s) is missing "
90
+ "vLLM-specific keywords from its initializer: %s",
91
+ model,
92
+ missing_kws,
93
+ )
94
+
95
+ return len(missing_kws) == 0
96
+
97
+
98
+@overload
99
+def is_vllm_model(model: Type[object]) -> TypeIs[Type[VllmModel]]:
100
+ ...
101
+
102
+
103
+@overload
104
+def is_vllm_model(model: object) -> TypeIs[VllmModel]:
105
+ ...
106
+
107
+
108
+def is_vllm_model(
109
+ model: Union[Type[object], object],
110
+) -> Union[TypeIs[Type[VllmModel]], TypeIs[VllmModel]]:
111
+ return _check_vllm_model_init(model) and _check_vllm_model_forward(model)
112
+
113
+
114
+@runtime_checkable
115
+class VllmModelForTextGeneration(VllmModel[C_co, T], Protocol[C_co, T]):
116
+
117
+ def compute_logits(
118
+ self,
119
+ hidden_states: T,
120
+ sampling_metadata: "SamplingMetadata",
121
+ ) -> Optional[T]:
122
+ """Return `None` if TP rank > 0."""
123
+ ...
124
+
125
+ def sample(
126
+ self,
127
+ logits: T,
128
+ sampling_metadata: "SamplingMetadata",
129
+ ) -> "SamplerOutput":
130
+ """Only called on TP rank 0."""
131
+ ...
132
+
133
+
134
+@overload
135
+def is_text_generation_model(
136
+ model: Type[object]) -> TypeIs[Type[VllmModelForTextGeneration]]:
137
+ ...
138
+
139
+
140
+@overload
141
+def is_text_generation_model(
142
+ model: object) -> TypeIs[VllmModelForTextGeneration]:
143
+ ...
144
+
145
+
146
+def is_text_generation_model(
147
+ model: Union[Type[object], object],
148
+) -> Union[TypeIs[Type[VllmModelForTextGeneration]],
149
+ TypeIs[VllmModelForTextGeneration]]:
150
+ if not is_vllm_model(model):
151
+ return False
152
+
153
+ if isinstance(model, type):
154
+ return isinstance(model, VllmModelForTextGeneration)
155
+
156
+ return isinstance(model, VllmModelForTextGeneration)
157
+
158
+
159
+@runtime_checkable
160
+class VllmModelForEmbedding(VllmModel[C_co, T], Protocol[C_co, T]):
161
+
162
+ def pooler(
163
+ self,
164
+ hidden_states: T,
165
+ pooling_metadata: "PoolingMetadata",
166
+ ) -> "PoolerOutput":
167
+ """Only called on TP rank 0."""
168
+ ...
169
+
170
+
171
+@overload
172
+def is_embedding_model(
173
+ model: Type[object]) -> TypeIs[Type[VllmModelForEmbedding]]:
174
+ ...
175
+
176
+
177
+@overload
178
+def is_embedding_model(model: object) -> TypeIs[VllmModelForEmbedding]:
179
+ ...
180
+
181
+
182
+def is_embedding_model(
183
+ model: Union[Type[object], object],
184
+) -> Union[TypeIs[Type[VllmModelForEmbedding]], TypeIs[VllmModelForEmbedding]]:
185
+ if not is_vllm_model(model):
186
+ return False
187
+
188
+ if isinstance(model, type):
189
+ return isinstance(model, VllmModelForEmbedding)
190
+
191
+ return isinstance(model, VllmModelForEmbedding)
vllm/model_executor/models/registry.py
+31
-11
@@ -12,10 +12,12 @@ from vllm.logger import init_logger
12
from vllm.utils import is_hip
13
14
from .interfaces import supports_multimodal, supports_pp
15
+from .interfaces_base import is_embedding_model, is_text_generation_model
16
17
logger = init_logger(__name__)
18
18
-_GENERATION_MODELS = {
19
+_TEXT_GENERATION_MODELS = {
20
+ # [Decoder-only]
21
"AquilaModel": ("llama", "LlamaForCausalLM"),
22
"AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
23
"ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
@@ -74,10 +76,9 @@ _GENERATION_MODELS = {
76
"Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
77
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
78
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
77
- # NOTE: The below models are for speculative decoding only
78
- "MedusaModel": ("medusa", "Medusa"),
79
- "EAGLEModel": ("eagle", "EAGLE"),
80
- "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
79
+ # [Encoder-decoder]
80
+ "BartModel": ("bart", "BartForConditionalGeneration"),
81
+ "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
82
}
83
84
_EMBEDDING_MODELS = {
@@ -114,16 +115,18 @@ _MULTIMODAL_MODELS = {
115
"MllamaForConditionalGeneration": ("mllama",
116
"MllamaForConditionalGeneration"),
117
}
117
-_CONDITIONAL_GENERATION_MODELS = {
118
- "BartModel": ("bart", "BartForConditionalGeneration"),
119
- "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
118
+
119
+_SPECULATIVE_DECODING_MODELS = {
120
+ "EAGLEModel": ("eagle", "EAGLE"),
121
+ "MedusaModel": ("medusa", "Medusa"),
122
+ "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
123
}
124
125
_MODELS = {
123
- **_GENERATION_MODELS,
126
+ **_TEXT_GENERATION_MODELS,
127
**_EMBEDDING_MODELS,
128
**_MULTIMODAL_MODELS,
126
- **_CONDITIONAL_GENERATION_MODELS,
129
+ **_SPECULATIVE_DECODING_MODELS,
130
}
131
132
# Architecture -> type or (module, class).
@@ -317,6 +320,19 @@ class ModelRegistry:
320
321
return result.returncode == 0
322
323
+ @staticmethod
324
+ def is_text_generation_model(architectures: Union[str, List[str]]) -> bool:
325
+ if isinstance(architectures, str):
326
+ architectures = [architectures]
327
+ if not architectures:
328
+ logger.warning("No model architectures are specified")
329
+
330
+ is_txt_gen = partial(ModelRegistry._check_stateless,
331
+ is_text_generation_model,
332
+ default=False)
333
+
334
+ return any(is_txt_gen(arch) for arch in architectures)
335
+
336
@staticmethod
337
def is_embedding_model(architectures: Union[str, List[str]]) -> bool:
338
if isinstance(architectures, str):
@@ -324,7 +340,11 @@ class ModelRegistry:
340
if not architectures:
341
logger.warning("No model architectures are specified")
342
327
- return any(arch in _EMBEDDING_MODELS for arch in architectures)
343
+ is_emb = partial(ModelRegistry._check_stateless,
344
+ is_embedding_model,
345
+ default=False)
346
+
347
+ return any(is_emb(arch) for arch in architectures)
348
349
@staticmethod
350
def is_multimodal_model(architectures: Union[str, List[str]]) -> bool:
vllm/utils.py
+9
@@ -1277,6 +1277,15 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1277
return await task(*args, **kwargs)
1278
1279
1280
+def supports_kw(callable: Callable[..., object], kw_name: str) -> bool:
1281
+ params = inspect.signature(callable).parameters
1282
+ if kw_name in params:
1283
+ return True
1284
+
1285
+ return any(param.kind == inspect.Parameter.VAR_KEYWORD
1286
+ for param in params.values())
1287
+
1288
+
1289
def get_allowed_kwarg_only_overrides(
1290
callable: Callable[..., object],
1291
overrides: Optional[Dict[str, Any]],