@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c6de96ea1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Explicit interface for vLLM models and support OOT embedding models (#9108)

Cyrus Leung committed Oct 7, 2024 at 14:10 UTC 8c6de96ea1e6e51e49a170c28ad3efc16db9413e
10 files changed +342 -37
tests/conftest.py
+20
@@ -871,6 +871,7 @@ def num_gpus_available():
871 temp_dir = tempfile.gettempdir()
872 _dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
873 _dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
874 +_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")
875
876
877 @pytest.fixture
@@ -909,3 +910,22 @@ def dummy_llava_path():
910 with open(json_path, "w") as f:
911 json.dump(config, f)
912 return _dummy_llava_path
913 +
914 +
915 +@pytest.fixture
916 +def dummy_gemma2_embedding_path():
917 + json_path = os.path.join(_dummy_gemma2_embedding_path, "config.json")
918 + if not os.path.exists(_dummy_gemma2_embedding_path):
919 + snapshot_download(repo_id="BAAI/bge-multilingual-gemma2",
920 + local_dir=_dummy_gemma2_embedding_path,
921 + ignore_patterns=[
922 + "*.bin", "*.bin.index.json", "*.pt", "*.h5",
923 + "*.msgpack"
924 + ])
925 + assert os.path.exists(json_path)
926 + with open(json_path, "r") as f:
927 + config = json.load(f)
928 + config["architectures"] = ["MyGemma2Embedding"]
929 + with open(json_path, "w") as f:
930 + json.dump(config, f)
931 + return _dummy_gemma2_embedding_path
tests/models/test_oot_registration.py
+15 -3
@@ -2,7 +2,7 @@ import os
2
3 import pytest
4
5 -from vllm import LLM, SamplingParams
5 +from vllm import LLM, PoolingParams, SamplingParams
6 from vllm.assets.image import ImageAsset
7
8 from ..utils import fork_new_process_for_each_test
@@ -17,7 +17,7 @@ def test_plugin(dummy_opt_path):
17
18
19 @fork_new_process_for_each_test
20 -def test_oot_registration(dummy_opt_path):
20 +def test_oot_registration_text_generation(dummy_opt_path):
21 os.environ["VLLM_PLUGINS"] = "register_dummy_model"
22 prompts = ["Hello, my name is", "The text does not matter"]
23 sampling_params = SamplingParams(temperature=0)
@@ -32,11 +32,23 @@ def test_oot_registration(dummy_opt_path):
32 assert rest == ""
33
34
35 +@fork_new_process_for_each_test
36 +def test_oot_registration_embedding(dummy_gemma2_embedding_path):
37 + os.environ["VLLM_PLUGINS"] = "register_dummy_model"
38 + prompts = ["Hello, my name is", "The text does not matter"]
39 + sampling_params = PoolingParams()
40 + llm = LLM(model=dummy_gemma2_embedding_path, load_format="dummy")
41 + outputs = llm.encode(prompts, sampling_params)
42 +
43 + for output in outputs:
44 + assert all(v == 0 for v in output.outputs.embedding)
45 +
46 +
47 image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
48
49
50 @fork_new_process_for_each_test
39 -def test_oot_multimodal_registration(dummy_llava_path):
51 +def test_oot_registration_multimodal(dummy_llava_path):
52 os.environ["VLLM_PLUGINS"] = "register_dummy_model"
53 prompts = [{
54 "prompt": "What's in the image?<image>",
tests/models/test_registry.py
+22 -2
@@ -3,7 +3,14 @@ import warnings
3 import pytest
4 import torch.cuda
5
6 -from vllm.model_executor.models import ModelRegistry
6 +from vllm.model_executor.models import (is_embedding_model,
7 + is_text_generation_model,
8 + supports_multimodal)
9 +from vllm.model_executor.models.registry import (_EMBEDDING_MODELS,
10 + _MULTIMODAL_MODELS,
11 + _SPECULATIVE_DECODING_MODELS,
12 + _TEXT_GENERATION_MODELS,
13 + ModelRegistry)
14 from vllm.platforms import current_platform
15
16 from ..utils import fork_new_process_for_each_test
@@ -12,7 +19,20 @@ from ..utils import fork_new_process_for_each_test
19 @pytest.mark.parametrize("model_arch", ModelRegistry.get_supported_archs())
20 def test_registry_imports(model_arch):
21 # Ensure all model classes can be imported successfully
15 - ModelRegistry.resolve_model_cls(model_arch)
22 + model_cls, _ = ModelRegistry.resolve_model_cls(model_arch)
23 +
24 + if model_arch in _SPECULATIVE_DECODING_MODELS:
25 + pass # Ignore these models which do not have a unified format
26 + else:
27 + assert is_text_generation_model(model_cls) is (
28 + model_arch in _TEXT_GENERATION_MODELS
29 + or model_arch in _MULTIMODAL_MODELS)
30 +
31 + assert is_embedding_model(model_cls) is (model_arch
32 + in _EMBEDDING_MODELS)
33 +
34 + assert supports_multimodal(model_cls) is (model_arch
35 + in _MULTIMODAL_MODELS)
36
37
38 @fork_new_process_for_each_test
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/__init__.py
+6
@@ -9,6 +9,12 @@ def register():
9 ModelRegistry.register_model("MyOPTForCausalLM", MyOPTForCausalLM)
10
11 # Test passing lazy model
12 + if "MyGemma2Embedding" not in ModelRegistry.get_supported_archs():
13 + ModelRegistry.register_model(
14 + "MyGemma2Embedding",
15 + "vllm_add_dummy_model.my_gemma_embedding:MyGemma2Embedding",
16 + )
17 +
18 if "MyLlava" not in ModelRegistry.get_supported_archs():
19 ModelRegistry.register_model("MyLlava",
20 "vllm_add_dummy_model.my_llava:MyLlava")
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_gemma_embedding.py new
+34
@@ -0,0 +1,34 @@
1 +from typing import List, Optional, Union
2 +
3 +import torch
4 +
5 +from vllm.attention import AttentionMetadata
6 +from vllm.model_executor.models.gemma2_embedding import Gemma2EmbeddingModel
7 +from vllm.sequence import IntermediateTensors
8 +
9 +
10 +class MyGemma2Embedding(Gemma2EmbeddingModel):
11 +
12 + def forward(
13 + self,
14 + input_ids: torch.Tensor,
15 + positions: torch.Tensor,
16 + kv_caches: List[torch.Tensor],
17 + attn_metadata: AttentionMetadata,
18 + intermediate_tensors: Optional[IntermediateTensors] = None,
19 + inputs_embeds: Optional[torch.Tensor] = None,
20 + ) -> Union[torch.Tensor, IntermediateTensors]:
21 + hidden_states = super().forward(
22 + input_ids,
23 + positions,
24 + kv_caches,
25 + attn_metadata,
26 + intermediate_tensors=intermediate_tensors,
27 + inputs_embeds=inputs_embeds,
28 + )
29 +
30 + if isinstance(hidden_states, IntermediateTensors):
31 + return hidden_states
32 +
33 + # Return all-zero embeddings
34 + return torch.zeros_like(hidden_states)
vllm/model_executor/models/__init__.py
+7
@@ -1,10 +1,17 @@
1 from .interfaces import (HasInnerState, SupportsLoRA, SupportsMultiModal,
2 SupportsPP, has_inner_state, supports_lora,
3 supports_multimodal, supports_pp)
4 +from .interfaces_base import (VllmModelForEmbedding,
5 + VllmModelForTextGeneration, is_embedding_model,
6 + is_text_generation_model)
7 from .registry import ModelRegistry
8
9 __all__ = [
10 "ModelRegistry",
11 + "VllmModelForEmbedding",
12 + "is_embedding_model",
13 + "VllmModelForTextGeneration",
14 + "is_text_generation_model",
15 "HasInnerState",
16 "has_inner_state",
17 "SupportsLoRA",
vllm/model_executor/models/interfaces.py
+7 -21
@@ -1,4 +1,3 @@
1 -import inspect
1 from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,
2 Protocol, Type, Union, overload, runtime_checkable)
3
@@ -6,9 +5,9 @@ import torch
5 from typing_extensions import TypeIs
6
7 from vllm.logger import init_logger
8 +from vllm.utils import supports_kw
9
10 if TYPE_CHECKING:
11 - from vllm.attention import AttentionMetadata
11 from vllm.config import LoRAConfig, MultiModalConfig, SchedulerConfig
12 from vllm.sequence import IntermediateTensors
13
@@ -142,9 +141,7 @@ def supports_lora(
141 return result
142
143
145 -def _supports_lora(
146 - model: Union[Type[object], object],
147 -) -> Union[TypeIs[Type[SupportsLoRA]], TypeIs[SupportsLoRA]]:
144 +def _supports_lora(model: Union[Type[object], object]) -> bool:
145 if isinstance(model, type):
146 return isinstance(model, _SupportsLoRAType)
147
@@ -175,10 +172,7 @@ class SupportsPP(Protocol):
172
173 def forward(
174 self,
178 - input_ids: torch.Tensor,
179 - position_ids: torch.Tensor,
180 - kv_caches: List[torch.Tensor],
181 - attn_metadata: "AttentionMetadata",
175 + *,
176 intermediate_tensors: Optional["IntermediateTensors"],
177 ) -> Union[torch.Tensor, "IntermediateTensors"]:
178 """
@@ -205,10 +199,7 @@ class _SupportsPPType(Protocol):
199
200 def forward(
201 self,
208 - input_ids: torch.Tensor,
209 - position_ids: torch.Tensor,
210 - kv_caches: List[torch.Tensor],
211 - attn_metadata: "AttentionMetadata",
202 + *,
203 intermediate_tensors: Optional["IntermediateTensors"],
204 ) -> Union[torch.Tensor, "IntermediateTensors"]:
205 ...
@@ -257,24 +248,19 @@ def supports_pp(
248 return supports_attributes and supports_inspect
249
250
260 -def _supports_pp_attributes(
261 - model: Union[Type[object], object],
262 -) -> Union[bool, TypeIs[Type[SupportsPP]], TypeIs[SupportsPP]]:
251 +def _supports_pp_attributes(model: Union[Type[object], object]) -> bool:
252 if isinstance(model, type):
253 return isinstance(model, _SupportsPPType)
254
255 return isinstance(model, SupportsPP)
256
257
269 -def _supports_pp_inspect(
270 - model: Union[Type[object], object],
271 -) -> Union[bool, TypeIs[Type[SupportsPP]], TypeIs[SupportsPP]]:
258 +def _supports_pp_inspect(model: Union[Type[object], object]) -> bool:
259 model_forward = getattr(model, "forward", None)
260 if not callable(model_forward):
261 return False
262
276 - forward_params = inspect.signature(model_forward).parameters
277 - return "intermediate_tensors" in forward_params
263 + return supports_kw(model_forward, "intermediate_tensors")
264
265
266 @runtime_checkable
vllm/model_executor/models/interfaces_base.py new
+191
@@ -0,0 +1,191 @@
1 +from typing import (TYPE_CHECKING, List, Optional, Protocol, Type, Union,
2 + overload, runtime_checkable)
3 +
4 +import torch
5 +import torch.nn as nn
6 +from transformers import PretrainedConfig
7 +from typing_extensions import TypeIs, TypeVar
8 +
9 +from vllm.logger import init_logger
10 +from vllm.utils import supports_kw
11 +
12 +if TYPE_CHECKING:
13 + from vllm.attention import AttentionMetadata
14 + from vllm.config import CacheConfig
15 + from vllm.model_executor.layers.pooler import PoolerOutput
16 + from vllm.model_executor.layers.quantization import QuantizationConfig
17 + from vllm.model_executor.layers.sampler import SamplerOutput
18 + from vllm.model_executor.pooling_metadata import PoolingMetadata
19 + from vllm.model_executor.sampling_metadata import SamplingMetadata
20 +
21 +logger = init_logger(__name__)
22 +
23 +# The type of HF config
24 +C_co = TypeVar("C_co", bound=PretrainedConfig, covariant=True)
25 +
26 +# The type of hidden states
27 +# Currently, T = torch.Tensor for all models except for Medusa
28 +# which has T = List[torch.Tensor]
29 +T = TypeVar("T", default=torch.Tensor)
30 +T_co = TypeVar("T_co", default=torch.Tensor, covariant=True)
31 +
32 +# NOTE: Unlike those in `interfaces.py`, we don't define `ClassVar` tags
33 +# for the base interfaces to avoid breaking OOT registration for existing models
34 +# that don't inherit from the base interface classes
35 +
36 +
37 +@runtime_checkable
38 +class VllmModel(Protocol[C_co, T_co]):
39 +
40 + def __init__(
41 + self,
42 + config: C_co,
43 + *,
44 + cache_config: Optional["CacheConfig"],
45 + quant_config: Optional["QuantizationConfig"],
46 + ) -> None:
47 + ...
48 +
49 + def forward(
50 + self,
51 + input_ids: torch.Tensor,
52 + positions: torch.Tensor,
53 + kv_caches: List[torch.Tensor],
54 + attn_metadata: "AttentionMetadata",
55 + ) -> T_co:
56 + ...
57 +
58 +
59 +def _check_vllm_model_init(model: Union[Type[object], object]) -> bool:
60 + model_init = model.__init__
61 + vllm_kws = ("cache_config", "quant_config")
62 + missing_kws = tuple(kw for kw in vllm_kws
63 + if not supports_kw(model_init, kw))
64 +
65 + if missing_kws and (isinstance(model, type)
66 + and issubclass(model, nn.Module)):
67 + logger.warning(
68 + "The model (%s) is missing "
69 + "vLLM-specific keywords from its initializer: %s",
70 + model,
71 + missing_kws,
72 + )
73 +
74 + return len(missing_kws) == 0
75 +
76 +
77 +def _check_vllm_model_forward(model: Union[Type[object], object]) -> bool:
78 + model_forward = getattr(model, "forward", None)
79 + if not callable(model_forward):
80 + return False
81 +
82 + vllm_kws = ("input_ids", "positions", "kv_caches", "attn_metadata")
83 + missing_kws = tuple(kw for kw in vllm_kws
84 + if not supports_kw(model_forward, kw))
85 +
86 + if missing_kws and (isinstance(model, type)
87 + and issubclass(model, nn.Module)):
88 + logger.warning(
89 + "The model (%s) is missing "
90 + "vLLM-specific keywords from its initializer: %s",
91 + model,
92 + missing_kws,
93 + )
94 +
95 + return len(missing_kws) == 0
96 +
97 +
98 +@overload
99 +def is_vllm_model(model: Type[object]) -> TypeIs[Type[VllmModel]]:
100 + ...
101 +
102 +
103 +@overload
104 +def is_vllm_model(model: object) -> TypeIs[VllmModel]:
105 + ...
106 +
107 +
108 +def is_vllm_model(
109 + model: Union[Type[object], object],
110 +) -> Union[TypeIs[Type[VllmModel]], TypeIs[VllmModel]]:
111 + return _check_vllm_model_init(model) and _check_vllm_model_forward(model)
112 +
113 +
114 +@runtime_checkable
115 +class VllmModelForTextGeneration(VllmModel[C_co, T], Protocol[C_co, T]):
116 +
117 + def compute_logits(
118 + self,
119 + hidden_states: T,
120 + sampling_metadata: "SamplingMetadata",
121 + ) -> Optional[T]:
122 + """Return `None` if TP rank > 0."""
123 + ...
124 +
125 + def sample(
126 + self,
127 + logits: T,
128 + sampling_metadata: "SamplingMetadata",
129 + ) -> "SamplerOutput":
130 + """Only called on TP rank 0."""
131 + ...
132 +
133 +
134 +@overload
135 +def is_text_generation_model(
136 + model: Type[object]) -> TypeIs[Type[VllmModelForTextGeneration]]:
137 + ...
138 +
139 +
140 +@overload
141 +def is_text_generation_model(
142 + model: object) -> TypeIs[VllmModelForTextGeneration]:
143 + ...
144 +
145 +
146 +def is_text_generation_model(
147 + model: Union[Type[object], object],
148 +) -> Union[TypeIs[Type[VllmModelForTextGeneration]],
149 + TypeIs[VllmModelForTextGeneration]]:
150 + if not is_vllm_model(model):
151 + return False
152 +
153 + if isinstance(model, type):
154 + return isinstance(model, VllmModelForTextGeneration)
155 +
156 + return isinstance(model, VllmModelForTextGeneration)
157 +
158 +
159 +@runtime_checkable
160 +class VllmModelForEmbedding(VllmModel[C_co, T], Protocol[C_co, T]):
161 +
162 + def pooler(
163 + self,
164 + hidden_states: T,
165 + pooling_metadata: "PoolingMetadata",
166 + ) -> "PoolerOutput":
167 + """Only called on TP rank 0."""
168 + ...
169 +
170 +
171 +@overload
172 +def is_embedding_model(
173 + model: Type[object]) -> TypeIs[Type[VllmModelForEmbedding]]:
174 + ...
175 +
176 +
177 +@overload
178 +def is_embedding_model(model: object) -> TypeIs[VllmModelForEmbedding]:
179 + ...
180 +
181 +
182 +def is_embedding_model(
183 + model: Union[Type[object], object],
184 +) -> Union[TypeIs[Type[VllmModelForEmbedding]], TypeIs[VllmModelForEmbedding]]:
185 + if not is_vllm_model(model):
186 + return False
187 +
188 + if isinstance(model, type):
189 + return isinstance(model, VllmModelForEmbedding)
190 +
191 + return isinstance(model, VllmModelForEmbedding)
vllm/model_executor/models/registry.py
+31 -11
@@ -12,10 +12,12 @@ from vllm.logger import init_logger
12 from vllm.utils import is_hip
13
14 from .interfaces import supports_multimodal, supports_pp
15 +from .interfaces_base import is_embedding_model, is_text_generation_model
16
17 logger = init_logger(__name__)
18
18 -_GENERATION_MODELS = {
19 +_TEXT_GENERATION_MODELS = {
20 + # [Decoder-only]
21 "AquilaModel": ("llama", "LlamaForCausalLM"),
22 "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
23 "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
@@ -74,10 +76,9 @@ _GENERATION_MODELS = {
76 "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
77 "SolarForCausalLM": ("solar", "SolarForCausalLM"),
78 "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
77 - # NOTE: The below models are for speculative decoding only
78 - "MedusaModel": ("medusa", "Medusa"),
79 - "EAGLEModel": ("eagle", "EAGLE"),
80 - "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
79 + # [Encoder-decoder]
80 + "BartModel": ("bart", "BartForConditionalGeneration"),
81 + "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
82 }
83
84 _EMBEDDING_MODELS = {
@@ -114,16 +115,18 @@ _MULTIMODAL_MODELS = {
115 "MllamaForConditionalGeneration": ("mllama",
116 "MllamaForConditionalGeneration"),
117 }
117 -_CONDITIONAL_GENERATION_MODELS = {
118 - "BartModel": ("bart", "BartForConditionalGeneration"),
119 - "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
118 +
119 +_SPECULATIVE_DECODING_MODELS = {
120 + "EAGLEModel": ("eagle", "EAGLE"),
121 + "MedusaModel": ("medusa", "Medusa"),
122 + "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
123 }
124
125 _MODELS = {
123 - **_GENERATION_MODELS,
126 + **_TEXT_GENERATION_MODELS,
127 **_EMBEDDING_MODELS,
128 **_MULTIMODAL_MODELS,
126 - **_CONDITIONAL_GENERATION_MODELS,
129 + **_SPECULATIVE_DECODING_MODELS,
130 }
131
132 # Architecture -> type or (module, class).
@@ -317,6 +320,19 @@ class ModelRegistry:
320
321 return result.returncode == 0
322
323 + @staticmethod
324 + def is_text_generation_model(architectures: Union[str, List[str]]) -> bool:
325 + if isinstance(architectures, str):
326 + architectures = [architectures]
327 + if not architectures:
328 + logger.warning("No model architectures are specified")
329 +
330 + is_txt_gen = partial(ModelRegistry._check_stateless,
331 + is_text_generation_model,
332 + default=False)
333 +
334 + return any(is_txt_gen(arch) for arch in architectures)
335 +
336 @staticmethod
337 def is_embedding_model(architectures: Union[str, List[str]]) -> bool:
338 if isinstance(architectures, str):
@@ -324,7 +340,11 @@ class ModelRegistry:
340 if not architectures:
341 logger.warning("No model architectures are specified")
342
327 - return any(arch in _EMBEDDING_MODELS for arch in architectures)
343 + is_emb = partial(ModelRegistry._check_stateless,
344 + is_embedding_model,
345 + default=False)
346 +
347 + return any(is_emb(arch) for arch in architectures)
348
349 @staticmethod
350 def is_multimodal_model(architectures: Union[str, List[str]]) -> bool:
vllm/utils.py
+9
@@ -1277,6 +1277,15 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1277 return await task(*args, **kwargs)
1278
1279
1280 +def supports_kw(callable: Callable[..., object], kw_name: str) -> bool:
1281 + params = inspect.signature(callable).parameters
1282 + if kw_name in params:
1283 + return True
1284 +
1285 + return any(param.kind == inspect.Parameter.VAR_KEYWORD
1286 + for param in params.values())
1287 +
1288 +
1289 def get_allowed_kwarg_only_overrides(
1290 callable: Callable[..., object],
1291 overrides: Optional[Dict[str, Any]],