A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Model] Automatic conversion of classification and reward models (#11469)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 25, 2024 at 02:22 UTC
3f3e92e1f2e332547f7d4bac2358bc4e8e5d018b
9 files changed
+206
-161
docs/source/models/supported_models.md
+12
-10
@@ -28,7 +28,7 @@ llm = LLM(model=..., task="generate") # Name or path of your model
28
output = llm.generate("Hello, my name is")
29
print(output)
30
31
-# For pooling models (task={embed,classify,reward}) only
31
+# For pooling models (task={embed,classify,reward,score}) only
32
llm = LLM(model=..., task="embed") # Name or path of your model
33
output = llm.encode("Hello, my name is")
34
print(output)
@@ -59,7 +59,7 @@ llm = LLM(model=..., revision=..., task=..., trust_remote_code=True)
59
output = llm.generate("Hello, my name is")
60
print(output)
61
62
-# For pooling models (task={embed,classify,reward}) only
62
+# For pooling models (task={embed,classify,reward,score}) only
63
output = llm.encode("Hello, my name is")
64
print(output)
65
```
@@ -369,14 +369,6 @@ you should explicitly specify the task type to ensure that the model is used in
369
370
#### Text Embedding (`--task embed`)
371
372
-Any text generation model can be converted into an embedding model by passing {code}`--task embed`.
373
-
374
-```{note}
375
-To get the best results, you should use pooling models that are specifically trained as such.
376
-```
377
-
378
-The following table lists those that are tested in vLLM.
379
-
372
```{eval-rst}
373
.. list-table::
374
:widths: 25 25 50 5 5
@@ -437,6 +429,10 @@ On the other hand, its 1.5B variant ({code}`Alibaba-NLP/gte-Qwen2-1.5B-instruct`
429
despite being described otherwise on its model card.
430
```
431
432
+If your model is not in the above list, we will try to automatically convert the model using
433
+:func:`vllm.model_executor.models.adapters.as_embedding_model`. By default, the embeddings
434
+of the whole prompt are extracted from the normalized hidden state corresponding to the last token.
435
+
436
#### Reward Modeling (`--task reward`)
437
438
```{eval-rst}
@@ -461,6 +457,9 @@ despite being described otherwise on its model card.
457
- ✅︎
458
```
459
460
+If your model is not in the above list, we will try to automatically convert the model using
461
+:func:`vllm.model_executor.models.adapters.as_reward_model`. By default, we return the hidden states of each token directly.
462
+
463
```{important}
464
For process-supervised reward models such as {code}`peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
465
e.g.: {code}`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
@@ -490,6 +489,9 @@ e.g.: {code}`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 1
489
- ✅︎
490
```
491
492
+If your model is not in the above list, we will try to automatically convert the model using
493
+:func:`vllm.model_executor.models.adapters.as_classification_model`. By default, the class probabilities are extracted from the softmaxed hidden state corresponding to the last token.
494
+
495
#### Sentence Pair Scoring (`--task score`)
496
497
```{eval-rst}
tests/models/embedding/language/test_cls_models.py
+1
-4
@@ -1,7 +1,4 @@
1
-"""Compare the outputs of HF and vLLM when using greedy sampling.
2
-
3
-This test only tests small models. Big models such as 7B should be tested from
4
-test_big_models.py because it could use a larger instance to run tests.
1
+"""Compare the classification outputs of HF and vLLM models.
2
3
Run `pytest tests/models/test_cls_models.py`.
4
"""
tests/models/embedding/language/test_scoring.py
+2
-2
@@ -1,6 +1,6 @@
1
-"""Compare the embedding outputs of HF and vLLM models.
1
+"""Compare the scoring outputs of HF and vLLM models.
2
3
-Run `pytest tests/models/embedding/language/test_embedding.py`.
3
+Run `pytest tests/models/embedding/language/test_scoring.py`.
4
"""
5
import math
6
tests/models/test_registry.py
+7
-4
@@ -6,7 +6,9 @@ import torch.cuda
6
from vllm.model_executor.models import (is_pooling_model,
7
is_text_generation_model,
8
supports_multimodal)
9
-from vllm.model_executor.models.adapters import as_embedding_model
9
+from vllm.model_executor.models.adapters import (as_classification_model,
10
+ as_embedding_model,
11
+ as_reward_model)
12
from vllm.model_executor.models.registry import (_MULTIMODAL_MODELS,
13
_SPECULATIVE_DECODING_MODELS,
14
_TEXT_GENERATION_MODELS,
@@ -29,9 +31,10 @@ def test_registry_imports(model_arch):
31
or model_arch in _MULTIMODAL_MODELS):
32
assert is_text_generation_model(model_cls)
33
32
- # All vLLM models should be convertible to an embedding model
33
- embed_model = as_embedding_model(model_cls)
34
- assert is_pooling_model(embed_model)
34
+ # All vLLM models should be convertible to a pooling model
35
+ assert is_pooling_model(as_classification_model(model_cls))
36
+ assert is_pooling_model(as_embedding_model(model_cls))
37
+ assert is_pooling_model(as_reward_model(model_cls))
38
39
if model_arch in _MULTIMODAL_MODELS:
40
assert supports_multimodal(model_cls)
vllm/model_executor/model_loader/utils.py
+8
-2
@@ -7,7 +7,9 @@ from torch import nn
7
8
from vllm.config import ModelConfig
9
from vllm.model_executor.models import ModelRegistry
10
-from vllm.model_executor.models.adapters import as_embedding_model
10
+from vllm.model_executor.models.adapters import (as_classification_model,
11
+ as_embedding_model,
12
+ as_reward_model)
13
14
15
@contextlib.contextmanager
@@ -35,8 +37,12 @@ def get_model_architecture(
37
architectures = ["QuantMixtralForCausalLM"]
38
39
model_cls, arch = ModelRegistry.resolve_model_cls(architectures)
38
- if model_config.runner_type == "pooling":
40
+ if model_config.task == "embed":
41
model_cls = as_embedding_model(model_cls)
42
+ elif model_config.task == "classify":
43
+ model_cls = as_classification_model(model_cls)
44
+ elif model_config.task == "reward":
45
+ model_cls = as_reward_model(model_cls)
46
47
return model_cls, arch
48
vllm/model_executor/models/adapters.py
+170
-20
@@ -1,29 +1,48 @@
1
from collections.abc import Iterable
2
-from typing import Any, TypeVar
2
+from typing import TYPE_CHECKING, Any, Optional, TypeVar
3
4
import torch
5
import torch.nn as nn
6
7
from .interfaces_base import VllmModelForPooling, is_pooling_model
8
9
+if TYPE_CHECKING:
10
+ from vllm.model_executor.layers.pooler import PoolingType
11
+
12
_T = TypeVar("_T", bound=type[nn.Module])
13
14
+_GENERATE_SUFFIXES = [
15
+ "ForCausalLM",
16
+ "ForConditionalGeneration",
17
+ "ChatModel",
18
+ "LMHeadModel",
19
+]
20
12
-def as_embedding_model(cls: _T) -> _T:
13
- """Subclass an existing vLLM model to support embeddings."""
14
- # Avoid modifying existing embedding models
15
- if is_pooling_model(cls):
16
- return cls
21
22
+def _get_pooling_model_name(orig_model_name: str, pooling_suffix: str) -> str:
23
+ model_name = orig_model_name
24
+
25
+ for generate_suffix in _GENERATE_SUFFIXES:
26
+ model_name = model_name.removesuffix(generate_suffix)
27
+
28
+ return model_name + pooling_suffix
29
+
30
+
31
+def _create_pooling_model_cls(
32
+ orig_cls: _T,
33
+ *,
34
+ default_pooling_type: "PoolingType",
35
+ default_normalize: bool,
36
+ default_softmax: bool,
37
+) -> _T:
38
# Lazy import
39
from vllm.config import VllmConfig
20
- from vllm.model_executor.layers.pooler import (Pooler, PoolerOutput,
21
- PoolingType)
40
+ from vllm.model_executor.layers.pooler import Pooler, PoolerOutput
41
from vllm.model_executor.pooling_metadata import PoolingMetadata
42
43
from .utils import AutoWeightsLoader, WeightsMapper
44
26
- class ModelForEmbedding(cls, VllmModelForPooling):
45
+ class ModelForPooling(orig_cls, VllmModelForPooling):
46
47
def __init__(
48
self,
@@ -34,7 +53,7 @@ def as_embedding_model(cls: _T) -> _T:
53
) -> None:
54
super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
55
37
- # These are not used in embedding models
56
+ # These are not used in pooling models
57
for attr in ("lm_head", "logits_processor"):
58
if hasattr(self, attr):
59
delattr(self, attr)
@@ -46,9 +65,9 @@ def as_embedding_model(cls: _T) -> _T:
65
if not getattr(self, "_pooler", None):
66
self._pooler = Pooler.from_config_with_defaults(
67
pooler_config,
49
- pooling_type=PoolingType.LAST,
50
- normalize=True,
51
- softmax=False,
68
+ pooling_type=default_pooling_type,
69
+ normalize=default_normalize,
70
+ softmax=default_softmax,
71
)
72
73
def pooler(
@@ -82,17 +101,148 @@ def as_embedding_model(cls: _T) -> _T:
101
return
102
103
# For most other models
85
- if hasattr(cls, "load_weights"):
86
- cls.load_weights(self, weights) # type: ignore
104
+ if hasattr(orig_cls, "load_weights"):
105
+ orig_cls.load_weights(self, weights) # type: ignore
106
# Fallback
107
else:
108
loader = AutoWeightsLoader(self)
109
loader.load_weights(weights)
110
92
- ModelForEmbedding.__name__ = cls.__name__ \
93
- .removesuffix("ForCausalLM") \
94
- .removesuffix("ForConditionalGeneration") \
95
- .removesuffix("ChatModel") \
96
- .removesuffix("LMHeadModel") + "ForEmbedding"
111
+ return ModelForPooling # type: ignore
112
+
113
+
114
+def as_embedding_model(cls: _T) -> _T:
115
+ """
116
+ Subclass an existing vLLM model to support embeddings.
117
+
118
+ By default, the embeddings of the whole prompt are extracted from the
119
+ normalized hidden state corresponding to the last token.
120
+
121
+ Note:
122
+ We assume that no extra layers are added to the original model;
123
+ please implement your own model if this is not the case.
124
+ """
125
+ # Avoid modifying existing embedding models
126
+ if is_pooling_model(cls):
127
+ return cls
128
+
129
+ # Lazy import
130
+ from vllm.model_executor.layers.pooler import PoolingType
131
+
132
+ ModelForEmbedding = _create_pooling_model_cls(
133
+ cls,
134
+ default_pooling_type=PoolingType.LAST,
135
+ default_normalize=True,
136
+ default_softmax=False,
137
+ )
138
+ ModelForEmbedding.__name__ = \
139
+ _get_pooling_model_name(cls.__name__, "ForEmbedding")
140
141
return ModelForEmbedding # type: ignore
142
+
143
+
144
+def as_classification_model(cls: _T) -> _T:
145
+ """
146
+ Subclass an existing vLLM model to support classification.
147
+
148
+ By default, the class probabilities are extracted from the softmaxed
149
+ hidden state corresponding to the last token.
150
+
151
+ Note:
152
+ We assume that the classification head is a single linear layer
153
+ stored as the attribute `score` of the top-level model;
154
+ please implement your own model if this is not the case.
155
+ """
156
+ # Avoid modifying existing classification models
157
+ if is_pooling_model(cls):
158
+ return cls
159
+
160
+ # Lazy import
161
+ from vllm.attention import AttentionMetadata
162
+ from vllm.config import VllmConfig
163
+ from vllm.model_executor.layers.linear import RowParallelLinear
164
+ from vllm.model_executor.layers.pooler import PoolingType
165
+ from vllm.sequence import IntermediateTensors
166
+
167
+ from .utils import maybe_prefix
168
+
169
+ ModelForPooling = _create_pooling_model_cls(
170
+ cls,
171
+ default_pooling_type=PoolingType.LAST,
172
+ default_normalize=False,
173
+ default_softmax=True,
174
+ )
175
+
176
+ class ModelForClassification(ModelForPooling):
177
+
178
+ def __init__(
179
+ self,
180
+ *,
181
+ vllm_config: "VllmConfig",
182
+ prefix: str = "",
183
+ **kwargs: Any,
184
+ ) -> None:
185
+ super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
186
+
187
+ config = vllm_config.model_config.hf_config
188
+ quant_config = vllm_config.quant_config
189
+
190
+ self.score = RowParallelLinear(config.hidden_size,
191
+ config.num_labels,
192
+ quant_config=quant_config,
193
+ input_is_parallel=False,
194
+ bias=False,
195
+ prefix=maybe_prefix(
196
+ prefix, "score"))
197
+
198
+ def forward(
199
+ self,
200
+ input_ids: torch.Tensor,
201
+ positions: torch.Tensor,
202
+ kv_caches: list[torch.Tensor],
203
+ attn_metadata: AttentionMetadata,
204
+ intermediate_tensors: Optional[IntermediateTensors] = None,
205
+ inputs_embeds: Optional[torch.Tensor] = None,
206
+ ) -> torch.Tensor:
207
+ hidden_states = super().forward(input_ids, positions, kv_caches,
208
+ attn_metadata,
209
+ intermediate_tensors,
210
+ inputs_embeds)
211
+ logits, _ = self.score(hidden_states)
212
+ return logits
213
+
214
+
215
+ ModelForClassification.__name__ = \
216
+ _get_pooling_model_name(cls.__name__, "ForClassification")
217
+
218
+ return ModelForClassification # type: ignore
219
+
220
+
221
+def as_reward_model(cls: _T) -> _T:
222
+ """
223
+ Subclass an existing vLLM model to support reward modeling.
224
+
225
+ By default, we return the hidden states of each token directly.
226
+
227
+ Note:
228
+ We assume that no extra layers are added to the original model;
229
+ please implement your own model if this is not the case.
230
+ """
231
+ # Avoid modifying existing reward models
232
+ if is_pooling_model(cls):
233
+ return cls
234
+
235
+ # Lazy import
236
+ from vllm.model_executor.layers.pooler import PoolingType
237
+
238
+ ModelForReward = _create_pooling_model_cls(
239
+ cls,
240
+ default_pooling_type=PoolingType.ALL,
241
+ default_normalize=False,
242
+ default_softmax=False,
243
+ )
244
+
245
+ ModelForReward.__name__ = \
246
+ _get_pooling_model_name(cls.__name__, "ForReward")
247
+
248
+ return ModelForReward # type: ignore
vllm/model_executor/models/qwen2.py
+2
-2
@@ -545,8 +545,8 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
545
self.model = Qwen2Model(vllm_config=vllm_config,
546
prefix=maybe_prefix(prefix, "model"))
547
548
- # TODO: Replace this model class with for_embedding(Qwen2ForCausalLM),
549
- # after changing the default pooling method
548
+ # TODO: Replace this model class with as_embedding_model(
549
+ # Qwen2ForCausalLM) after changing the default pooling method
550
if pooler_config.pooling_type is None:
551
logger.warning(
552
"This embedding model will default to last-token pooling in "
vllm/model_executor/models/qwen2_cls.py
deleted
-104
@@ -1,104 +0,0 @@
1
-# Adapted from
2
-# https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B/blob/main/modeling_qwen2_rm.py
3
-# Copyright 2024 Kakao Corp. (Kanana-X Team)
4
-# Copyright 2024 The Qwen team.
5
-# Copyright 2023 The vLLM team.
6
-"""Inference-only Qwen2-Classification model compatible with HF weights."""
7
-from typing import Iterable, List, Optional, Set, Tuple
8
-
9
-import torch
10
-from torch import nn
11
-
12
-from vllm.attention import AttentionMetadata
13
-from vllm.config import VllmConfig
14
-from vllm.model_executor.layers.linear import RowParallelLinear
15
-from vllm.model_executor.layers.pooler import Pooler, PoolingType
16
-from vllm.model_executor.models.qwen2 import Qwen2Model
17
-from vllm.model_executor.pooling_metadata import PoolingMetadata
18
-from vllm.sequence import IntermediateTensors, PoolerOutput
19
-
20
-from .interfaces import SupportsLoRA, SupportsPP
21
-from .utils import AutoWeightsLoader, maybe_prefix
22
-
23
-
24
-class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
25
- packed_modules_mapping = {
26
- "qkv_proj": [
27
- "q_proj",
28
- "k_proj",
29
- "v_proj",
30
- ],
31
- "gate_up_proj": [
32
- "gate_proj",
33
- "up_proj",
34
- ],
35
- }
36
-
37
- # LoRA specific attributes
38
- supported_lora_modules = [
39
- "qkv_proj",
40
- "o_proj",
41
- "gate_up_proj",
42
- "down_proj",
43
- ]
44
- embedding_modules = {}
45
- embedding_padding_modules = []
46
-
47
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
48
- super().__init__()
49
- config = vllm_config.model_config.hf_config
50
- quant_config = vllm_config.quant_config
51
- lora_config = vllm_config.lora_config
52
- pooler_config = vllm_config.model_config.pooler_config
53
-
54
- self.config = config
55
- self.lora_config = lora_config
56
-
57
- self.quant_config = quant_config
58
- self.model = Qwen2Model(vllm_config=vllm_config,
59
- prefix=maybe_prefix(prefix, "model"))
60
-
61
- # hidden_states from Qwen2Model has been reduced,
62
- # the input of score layer is not parallelized.
63
- self.score = RowParallelLinear(config.hidden_size,
64
- config.num_labels,
65
- quant_config=quant_config,
66
- input_is_parallel=False,
67
- bias=False,
68
- prefix=maybe_prefix(prefix, "score"))
69
- self._pooler = Pooler.from_config_with_defaults(
70
- pooler_config,
71
- pooling_type=PoolingType.LAST,
72
- normalize=False,
73
- softmax=True)
74
-
75
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
76
- return self.model.get_input_embeddings(input_ids)
77
-
78
- def forward(
79
- self,
80
- input_ids: torch.Tensor,
81
- positions: torch.Tensor,
82
- kv_caches: List[torch.Tensor],
83
- attn_metadata: AttentionMetadata,
84
- intermediate_tensors: Optional[IntermediateTensors] = None,
85
- inputs_embeds: Optional[torch.Tensor] = None,
86
- ) -> torch.Tensor:
87
- hidden_states = self.model(input_ids, positions, kv_caches,
88
- attn_metadata, intermediate_tensors,
89
- inputs_embeds)
90
- logits, _ = self.score(hidden_states)
91
- return logits
92
-
93
- def pooler(
94
- self,
95
- hidden_states: torch.Tensor,
96
- pooling_metadata: PoolingMetadata,
97
- ) -> Optional[PoolerOutput]:
98
- return self._pooler(hidden_states, pooling_metadata)
99
-
100
- def load_weights(self, weights: Iterable[Tuple[str,
101
- torch.Tensor]]) -> Set[str]:
102
- loader = AutoWeightsLoader(self,
103
- ignore_unexpected_prefixes=["lm_head."])
104
- return loader.load_weights(weights)
vllm/model_executor/models/registry.py
+4
-13
@@ -20,11 +20,10 @@ import torch.nn as nn
20
from vllm.logger import init_logger
21
from vllm.platforms import current_platform
22
23
-from .adapters import as_embedding_model
23
from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
24
supports_cross_encoding, supports_multimodal,
25
supports_pp)
27
-from .interfaces_base import is_pooling_model, is_text_generation_model
26
+from .interfaces_base import is_text_generation_model
27
28
logger = init_logger(__name__)
29
@@ -125,12 +124,13 @@ _EMBEDDING_MODELS = {
124
"Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
125
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
126
"Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
128
- "Qwen2ForSequenceClassification": ("qwen2_cls", "Qwen2ForSequenceClassification"), # noqa: E501
127
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
128
# [Multimodal]
129
"LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
130
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
131
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
132
+ # [Auto-converted (see adapters.py)]
133
+ "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"),
134
}
135
136
_CROSS_ENCODER_MODELS = {
@@ -226,19 +226,10 @@ class _ModelInfo:
226
227
@staticmethod
228
def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
229
- is_pooling_model_ = is_pooling_model(model)
230
- if not is_pooling_model_:
231
- try:
232
- as_embedding_model(model)
233
- except Exception:
234
- pass
235
- else:
236
- is_pooling_model_ = True
237
-
229
return _ModelInfo(
230
architecture=model.__name__,
231
is_text_generation_model=is_text_generation_model(model),
241
- is_pooling_model=is_pooling_model_,
232
+ is_pooling_model=True, # Can convert any model into a pooling model
233
supports_cross_encoding=supports_cross_encoding(model),
234
supports_multimodal=supports_multimodal(model),
235
supports_pp=supports_pp(model),