@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f3e92e1f2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Model] Automatic conversion of classification and reward models (#11469)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 25, 2024 at 02:22 UTC 3f3e92e1f2e332547f7d4bac2358bc4e8e5d018b
9 files changed +206 -161
docs/source/models/supported_models.md
+12 -10
@@ -28,7 +28,7 @@ llm = LLM(model=..., task="generate") # Name or path of your model
28 output = llm.generate("Hello, my name is")
29 print(output)
30
31 -# For pooling models (task={embed,classify,reward}) only
31 +# For pooling models (task={embed,classify,reward,score}) only
32 llm = LLM(model=..., task="embed") # Name or path of your model
33 output = llm.encode("Hello, my name is")
34 print(output)
@@ -59,7 +59,7 @@ llm = LLM(model=..., revision=..., task=..., trust_remote_code=True)
59 output = llm.generate("Hello, my name is")
60 print(output)
61
62 -# For pooling models (task={embed,classify,reward}) only
62 +# For pooling models (task={embed,classify,reward,score}) only
63 output = llm.encode("Hello, my name is")
64 print(output)
65 ```
@@ -369,14 +369,6 @@ you should explicitly specify the task type to ensure that the model is used in
369
370 #### Text Embedding (`--task embed`)
371
372 -Any text generation model can be converted into an embedding model by passing {code}`--task embed`.
373 -
374 -```{note}
375 -To get the best results, you should use pooling models that are specifically trained as such.
376 -```
377 -
378 -The following table lists those that are tested in vLLM.
379 -
372 ```{eval-rst}
373 .. list-table::
374 :widths: 25 25 50 5 5
@@ -437,6 +429,10 @@ On the other hand, its 1.5B variant ({code}`Alibaba-NLP/gte-Qwen2-1.5B-instruct`
429 despite being described otherwise on its model card.
430 ```
431
432 +If your model is not in the above list, we will try to automatically convert the model using
433 +:func:`vllm.model_executor.models.adapters.as_embedding_model`. By default, the embeddings
434 +of the whole prompt are extracted from the normalized hidden state corresponding to the last token.
435 +
436 #### Reward Modeling (`--task reward`)
437
438 ```{eval-rst}
@@ -461,6 +457,9 @@ despite being described otherwise on its model card.
457 - ✅︎
458 ```
459
460 +If your model is not in the above list, we will try to automatically convert the model using
461 +:func:`vllm.model_executor.models.adapters.as_reward_model`. By default, we return the hidden states of each token directly.
462 +
463 ```{important}
464 For process-supervised reward models such as {code}`peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
465 e.g.: {code}`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
@@ -490,6 +489,9 @@ e.g.: {code}`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 1
489 - ✅︎
490 ```
491
492 +If your model is not in the above list, we will try to automatically convert the model using
493 +:func:`vllm.model_executor.models.adapters.as_classification_model`. By default, the class probabilities are extracted from the softmaxed hidden state corresponding to the last token.
494 +
495 #### Sentence Pair Scoring (`--task score`)
496
497 ```{eval-rst}
tests/models/embedding/language/test_cls_models.py
+1 -4
@@ -1,7 +1,4 @@
1 -"""Compare the outputs of HF and vLLM when using greedy sampling.
2 -
3 -This test only tests small models. Big models such as 7B should be tested from
4 -test_big_models.py because it could use a larger instance to run tests.
1 +"""Compare the classification outputs of HF and vLLM models.
2
3 Run `pytest tests/models/test_cls_models.py`.
4 """
tests/models/embedding/language/test_scoring.py
+2 -2
@@ -1,6 +1,6 @@
1 -"""Compare the embedding outputs of HF and vLLM models.
1 +"""Compare the scoring outputs of HF and vLLM models.
2
3 -Run `pytest tests/models/embedding/language/test_embedding.py`.
3 +Run `pytest tests/models/embedding/language/test_scoring.py`.
4 """
5 import math
6
tests/models/test_registry.py
+7 -4
@@ -6,7 +6,9 @@ import torch.cuda
6 from vllm.model_executor.models import (is_pooling_model,
7 is_text_generation_model,
8 supports_multimodal)
9 -from vllm.model_executor.models.adapters import as_embedding_model
9 +from vllm.model_executor.models.adapters import (as_classification_model,
10 + as_embedding_model,
11 + as_reward_model)
12 from vllm.model_executor.models.registry import (_MULTIMODAL_MODELS,
13 _SPECULATIVE_DECODING_MODELS,
14 _TEXT_GENERATION_MODELS,
@@ -29,9 +31,10 @@ def test_registry_imports(model_arch):
31 or model_arch in _MULTIMODAL_MODELS):
32 assert is_text_generation_model(model_cls)
33
32 - # All vLLM models should be convertible to an embedding model
33 - embed_model = as_embedding_model(model_cls)
34 - assert is_pooling_model(embed_model)
34 + # All vLLM models should be convertible to a pooling model
35 + assert is_pooling_model(as_classification_model(model_cls))
36 + assert is_pooling_model(as_embedding_model(model_cls))
37 + assert is_pooling_model(as_reward_model(model_cls))
38
39 if model_arch in _MULTIMODAL_MODELS:
40 assert supports_multimodal(model_cls)
vllm/model_executor/model_loader/utils.py
+8 -2
@@ -7,7 +7,9 @@ from torch import nn
7
8 from vllm.config import ModelConfig
9 from vllm.model_executor.models import ModelRegistry
10 -from vllm.model_executor.models.adapters import as_embedding_model
10 +from vllm.model_executor.models.adapters import (as_classification_model,
11 + as_embedding_model,
12 + as_reward_model)
13
14
15 @contextlib.contextmanager
@@ -35,8 +37,12 @@ def get_model_architecture(
37 architectures = ["QuantMixtralForCausalLM"]
38
39 model_cls, arch = ModelRegistry.resolve_model_cls(architectures)
38 - if model_config.runner_type == "pooling":
40 + if model_config.task == "embed":
41 model_cls = as_embedding_model(model_cls)
42 + elif model_config.task == "classify":
43 + model_cls = as_classification_model(model_cls)
44 + elif model_config.task == "reward":
45 + model_cls = as_reward_model(model_cls)
46
47 return model_cls, arch
48
vllm/model_executor/models/adapters.py
+170 -20
@@ -1,29 +1,48 @@
1 from collections.abc import Iterable
2 -from typing import Any, TypeVar
2 +from typing import TYPE_CHECKING, Any, Optional, TypeVar
3
4 import torch
5 import torch.nn as nn
6
7 from .interfaces_base import VllmModelForPooling, is_pooling_model
8
9 +if TYPE_CHECKING:
10 + from vllm.model_executor.layers.pooler import PoolingType
11 +
12 _T = TypeVar("_T", bound=type[nn.Module])
13
14 +_GENERATE_SUFFIXES = [
15 + "ForCausalLM",
16 + "ForConditionalGeneration",
17 + "ChatModel",
18 + "LMHeadModel",
19 +]
20
12 -def as_embedding_model(cls: _T) -> _T:
13 - """Subclass an existing vLLM model to support embeddings."""
14 - # Avoid modifying existing embedding models
15 - if is_pooling_model(cls):
16 - return cls
21
22 +def _get_pooling_model_name(orig_model_name: str, pooling_suffix: str) -> str:
23 + model_name = orig_model_name
24 +
25 + for generate_suffix in _GENERATE_SUFFIXES:
26 + model_name = model_name.removesuffix(generate_suffix)
27 +
28 + return model_name + pooling_suffix
29 +
30 +
31 +def _create_pooling_model_cls(
32 + orig_cls: _T,
33 + *,
34 + default_pooling_type: "PoolingType",
35 + default_normalize: bool,
36 + default_softmax: bool,
37 +) -> _T:
38 # Lazy import
39 from vllm.config import VllmConfig
20 - from vllm.model_executor.layers.pooler import (Pooler, PoolerOutput,
21 - PoolingType)
40 + from vllm.model_executor.layers.pooler import Pooler, PoolerOutput
41 from vllm.model_executor.pooling_metadata import PoolingMetadata
42
43 from .utils import AutoWeightsLoader, WeightsMapper
44
26 - class ModelForEmbedding(cls, VllmModelForPooling):
45 + class ModelForPooling(orig_cls, VllmModelForPooling):
46
47 def __init__(
48 self,
@@ -34,7 +53,7 @@ def as_embedding_model(cls: _T) -> _T:
53 ) -> None:
54 super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
55
37 - # These are not used in embedding models
56 + # These are not used in pooling models
57 for attr in ("lm_head", "logits_processor"):
58 if hasattr(self, attr):
59 delattr(self, attr)
@@ -46,9 +65,9 @@ def as_embedding_model(cls: _T) -> _T:
65 if not getattr(self, "_pooler", None):
66 self._pooler = Pooler.from_config_with_defaults(
67 pooler_config,
49 - pooling_type=PoolingType.LAST,
50 - normalize=True,
51 - softmax=False,
68 + pooling_type=default_pooling_type,
69 + normalize=default_normalize,
70 + softmax=default_softmax,
71 )
72
73 def pooler(
@@ -82,17 +101,148 @@ def as_embedding_model(cls: _T) -> _T:
101 return
102
103 # For most other models
85 - if hasattr(cls, "load_weights"):
86 - cls.load_weights(self, weights) # type: ignore
104 + if hasattr(orig_cls, "load_weights"):
105 + orig_cls.load_weights(self, weights) # type: ignore
106 # Fallback
107 else:
108 loader = AutoWeightsLoader(self)
109 loader.load_weights(weights)
110
92 - ModelForEmbedding.__name__ = cls.__name__ \
93 - .removesuffix("ForCausalLM") \
94 - .removesuffix("ForConditionalGeneration") \
95 - .removesuffix("ChatModel") \
96 - .removesuffix("LMHeadModel") + "ForEmbedding"
111 + return ModelForPooling # type: ignore
112 +
113 +
114 +def as_embedding_model(cls: _T) -> _T:
115 + """
116 + Subclass an existing vLLM model to support embeddings.
117 +
118 + By default, the embeddings of the whole prompt are extracted from the
119 + normalized hidden state corresponding to the last token.
120 +
121 + Note:
122 + We assume that no extra layers are added to the original model;
123 + please implement your own model if this is not the case.
124 + """
125 + # Avoid modifying existing embedding models
126 + if is_pooling_model(cls):
127 + return cls
128 +
129 + # Lazy import
130 + from vllm.model_executor.layers.pooler import PoolingType
131 +
132 + ModelForEmbedding = _create_pooling_model_cls(
133 + cls,
134 + default_pooling_type=PoolingType.LAST,
135 + default_normalize=True,
136 + default_softmax=False,
137 + )
138 + ModelForEmbedding.__name__ = \
139 + _get_pooling_model_name(cls.__name__, "ForEmbedding")
140
141 return ModelForEmbedding # type: ignore
142 +
143 +
144 +def as_classification_model(cls: _T) -> _T:
145 + """
146 + Subclass an existing vLLM model to support classification.
147 +
148 + By default, the class probabilities are extracted from the softmaxed
149 + hidden state corresponding to the last token.
150 +
151 + Note:
152 + We assume that the classification head is a single linear layer
153 + stored as the attribute `score` of the top-level model;
154 + please implement your own model if this is not the case.
155 + """
156 + # Avoid modifying existing classification models
157 + if is_pooling_model(cls):
158 + return cls
159 +
160 + # Lazy import
161 + from vllm.attention import AttentionMetadata
162 + from vllm.config import VllmConfig
163 + from vllm.model_executor.layers.linear import RowParallelLinear
164 + from vllm.model_executor.layers.pooler import PoolingType
165 + from vllm.sequence import IntermediateTensors
166 +
167 + from .utils import maybe_prefix
168 +
169 + ModelForPooling = _create_pooling_model_cls(
170 + cls,
171 + default_pooling_type=PoolingType.LAST,
172 + default_normalize=False,
173 + default_softmax=True,
174 + )
175 +
176 + class ModelForClassification(ModelForPooling):
177 +
178 + def __init__(
179 + self,
180 + *,
181 + vllm_config: "VllmConfig",
182 + prefix: str = "",
183 + **kwargs: Any,
184 + ) -> None:
185 + super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
186 +
187 + config = vllm_config.model_config.hf_config
188 + quant_config = vllm_config.quant_config
189 +
190 + self.score = RowParallelLinear(config.hidden_size,
191 + config.num_labels,
192 + quant_config=quant_config,
193 + input_is_parallel=False,
194 + bias=False,
195 + prefix=maybe_prefix(
196 + prefix, "score"))
197 +
198 + def forward(
199 + self,
200 + input_ids: torch.Tensor,
201 + positions: torch.Tensor,
202 + kv_caches: list[torch.Tensor],
203 + attn_metadata: AttentionMetadata,
204 + intermediate_tensors: Optional[IntermediateTensors] = None,
205 + inputs_embeds: Optional[torch.Tensor] = None,
206 + ) -> torch.Tensor:
207 + hidden_states = super().forward(input_ids, positions, kv_caches,
208 + attn_metadata,
209 + intermediate_tensors,
210 + inputs_embeds)
211 + logits, _ = self.score(hidden_states)
212 + return logits
213 +
214 +
215 + ModelForClassification.__name__ = \
216 + _get_pooling_model_name(cls.__name__, "ForClassification")
217 +
218 + return ModelForClassification # type: ignore
219 +
220 +
221 +def as_reward_model(cls: _T) -> _T:
222 + """
223 + Subclass an existing vLLM model to support reward modeling.
224 +
225 + By default, we return the hidden states of each token directly.
226 +
227 + Note:
228 + We assume that no extra layers are added to the original model;
229 + please implement your own model if this is not the case.
230 + """
231 + # Avoid modifying existing reward models
232 + if is_pooling_model(cls):
233 + return cls
234 +
235 + # Lazy import
236 + from vllm.model_executor.layers.pooler import PoolingType
237 +
238 + ModelForReward = _create_pooling_model_cls(
239 + cls,
240 + default_pooling_type=PoolingType.ALL,
241 + default_normalize=False,
242 + default_softmax=False,
243 + )
244 +
245 + ModelForReward.__name__ = \
246 + _get_pooling_model_name(cls.__name__, "ForReward")
247 +
248 + return ModelForReward # type: ignore
vllm/model_executor/models/qwen2.py
+2 -2
@@ -545,8 +545,8 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
545 self.model = Qwen2Model(vllm_config=vllm_config,
546 prefix=maybe_prefix(prefix, "model"))
547
548 - # TODO: Replace this model class with for_embedding(Qwen2ForCausalLM),
549 - # after changing the default pooling method
548 + # TODO: Replace this model class with as_embedding_model(
549 + # Qwen2ForCausalLM) after changing the default pooling method
550 if pooler_config.pooling_type is None:
551 logger.warning(
552 "This embedding model will default to last-token pooling in "
vllm/model_executor/models/qwen2_cls.py deleted
-104
@@ -1,104 +0,0 @@
1 -# Adapted from
2 -# https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B/blob/main/modeling_qwen2_rm.py
3 -# Copyright 2024 Kakao Corp. (Kanana-X Team)
4 -# Copyright 2024 The Qwen team.
5 -# Copyright 2023 The vLLM team.
6 -"""Inference-only Qwen2-Classification model compatible with HF weights."""
7 -from typing import Iterable, List, Optional, Set, Tuple
8 -
9 -import torch
10 -from torch import nn
11 -
12 -from vllm.attention import AttentionMetadata
13 -from vllm.config import VllmConfig
14 -from vllm.model_executor.layers.linear import RowParallelLinear
15 -from vllm.model_executor.layers.pooler import Pooler, PoolingType
16 -from vllm.model_executor.models.qwen2 import Qwen2Model
17 -from vllm.model_executor.pooling_metadata import PoolingMetadata
18 -from vllm.sequence import IntermediateTensors, PoolerOutput
19 -
20 -from .interfaces import SupportsLoRA, SupportsPP
21 -from .utils import AutoWeightsLoader, maybe_prefix
22 -
23 -
24 -class Qwen2ForSequenceClassification(nn.Module, SupportsLoRA, SupportsPP):
25 - packed_modules_mapping = {
26 - "qkv_proj": [
27 - "q_proj",
28 - "k_proj",
29 - "v_proj",
30 - ],
31 - "gate_up_proj": [
32 - "gate_proj",
33 - "up_proj",
34 - ],
35 - }
36 -
37 - # LoRA specific attributes
38 - supported_lora_modules = [
39 - "qkv_proj",
40 - "o_proj",
41 - "gate_up_proj",
42 - "down_proj",
43 - ]
44 - embedding_modules = {}
45 - embedding_padding_modules = []
46 -
47 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
48 - super().__init__()
49 - config = vllm_config.model_config.hf_config
50 - quant_config = vllm_config.quant_config
51 - lora_config = vllm_config.lora_config
52 - pooler_config = vllm_config.model_config.pooler_config
53 -
54 - self.config = config
55 - self.lora_config = lora_config
56 -
57 - self.quant_config = quant_config
58 - self.model = Qwen2Model(vllm_config=vllm_config,
59 - prefix=maybe_prefix(prefix, "model"))
60 -
61 - # hidden_states from Qwen2Model has been reduced,
62 - # the input of score layer is not parallelized.
63 - self.score = RowParallelLinear(config.hidden_size,
64 - config.num_labels,
65 - quant_config=quant_config,
66 - input_is_parallel=False,
67 - bias=False,
68 - prefix=maybe_prefix(prefix, "score"))
69 - self._pooler = Pooler.from_config_with_defaults(
70 - pooler_config,
71 - pooling_type=PoolingType.LAST,
72 - normalize=False,
73 - softmax=True)
74 -
75 - def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
76 - return self.model.get_input_embeddings(input_ids)
77 -
78 - def forward(
79 - self,
80 - input_ids: torch.Tensor,
81 - positions: torch.Tensor,
82 - kv_caches: List[torch.Tensor],
83 - attn_metadata: AttentionMetadata,
84 - intermediate_tensors: Optional[IntermediateTensors] = None,
85 - inputs_embeds: Optional[torch.Tensor] = None,
86 - ) -> torch.Tensor:
87 - hidden_states = self.model(input_ids, positions, kv_caches,
88 - attn_metadata, intermediate_tensors,
89 - inputs_embeds)
90 - logits, _ = self.score(hidden_states)
91 - return logits
92 -
93 - def pooler(
94 - self,
95 - hidden_states: torch.Tensor,
96 - pooling_metadata: PoolingMetadata,
97 - ) -> Optional[PoolerOutput]:
98 - return self._pooler(hidden_states, pooling_metadata)
99 -
100 - def load_weights(self, weights: Iterable[Tuple[str,
101 - torch.Tensor]]) -> Set[str]:
102 - loader = AutoWeightsLoader(self,
103 - ignore_unexpected_prefixes=["lm_head."])
104 - return loader.load_weights(weights)
vllm/model_executor/models/registry.py
+4 -13
@@ -20,11 +20,10 @@ import torch.nn as nn
20 from vllm.logger import init_logger
21 from vllm.platforms import current_platform
22
23 -from .adapters import as_embedding_model
23 from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
24 supports_cross_encoding, supports_multimodal,
25 supports_pp)
27 -from .interfaces_base import is_pooling_model, is_text_generation_model
26 +from .interfaces_base import is_text_generation_model
27
28 logger = init_logger(__name__)
29
@@ -125,12 +124,13 @@ _EMBEDDING_MODELS = {
124 "Qwen2Model": ("qwen2", "Qwen2EmbeddingModel"),
125 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
126 "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
128 - "Qwen2ForSequenceClassification": ("qwen2_cls", "Qwen2ForSequenceClassification"), # noqa: E501
127 "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
128 # [Multimodal]
129 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
130 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
131 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
132 + # [Auto-converted (see adapters.py)]
133 + "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"),
134 }
135
136 _CROSS_ENCODER_MODELS = {
@@ -226,19 +226,10 @@ class _ModelInfo:
226
227 @staticmethod
228 def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
229 - is_pooling_model_ = is_pooling_model(model)
230 - if not is_pooling_model_:
231 - try:
232 - as_embedding_model(model)
233 - except Exception:
234 - pass
235 - else:
236 - is_pooling_model_ = True
237 -
229 return _ModelInfo(
230 architecture=model.__name__,
231 is_text_generation_model=is_text_generation_model(model),
241 - is_pooling_model=is_pooling_model_,
232 + is_pooling_model=True, # Can convert any model into a pooling model
233 supports_cross_encoding=supports_cross_encoding(model),
234 supports_multimodal=supports_multimodal(model),
235 supports_pp=supports_pp(model),