A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Support math-shepherd-mistral-7b-prm model (#9697)
Signed-off-by: Went-Liang <wenteng_liang@163.com>
Went-Liang committed
Oct 31, 2024 at 00:33 UTC
81f09cfd80a5a2e1572ee79facd60bb823923367
14 files changed
+312
-62
vllm/config.py
+85
-30
@@ -112,38 +112,58 @@ class ModelConfig:
112
Defaults to 'auto' which defaults to 'hf'.
113
mm_processor_kwargs: Arguments to be forwarded to the model's processor
114
for multi-modal data, e.g., image processor.
115
+ pooling_type: Used to configure the pooling method in the embedding
116
+ model.
117
+ pooling_norm: Used to determine whether to normalize the pooled
118
+ data in the embedding model.
119
+ pooling_softmax: Used to determine whether to softmax the pooled
120
+ data in the embedding model.
121
+ pooling_step_tag_id: When pooling_step_tag_id is not -1, it indicates
122
+ that the score corresponding to the pooling_step_tag_id in the
123
+ generated sentence should be returned. Otherwise, it returns
124
+ the scores for all tokens.
125
+ pooling_returned_token_ids: pooling_returned_token_ids represents a
126
+ list of indices for the vocabulary dimensions to be extracted,
127
+ such as the token IDs of good_token and bad_token in the
128
+ math-shepherd-mistral-7b-prm model.
129
"""
130
117
- def __init__(self,
118
- model: str,
119
- task: Union[TaskOption, _Task],
120
- tokenizer: str,
121
- tokenizer_mode: str,
122
- trust_remote_code: bool,
123
- dtype: Union[str, torch.dtype],
124
- seed: int,
125
- revision: Optional[str] = None,
126
- code_revision: Optional[str] = None,
127
- rope_scaling: Optional[dict] = None,
128
- rope_theta: Optional[float] = None,
129
- tokenizer_revision: Optional[str] = None,
130
- max_model_len: Optional[int] = None,
131
- spec_target_max_model_len: Optional[int] = None,
132
- quantization: Optional[str] = None,
133
- quantization_param_path: Optional[str] = None,
134
- enforce_eager: Optional[bool] = None,
135
- max_context_len_to_capture: Optional[int] = None,
136
- max_seq_len_to_capture: Optional[int] = None,
137
- max_logprobs: int = 20,
138
- disable_sliding_window: bool = False,
139
- skip_tokenizer_init: bool = False,
140
- served_model_name: Optional[Union[str, List[str]]] = None,
141
- limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
142
- use_async_output_proc: bool = True,
143
- override_neuron_config: Optional[Dict[str, Any]] = None,
144
- config_format: ConfigFormat = ConfigFormat.AUTO,
145
- chat_template_text_format: str = "string",
146
- mm_processor_kwargs: Optional[Dict[str, Any]] = None) -> None:
131
+ def __init__(
132
+ self,
133
+ model: str,
134
+ task: Union[TaskOption, _Task],
135
+ tokenizer: str,
136
+ tokenizer_mode: str,
137
+ trust_remote_code: bool,
138
+ dtype: Union[str, torch.dtype],
139
+ seed: int,
140
+ revision: Optional[str] = None,
141
+ code_revision: Optional[str] = None,
142
+ rope_scaling: Optional[dict] = None,
143
+ rope_theta: Optional[float] = None,
144
+ tokenizer_revision: Optional[str] = None,
145
+ max_model_len: Optional[int] = None,
146
+ spec_target_max_model_len: Optional[int] = None,
147
+ quantization: Optional[str] = None,
148
+ quantization_param_path: Optional[str] = None,
149
+ enforce_eager: Optional[bool] = None,
150
+ max_context_len_to_capture: Optional[int] = None,
151
+ max_seq_len_to_capture: Optional[int] = None,
152
+ max_logprobs: int = 20,
153
+ disable_sliding_window: bool = False,
154
+ skip_tokenizer_init: bool = False,
155
+ served_model_name: Optional[Union[str, List[str]]] = None,
156
+ limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
157
+ use_async_output_proc: bool = True,
158
+ override_neuron_config: Optional[Dict[str, Any]] = None,
159
+ config_format: ConfigFormat = ConfigFormat.AUTO,
160
+ chat_template_text_format: str = "string",
161
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
162
+ pooling_type: Optional[str] = None,
163
+ pooling_norm: Optional[bool] = None,
164
+ pooling_softmax: Optional[bool] = None,
165
+ pooling_step_tag_id: Optional[int] = None,
166
+ pooling_returned_token_ids: Optional[List[int]] = None) -> None:
167
self.model = model
168
self.tokenizer = tokenizer
169
self.tokenizer_mode = tokenizer_mode
@@ -224,6 +244,13 @@ class ModelConfig:
244
supported_tasks, task = self._resolve_task(task, self.hf_config)
245
self.supported_tasks = supported_tasks
246
self.task: Final = task
247
+ self.pooler_config = self._init_pooler_config(
248
+ pooling_type,
249
+ pooling_norm,
250
+ pooling_softmax,
251
+ pooling_step_tag_id,
252
+ pooling_returned_token_ids,
253
+ )
254
255
self._verify_quantization()
256
self._verify_cuda_graph()
@@ -242,6 +269,23 @@ class ModelConfig:
269
270
return None
271
272
+ def _init_pooler_config(
273
+ self,
274
+ pooling_type: Optional[str] = None,
275
+ pooling_norm: Optional[bool] = None,
276
+ pooling_softmax: Optional[bool] = None,
277
+ pooling_step_tag_id: Optional[int] = None,
278
+ pooling_returned_token_ids: Optional[List[int]] = None
279
+ ) -> Optional["PoolerConfig"]:
280
+ if self.task == "embedding":
281
+ return PoolerConfig(
282
+ pooling_type=pooling_type,
283
+ pooling_norm=pooling_norm,
284
+ pooling_softmax=pooling_softmax,
285
+ pooling_step_tag_id=pooling_step_tag_id,
286
+ pooling_returned_token_ids=pooling_returned_token_ids)
287
+ return None
288
+
289
def _init_attention_free(self) -> bool:
290
architectures = getattr(self.hf_config, "architectures", [])
291
return ModelRegistry.is_attention_free_model(architectures)
@@ -1647,6 +1691,17 @@ class MultiModalConfig:
1691
# TODO: Add configs to init vision tower or not.
1692
1693
1694
+@dataclass
1695
+class PoolerConfig:
1696
+ """Controls the behavior of pooler in embedding model"""
1697
+
1698
+ pooling_type: Optional[str] = None
1699
+ pooling_norm: Optional[bool] = None
1700
+ pooling_softmax: Optional[bool] = None
1701
+ pooling_step_tag_id: Optional[int] = None
1702
+ pooling_returned_token_ids: Optional[List[int]] = None
1703
+
1704
+
1705
_STR_DTYPE_TO_TORCH_DTYPE = {
1706
"half": torch.float16,
1707
"float16": torch.float16,
vllm/engine/arg_utils.py
+64
@@ -184,6 +184,13 @@ class EngineArgs:
184
mm_processor_kwargs: Optional[Dict[str, Any]] = None
185
scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
186
187
+ # Pooling configuration.
188
+ pooling_type: Optional[str] = None
189
+ pooling_norm: Optional[bool] = None
190
+ pooling_softmax: Optional[bool] = None
191
+ pooling_step_tag_id: Optional[int] = None
192
+ pooling_returned_token_ids: Optional[List[int]] = None
193
+
194
def __post_init__(self):
195
if not self.tokenizer:
196
self.tokenizer = self.model
@@ -850,6 +857,58 @@ class EngineArgs:
857
'priority (lower value means earlier handling) and time of '
858
'arrival deciding any ties).')
859
860
+ parser.add_argument(
861
+ '--pooling-type',
862
+ choices=['LAST', 'ALL', 'CLS', 'STEP'],
863
+ default=None,
864
+ help='Used to configure the pooling method in the embedding model.'
865
+ )
866
+
867
+ parser.add_argument('--pooling-norm',
868
+ default=None,
869
+ action='store_true',
870
+ help="Used to determine whether to normalize "
871
+ "the pooled data in the embedding model.")
872
+
873
+ parser.add_argument('--no-pooling-norm',
874
+ default=None,
875
+ action='store_false',
876
+ dest='pooling_norm',
877
+ help="Used to determine whether to normalize "
878
+ "the pooled data in the embedding model.")
879
+
880
+ parser.add_argument('--pooling-softmax',
881
+ default=None,
882
+ action='store_true',
883
+ help="Used to determine whether to softmax "
884
+ "the pooled data in the embedding model.")
885
+
886
+ parser.add_argument('--no-pooling-softmax',
887
+ default=None,
888
+ action='store_false',
889
+ dest='pooling_softmax',
890
+ help="Used to determine whether to softmax "
891
+ "the pooled data in the embedding model.")
892
+
893
+ parser.add_argument(
894
+ '--pooling-step-tag-id',
895
+ type=int,
896
+ default=None,
897
+ help="When pooling-step-tag-id is not -1, it indicates "
898
+ "that the score corresponding to the step-tag-ids in the "
899
+ "generated sentence should be returned. Otherwise, it "
900
+ "returns the scores for all tokens.")
901
+
902
+ parser.add_argument(
903
+ '--pooling-returned-token-ids',
904
+ nargs='+',
905
+ type=int,
906
+ default=None,
907
+ help="pooling-returned-token-ids represents a list of "
908
+ "indices for the vocabulary dimensions to be extracted, "
909
+ "such as the token IDs of good_token and bad_token in "
910
+ "the math-shepherd-mistral-7b-prm model.")
911
+
912
return parser
913
914
@classmethod
@@ -891,6 +950,11 @@ class EngineArgs:
950
override_neuron_config=self.override_neuron_config,
951
config_format=self.config_format,
952
mm_processor_kwargs=self.mm_processor_kwargs,
953
+ pooling_type=self.pooling_type,
954
+ pooling_norm=self.pooling_norm,
955
+ pooling_softmax=self.pooling_softmax,
956
+ pooling_step_tag_id=self.pooling_step_tag_id,
957
+ pooling_returned_token_ids=self.pooling_returned_token_ids,
958
)
959
960
def create_load_config(self) -> LoadConfig:
vllm/engine/llm_engine.py
+3
-1
@@ -257,7 +257,8 @@ class LLMEngine:
257
"num_scheduler_steps=%d, chunked_prefill_enabled=%s "
258
"multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
259
"use_async_output_proc=%s, use_cached_outputs=%s, "
260
- "chat_template_text_format=%s, mm_processor_kwargs=%s)",
260
+ "chat_template_text_format=%s, mm_processor_kwargs=%s, "
261
+ "pooler_config=%r)",
262
VLLM_VERSION,
263
model_config.model,
264
speculative_config,
@@ -294,6 +295,7 @@ class LLMEngine:
295
use_cached_outputs,
296
model_config.chat_template_text_format,
297
model_config.mm_processor_kwargs,
298
+ model_config.pooler_config,
299
)
300
# TODO(woosuk): Print more configs in debug mode.
301
self.model_config = model_config
vllm/entrypoints/llm.py
+10
@@ -159,6 +159,11 @@ class LLM:
159
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
160
# After positional args are removed, move this right below `model`
161
task: TaskOption = "auto",
162
+ pooling_type: Optional[str] = None,
163
+ pooling_norm: Optional[bool] = None,
164
+ pooling_softmax: Optional[bool] = None,
165
+ pooling_step_tag_id: Optional[int] = None,
166
+ pooling_returned_token_ids: Optional[List[int]] = None,
167
**kwargs,
168
) -> None:
169
'''
@@ -193,6 +198,11 @@ class LLM:
198
disable_custom_all_reduce=disable_custom_all_reduce,
199
disable_async_output_proc=disable_async_output_proc,
200
mm_processor_kwargs=mm_processor_kwargs,
201
+ pooling_type=pooling_type,
202
+ pooling_norm=pooling_norm,
203
+ pooling_softmax=pooling_softmax,
204
+ pooling_step_tag_id=pooling_step_tag_id,
205
+ pooling_returned_token_ids=pooling_returned_token_ids,
206
**kwargs,
207
)
208
self.llm_engine = LLMEngine.from_engine_args(
vllm/model_executor/layers/pooler.py
+58
-4
@@ -1,8 +1,10 @@
1
from enum import IntEnum
2
+from typing import List, Optional
3
4
import torch
5
import torch.nn as nn
6
7
+from vllm.config import PoolerConfig
8
from vllm.model_executor.pooling_metadata import (PoolingMetadata,
9
PoolingTensors)
10
from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
@@ -13,6 +15,7 @@ class PoolingType(IntEnum):
15
LAST = 0
16
ALL = 1
17
CLS = 2
18
+ STEP = 3
19
20
21
class Pooler(nn.Module):
@@ -28,15 +31,47 @@ class Pooler(nn.Module):
31
normalize: Whether to normalize the pooled data.
32
"""
33
31
- def __init__(self,
32
- pooling_type: PoolingType,
33
- normalize: bool,
34
- softmax: bool = False):
34
+ def __init__(
35
+ self,
36
+ pooling_type: PoolingType,
37
+ normalize: bool,
38
+ softmax: bool,
39
+ step_tag_id: Optional[int] = None,
40
+ returned_token_ids: Optional[List[int]] = None,
41
+ ):
42
super().__init__()
43
44
self.pooling_type = pooling_type
45
self.normalize = normalize
46
self.softmax = softmax
47
+ self.step_tag_id = step_tag_id
48
+ self.returned_token_ids = returned_token_ids
49
+
50
+ @classmethod
51
+ def from_config_with_defaults(
52
+ cls,
53
+ pooler_config: PoolerConfig,
54
+ pooling_type: PoolingType,
55
+ normalize: bool,
56
+ softmax: bool,
57
+ step_tag_id: Optional[int] = None,
58
+ returned_token_ids: Optional[List[int]] = None,
59
+ ) -> Optional["Pooler"]:
60
+ if pooler_config is None:
61
+ return None
62
+ return cls(
63
+ pooling_type=PoolingType[pooler_config.pooling_type]
64
+ if pooler_config.pooling_type is not None else pooling_type,
65
+ normalize=pooler_config.pooling_norm
66
+ if pooler_config.pooling_norm is not None else normalize,
67
+ softmax=pooler_config.pooling_softmax
68
+ if pooler_config.pooling_softmax is not None else softmax,
69
+ step_tag_id=pooler_config.pooling_step_tag_id
70
+ if pooler_config.pooling_step_tag_id is not None else step_tag_id,
71
+ returned_token_ids=pooler_config.pooling_returned_token_ids
72
+ if pooler_config.pooling_returned_token_ids is not None else
73
+ returned_token_ids,
74
+ )
75
76
def forward(
77
self,
@@ -62,6 +97,25 @@ class Pooler(nn.Module):
97
for prompt_len in prompt_lens:
98
pooled_data.append(hidden_states[offset:offset + prompt_len])
99
offset += prompt_len
100
+ elif self.pooling_type == PoolingType.STEP:
101
+ if self.returned_token_ids is not None and len(
102
+ self.returned_token_ids) > 0:
103
+ logits = hidden_states[:,
104
+ self.returned_token_ids].softmax(dim=-1)
105
+ else:
106
+ logits = hidden_states.softmax(dim=-1)
107
+ offset = 0
108
+ pooled_data = []
109
+ for prompt_len, seq_data_i in zip(
110
+ prompt_lens, pooling_metadata.seq_data.values()):
111
+ if self.step_tag_id is None:
112
+ pooled_data.append(logits[offset:offset + prompt_len])
113
+ else:
114
+ step_idxs = torch.tensor(
115
+ seq_data_i.prompt_token_ids) == self.step_tag_id
116
+ pooled_data.append(logits[offset:offset +
117
+ prompt_len][step_idxs])
118
+ offset += prompt_len
119
else:
120
raise ValueError(f"Invalid pooling type: {self.pooling_type}")
121
vllm/model_executor/model_loader/loader.py
+10
-5
@@ -23,7 +23,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
23
24
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
25
LoRAConfig, ModelConfig, MultiModalConfig,
26
- ParallelConfig, SchedulerConfig)
26
+ ParallelConfig, PoolerConfig, SchedulerConfig)
27
from vllm.distributed import (get_tensor_model_parallel_rank,
28
get_tensor_model_parallel_world_size)
29
from vllm.envs import VLLM_USE_MODELSCOPE
@@ -122,7 +122,8 @@ def _get_model_initialization_kwargs(
122
model_class: Type[nn.Module],
123
lora_config: Optional[LoRAConfig],
124
multimodal_config: Optional[MultiModalConfig],
125
- scheduler_config: Optional[SchedulerConfig] = None) -> Dict[str, Any]:
125
+ scheduler_config: Optional[SchedulerConfig] = None,
126
+ pooler_config: Optional[PoolerConfig] = None) -> Dict[str, Any]:
127
"""Get extra kwargs for model initialization."""
128
extra_kwargs: Dict[str, Any] = {}
129
@@ -143,7 +144,8 @@ def _get_model_initialization_kwargs(
144
145
if has_inner_state(model_class) and scheduler_config:
146
extra_kwargs["scheduler_config"] = scheduler_config
146
-
147
+ if pooler_config:
148
+ extra_kwargs["pooler_config"] = pooler_config
149
return extra_kwargs
150
151
@@ -155,10 +157,12 @@ def build_model(model_class: Type[nn.Module],
157
lora_config: Optional[LoRAConfig],
158
multimodal_config: Optional[MultiModalConfig],
159
scheduler_config: Optional[SchedulerConfig],
158
- prefix: Optional[str] = None) -> nn.Module:
160
+ prefix: Optional[str] = None,
161
+ pooler_config: Optional[PoolerConfig] = None) -> nn.Module:
162
extra_kwargs = _get_model_initialization_kwargs(model_class, lora_config,
163
multimodal_config,
161
- scheduler_config)
164
+ scheduler_config,
165
+ pooler_config)
166
if prefix:
167
extra_kwargs["prefix"] = prefix
168
@@ -185,6 +189,7 @@ def _initialize_model(
189
lora_config=lora_config,
190
multimodal_config=model_config.multimodal_config,
191
scheduler_config=scheduler_config,
192
+ pooler_config=model_config.pooler_config,
193
)
194
195
vllm/model_executor/models/bert.py
+7
-2
@@ -6,7 +6,7 @@ from transformers import BertConfig
6
7
from vllm.attention import Attention, AttentionMetadata, AttentionType
8
from vllm.attention.backends.xformers import XFormersImpl
9
-from vllm.config import CacheConfig
9
+from vllm.config import CacheConfig, PoolerConfig
10
from vllm.distributed import get_tensor_model_parallel_world_size
11
from vllm.model_executor.layers.activation import get_act_fn
12
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -387,10 +387,15 @@ class BertEmbeddingModel(nn.Module):
387
config: BertConfig,
388
cache_config: Optional[CacheConfig] = None,
389
quant_config: Optional[QuantizationConfig] = None,
390
+ pooler_config: Optional[PoolerConfig] = None,
391
) -> None:
392
super().__init__()
393
self.model = BertModel(config, cache_config, quant_config)
393
- self._pooler = Pooler(pooling_type=PoolingType.CLS, normalize=True)
394
+ self._pooler = Pooler.from_config_with_defaults(
395
+ pooler_config,
396
+ pooling_type=PoolingType.CLS,
397
+ normalize=True,
398
+ softmax=False)
399
400
def forward(
401
self,
vllm/model_executor/models/gemma2.py
+7
-3
@@ -22,7 +22,7 @@ from transformers import Gemma2Config
22
23
from vllm.attention import Attention, AttentionMetadata
24
from vllm.compilation.decorators import support_torch_compile
25
-from vllm.config import CacheConfig, LoRAConfig
25
+from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
26
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
27
from vllm.logger import init_logger
28
from vllm.model_executor.layers.activation import GeluAndMul
@@ -473,13 +473,17 @@ class Gemma2EmbeddingModel(nn.Module, SupportsPP):
473
474
def __init__(
475
self,
476
+ pooler_config: Optional[PoolerConfig] = None,
477
**kwargs,
478
) -> None:
479
super().__init__()
480
481
self.model = Gemma2Model(**kwargs)
481
- self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
482
-
482
+ self._pooler = Pooler.from_config_with_defaults(
483
+ pooler_config,
484
+ pooling_type=PoolingType.LAST,
485
+ normalize=True,
486
+ softmax=False)
487
self.make_empty_intermediate_tensors = (
488
self.model.make_empty_intermediate_tensors)
489
vllm/model_executor/models/llama.py
+21
-2
@@ -29,7 +29,7 @@ from transformers import LlamaConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
-from vllm.config import CacheConfig, LoRAConfig
32
+from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
35
from vllm.model_executor.layers.activation import SiluAndMul
@@ -502,6 +502,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
502
quant_config: Optional[QuantizationConfig] = None,
503
lora_config: Optional[LoRAConfig] = None,
504
prefix: str = "",
505
+ pooler_config: Optional[PoolerConfig] = None,
506
) -> None:
507
super().__init__()
508
@@ -543,6 +544,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
544
self.lm_head = PPMissingLayer()
545
self.make_empty_intermediate_tensors = (
546
self.model.make_empty_intermediate_tensors)
547
+ self._pooler = Pooler.from_config_with_defaults(
548
+ pooler_config,
549
+ pooling_type=PoolingType.STEP,
550
+ normalize=False,
551
+ softmax=False)
552
553
def forward(
554
self,
@@ -565,6 +571,14 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
571
sampling_metadata)
572
return logits
573
574
+ def pooler(
575
+ self,
576
+ hidden_states: torch.Tensor,
577
+ pooling_metadata: PoolingMetadata,
578
+ ) -> Optional[PoolerOutput]:
579
+ logits = self.compute_logits(hidden_states, None)
580
+ return self._pooler(logits, pooling_metadata)
581
+
582
def sample(self, logits: torch.Tensor,
583
sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]:
584
next_tokens = self.sampler(logits, sampling_metadata)
@@ -630,12 +644,17 @@ class LlamaEmbeddingModel(nn.Module, SupportsPP):
644
645
def __init__(
646
self,
647
+ pooler_config: Optional[PoolerConfig] = None,
648
**kwargs,
649
) -> None:
650
super().__init__()
651
652
self.model = LlamaModel(**kwargs)
638
- self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
653
+ self._pooler = Pooler.from_config_with_defaults(
654
+ pooler_config,
655
+ pooling_type=PoolingType.LAST,
656
+ normalize=True,
657
+ softmax=False)
658
self.make_empty_intermediate_tensors = (
659
self.model.make_empty_intermediate_tensors)
660
vllm/model_executor/models/llava_next.py
+8
-4
@@ -11,7 +11,7 @@ from transformers.models.llava_next.modeling_llava_next import (
11
from typing_extensions import NotRequired
12
13
from vllm.attention import AttentionMetadata
14
-from vllm.config import CacheConfig, MultiModalConfig
14
+from vllm.config import CacheConfig, MultiModalConfig, PoolerConfig
15
from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
16
from vllm.model_executor.layers.pooler import Pooler, PoolingType
17
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -285,7 +285,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
285
config: LlavaNextConfig,
286
multimodal_config: MultiModalConfig,
287
cache_config: Optional[CacheConfig] = None,
288
- quant_config: Optional[QuantizationConfig] = None) -> None:
288
+ quant_config: Optional[QuantizationConfig] = None,
289
+ pooler_config: Optional[PoolerConfig] = None) -> None:
290
super().__init__()
291
292
self.config = config
@@ -312,8 +313,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
313
314
# The same model class supports both language generation and embedding
315
# because the architecture name is the same
315
- self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
316
-
316
+ self._pooler = Pooler.from_config_with_defaults(
317
+ pooler_config,
318
+ pooling_type=PoolingType.LAST,
319
+ normalize=True,
320
+ softmax=False)
321
self.make_empty_intermediate_tensors = (
322
self.language_model.make_empty_intermediate_tensors)
323
vllm/model_executor/models/phi3v.py
+9
-4
@@ -26,7 +26,8 @@ from PIL import Image
26
from transformers import CLIPVisionConfig, PretrainedConfig
27
28
from vllm.attention import AttentionMetadata
29
-from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
29
+from vllm.config import (CacheConfig, ModelConfig, MultiModalConfig,
30
+ PoolerConfig)
31
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
32
token_inputs)
33
from vllm.logger import init_logger
@@ -530,7 +531,8 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
531
config: PretrainedConfig,
532
multimodal_config: MultiModalConfig,
533
cache_config: Optional[CacheConfig] = None,
533
- quant_config: Optional[QuantizationConfig] = None) -> None:
534
+ quant_config: Optional[QuantizationConfig] = None,
535
+ pooler_config: Optional[PoolerConfig] = None) -> None:
536
super().__init__()
537
538
self.config = config
@@ -556,8 +558,11 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
558
559
# The same model class supports both language generation and embedding
560
# because the architecture name is the same
559
- self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
560
-
561
+ self._pooler = Pooler.from_config_with_defaults(
562
+ pooler_config,
563
+ pooling_type=PoolingType.LAST,
564
+ normalize=True,
565
+ softmax=False)
566
self.make_empty_intermediate_tensors = (
567
self.language_model.make_empty_intermediate_tensors)
568
vllm/model_executor/models/qwen2_cls.py
+7
-4
@@ -12,7 +12,7 @@ from torch import nn
12
from transformers import Qwen2Config
13
14
from vllm.attention import AttentionMetadata
15
-from vllm.config import CacheConfig, LoRAConfig
15
+from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
16
from vllm.model_executor.layers.linear import RowParallelLinear
17
from vllm.model_executor.layers.pooler import Pooler, PoolingType
18
from vllm.model_executor.layers.quantization.base_config import (
@@ -53,6 +53,7 @@ class Qwen2ForSequenceClassification(nn.Module):
53
cache_config: Optional[CacheConfig] = None,
54
quant_config: Optional[QuantizationConfig] = None,
55
lora_config: Optional[LoRAConfig] = None,
56
+ pooler_config: Optional[PoolerConfig] = None,
57
) -> None:
58
# TODO (@robertgshaw2): see if this can be moved out
59
if (cache_config.sliding_window is not None
@@ -77,9 +78,11 @@ class Qwen2ForSequenceClassification(nn.Module):
78
self.score = RowParallelLinear(config.hidden_size,
79
config.num_labels,
80
quant_config=quant_config)
80
- self._pooler = Pooler(pooling_type=PoolingType.LAST,
81
- normalize=False,
82
- softmax=True)
81
+ self._pooler = Pooler.from_config_with_defaults(
82
+ pooler_config,
83
+ pooling_type=PoolingType.LAST,
84
+ normalize=False,
85
+ softmax=True)
86
87
def forward(
88
self,
vllm/model_executor/models/qwen2_rm.py
+7
-3
@@ -11,7 +11,7 @@ from torch import nn
11
from transformers import Qwen2Config
12
13
from vllm.attention import AttentionMetadata
14
-from vllm.config import CacheConfig, LoRAConfig
14
+from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
15
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
RowParallelLinear)
17
from vllm.model_executor.layers.pooler import Pooler, PoolingType
@@ -64,6 +64,7 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
64
cache_config: Optional[CacheConfig] = None,
65
quant_config: Optional[QuantizationConfig] = None,
66
lora_config: Optional[LoRAConfig] = None,
67
+ pooler_config: Optional[PoolerConfig] = None,
68
) -> None:
69
# TODO (@robertgshaw2): see if this can be moved out
70
if (cache_config.sliding_window is not None
@@ -93,8 +94,11 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
94
RowParallelLinear(config.hidden_size, 1,
95
quant_config=quant_config),
96
)
96
- self._pooler = Pooler(pooling_type=PoolingType.ALL, normalize=False)
97
-
97
+ self._pooler = Pooler.from_config_with_defaults(
98
+ pooler_config,
99
+ pooling_type=PoolingType.ALL,
100
+ normalize=False,
101
+ softmax=False)
102
self.make_empty_intermediate_tensors = (
103
self.model.make_empty_intermediate_tensors)
104
vllm/model_executor/models/registry.py
+16
@@ -100,11 +100,27 @@ _EMBEDDING_MODELS = {
100
"Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
101
"Qwen2ForSequenceClassification": (
102
"qwen2_cls", "Qwen2ForSequenceClassification"),
103
+ "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
104
+ "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
105
+ "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
106
# [Multimodal]
107
"LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
108
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
109
}
110
111
+def add_embedding_models(base_models, embedding_models):
112
+ with_pooler_method_models = {}
113
+ embedding_models_name = embedding_models.keys()
114
+ for name, (path, arch) in base_models.items():
115
+ if arch in embedding_models_name:
116
+ with_pooler_method_models[name] = (path, arch)
117
+ return with_pooler_method_models
118
+
119
+_EMBEDDING_MODELS = {
120
+ **add_embedding_models(_TEXT_GENERATION_MODELS, _EMBEDDING_MODELS),
121
+ **_EMBEDDING_MODELS,
122
+}
123
+
124
_MULTIMODAL_MODELS = {
125
# [Decoder-only]
126
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),