@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 81f09cfd80
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Support math-shepherd-mistral-7b-prm model (#9697)

Signed-off-by: Went-Liang <wenteng_liang@163.com>

Went-Liang committed Oct 31, 2024 at 00:33 UTC 81f09cfd80a5a2e1572ee79facd60bb823923367
14 files changed +312 -62
vllm/config.py
+85 -30
@@ -112,38 +112,58 @@ class ModelConfig:
112 Defaults to 'auto' which defaults to 'hf'.
113 mm_processor_kwargs: Arguments to be forwarded to the model's processor
114 for multi-modal data, e.g., image processor.
115 + pooling_type: Used to configure the pooling method in the embedding
116 + model.
117 + pooling_norm: Used to determine whether to normalize the pooled
118 + data in the embedding model.
119 + pooling_softmax: Used to determine whether to softmax the pooled
120 + data in the embedding model.
121 + pooling_step_tag_id: When pooling_step_tag_id is not -1, it indicates
122 + that the score corresponding to the pooling_step_tag_id in the
123 + generated sentence should be returned. Otherwise, it returns
124 + the scores for all tokens.
125 + pooling_returned_token_ids: pooling_returned_token_ids represents a
126 + list of indices for the vocabulary dimensions to be extracted,
127 + such as the token IDs of good_token and bad_token in the
128 + math-shepherd-mistral-7b-prm model.
129 """
130
117 - def __init__(self,
118 - model: str,
119 - task: Union[TaskOption, _Task],
120 - tokenizer: str,
121 - tokenizer_mode: str,
122 - trust_remote_code: bool,
123 - dtype: Union[str, torch.dtype],
124 - seed: int,
125 - revision: Optional[str] = None,
126 - code_revision: Optional[str] = None,
127 - rope_scaling: Optional[dict] = None,
128 - rope_theta: Optional[float] = None,
129 - tokenizer_revision: Optional[str] = None,
130 - max_model_len: Optional[int] = None,
131 - spec_target_max_model_len: Optional[int] = None,
132 - quantization: Optional[str] = None,
133 - quantization_param_path: Optional[str] = None,
134 - enforce_eager: Optional[bool] = None,
135 - max_context_len_to_capture: Optional[int] = None,
136 - max_seq_len_to_capture: Optional[int] = None,
137 - max_logprobs: int = 20,
138 - disable_sliding_window: bool = False,
139 - skip_tokenizer_init: bool = False,
140 - served_model_name: Optional[Union[str, List[str]]] = None,
141 - limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
142 - use_async_output_proc: bool = True,
143 - override_neuron_config: Optional[Dict[str, Any]] = None,
144 - config_format: ConfigFormat = ConfigFormat.AUTO,
145 - chat_template_text_format: str = "string",
146 - mm_processor_kwargs: Optional[Dict[str, Any]] = None) -> None:
131 + def __init__(
132 + self,
133 + model: str,
134 + task: Union[TaskOption, _Task],
135 + tokenizer: str,
136 + tokenizer_mode: str,
137 + trust_remote_code: bool,
138 + dtype: Union[str, torch.dtype],
139 + seed: int,
140 + revision: Optional[str] = None,
141 + code_revision: Optional[str] = None,
142 + rope_scaling: Optional[dict] = None,
143 + rope_theta: Optional[float] = None,
144 + tokenizer_revision: Optional[str] = None,
145 + max_model_len: Optional[int] = None,
146 + spec_target_max_model_len: Optional[int] = None,
147 + quantization: Optional[str] = None,
148 + quantization_param_path: Optional[str] = None,
149 + enforce_eager: Optional[bool] = None,
150 + max_context_len_to_capture: Optional[int] = None,
151 + max_seq_len_to_capture: Optional[int] = None,
152 + max_logprobs: int = 20,
153 + disable_sliding_window: bool = False,
154 + skip_tokenizer_init: bool = False,
155 + served_model_name: Optional[Union[str, List[str]]] = None,
156 + limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
157 + use_async_output_proc: bool = True,
158 + override_neuron_config: Optional[Dict[str, Any]] = None,
159 + config_format: ConfigFormat = ConfigFormat.AUTO,
160 + chat_template_text_format: str = "string",
161 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
162 + pooling_type: Optional[str] = None,
163 + pooling_norm: Optional[bool] = None,
164 + pooling_softmax: Optional[bool] = None,
165 + pooling_step_tag_id: Optional[int] = None,
166 + pooling_returned_token_ids: Optional[List[int]] = None) -> None:
167 self.model = model
168 self.tokenizer = tokenizer
169 self.tokenizer_mode = tokenizer_mode
@@ -224,6 +244,13 @@ class ModelConfig:
244 supported_tasks, task = self._resolve_task(task, self.hf_config)
245 self.supported_tasks = supported_tasks
246 self.task: Final = task
247 + self.pooler_config = self._init_pooler_config(
248 + pooling_type,
249 + pooling_norm,
250 + pooling_softmax,
251 + pooling_step_tag_id,
252 + pooling_returned_token_ids,
253 + )
254
255 self._verify_quantization()
256 self._verify_cuda_graph()
@@ -242,6 +269,23 @@ class ModelConfig:
269
270 return None
271
272 + def _init_pooler_config(
273 + self,
274 + pooling_type: Optional[str] = None,
275 + pooling_norm: Optional[bool] = None,
276 + pooling_softmax: Optional[bool] = None,
277 + pooling_step_tag_id: Optional[int] = None,
278 + pooling_returned_token_ids: Optional[List[int]] = None
279 + ) -> Optional["PoolerConfig"]:
280 + if self.task == "embedding":
281 + return PoolerConfig(
282 + pooling_type=pooling_type,
283 + pooling_norm=pooling_norm,
284 + pooling_softmax=pooling_softmax,
285 + pooling_step_tag_id=pooling_step_tag_id,
286 + pooling_returned_token_ids=pooling_returned_token_ids)
287 + return None
288 +
289 def _init_attention_free(self) -> bool:
290 architectures = getattr(self.hf_config, "architectures", [])
291 return ModelRegistry.is_attention_free_model(architectures)
@@ -1647,6 +1691,17 @@ class MultiModalConfig:
1691 # TODO: Add configs to init vision tower or not.
1692
1693
1694 +@dataclass
1695 +class PoolerConfig:
1696 + """Controls the behavior of pooler in embedding model"""
1697 +
1698 + pooling_type: Optional[str] = None
1699 + pooling_norm: Optional[bool] = None
1700 + pooling_softmax: Optional[bool] = None
1701 + pooling_step_tag_id: Optional[int] = None
1702 + pooling_returned_token_ids: Optional[List[int]] = None
1703 +
1704 +
1705 _STR_DTYPE_TO_TORCH_DTYPE = {
1706 "half": torch.float16,
1707 "float16": torch.float16,
vllm/engine/arg_utils.py
+64
@@ -184,6 +184,13 @@ class EngineArgs:
184 mm_processor_kwargs: Optional[Dict[str, Any]] = None
185 scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
186
187 + # Pooling configuration.
188 + pooling_type: Optional[str] = None
189 + pooling_norm: Optional[bool] = None
190 + pooling_softmax: Optional[bool] = None
191 + pooling_step_tag_id: Optional[int] = None
192 + pooling_returned_token_ids: Optional[List[int]] = None
193 +
194 def __post_init__(self):
195 if not self.tokenizer:
196 self.tokenizer = self.model
@@ -850,6 +857,58 @@ class EngineArgs:
857 'priority (lower value means earlier handling) and time of '
858 'arrival deciding any ties).')
859
860 + parser.add_argument(
861 + '--pooling-type',
862 + choices=['LAST', 'ALL', 'CLS', 'STEP'],
863 + default=None,
864 + help='Used to configure the pooling method in the embedding model.'
865 + )
866 +
867 + parser.add_argument('--pooling-norm',
868 + default=None,
869 + action='store_true',
870 + help="Used to determine whether to normalize "
871 + "the pooled data in the embedding model.")
872 +
873 + parser.add_argument('--no-pooling-norm',
874 + default=None,
875 + action='store_false',
876 + dest='pooling_norm',
877 + help="Used to determine whether to normalize "
878 + "the pooled data in the embedding model.")
879 +
880 + parser.add_argument('--pooling-softmax',
881 + default=None,
882 + action='store_true',
883 + help="Used to determine whether to softmax "
884 + "the pooled data in the embedding model.")
885 +
886 + parser.add_argument('--no-pooling-softmax',
887 + default=None,
888 + action='store_false',
889 + dest='pooling_softmax',
890 + help="Used to determine whether to softmax "
891 + "the pooled data in the embedding model.")
892 +
893 + parser.add_argument(
894 + '--pooling-step-tag-id',
895 + type=int,
896 + default=None,
897 + help="When pooling-step-tag-id is not -1, it indicates "
898 + "that the score corresponding to the step-tag-ids in the "
899 + "generated sentence should be returned. Otherwise, it "
900 + "returns the scores for all tokens.")
901 +
902 + parser.add_argument(
903 + '--pooling-returned-token-ids',
904 + nargs='+',
905 + type=int,
906 + default=None,
907 + help="pooling-returned-token-ids represents a list of "
908 + "indices for the vocabulary dimensions to be extracted, "
909 + "such as the token IDs of good_token and bad_token in "
910 + "the math-shepherd-mistral-7b-prm model.")
911 +
912 return parser
913
914 @classmethod
@@ -891,6 +950,11 @@ class EngineArgs:
950 override_neuron_config=self.override_neuron_config,
951 config_format=self.config_format,
952 mm_processor_kwargs=self.mm_processor_kwargs,
953 + pooling_type=self.pooling_type,
954 + pooling_norm=self.pooling_norm,
955 + pooling_softmax=self.pooling_softmax,
956 + pooling_step_tag_id=self.pooling_step_tag_id,
957 + pooling_returned_token_ids=self.pooling_returned_token_ids,
958 )
959
960 def create_load_config(self) -> LoadConfig:
vllm/engine/llm_engine.py
+3 -1
@@ -257,7 +257,8 @@ class LLMEngine:
257 "num_scheduler_steps=%d, chunked_prefill_enabled=%s "
258 "multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
259 "use_async_output_proc=%s, use_cached_outputs=%s, "
260 - "chat_template_text_format=%s, mm_processor_kwargs=%s)",
260 + "chat_template_text_format=%s, mm_processor_kwargs=%s, "
261 + "pooler_config=%r)",
262 VLLM_VERSION,
263 model_config.model,
264 speculative_config,
@@ -294,6 +295,7 @@ class LLMEngine:
295 use_cached_outputs,
296 model_config.chat_template_text_format,
297 model_config.mm_processor_kwargs,
298 + model_config.pooler_config,
299 )
300 # TODO(woosuk): Print more configs in debug mode.
301 self.model_config = model_config
vllm/entrypoints/llm.py
+10
@@ -159,6 +159,11 @@ class LLM:
159 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
160 # After positional args are removed, move this right below `model`
161 task: TaskOption = "auto",
162 + pooling_type: Optional[str] = None,
163 + pooling_norm: Optional[bool] = None,
164 + pooling_softmax: Optional[bool] = None,
165 + pooling_step_tag_id: Optional[int] = None,
166 + pooling_returned_token_ids: Optional[List[int]] = None,
167 **kwargs,
168 ) -> None:
169 '''
@@ -193,6 +198,11 @@ class LLM:
198 disable_custom_all_reduce=disable_custom_all_reduce,
199 disable_async_output_proc=disable_async_output_proc,
200 mm_processor_kwargs=mm_processor_kwargs,
201 + pooling_type=pooling_type,
202 + pooling_norm=pooling_norm,
203 + pooling_softmax=pooling_softmax,
204 + pooling_step_tag_id=pooling_step_tag_id,
205 + pooling_returned_token_ids=pooling_returned_token_ids,
206 **kwargs,
207 )
208 self.llm_engine = LLMEngine.from_engine_args(
vllm/model_executor/layers/pooler.py
+58 -4
@@ -1,8 +1,10 @@
1 from enum import IntEnum
2 +from typing import List, Optional
3
4 import torch
5 import torch.nn as nn
6
7 +from vllm.config import PoolerConfig
8 from vllm.model_executor.pooling_metadata import (PoolingMetadata,
9 PoolingTensors)
10 from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
@@ -13,6 +15,7 @@ class PoolingType(IntEnum):
15 LAST = 0
16 ALL = 1
17 CLS = 2
18 + STEP = 3
19
20
21 class Pooler(nn.Module):
@@ -28,15 +31,47 @@ class Pooler(nn.Module):
31 normalize: Whether to normalize the pooled data.
32 """
33
31 - def __init__(self,
32 - pooling_type: PoolingType,
33 - normalize: bool,
34 - softmax: bool = False):
34 + def __init__(
35 + self,
36 + pooling_type: PoolingType,
37 + normalize: bool,
38 + softmax: bool,
39 + step_tag_id: Optional[int] = None,
40 + returned_token_ids: Optional[List[int]] = None,
41 + ):
42 super().__init__()
43
44 self.pooling_type = pooling_type
45 self.normalize = normalize
46 self.softmax = softmax
47 + self.step_tag_id = step_tag_id
48 + self.returned_token_ids = returned_token_ids
49 +
50 + @classmethod
51 + def from_config_with_defaults(
52 + cls,
53 + pooler_config: PoolerConfig,
54 + pooling_type: PoolingType,
55 + normalize: bool,
56 + softmax: bool,
57 + step_tag_id: Optional[int] = None,
58 + returned_token_ids: Optional[List[int]] = None,
59 + ) -> Optional["Pooler"]:
60 + if pooler_config is None:
61 + return None
62 + return cls(
63 + pooling_type=PoolingType[pooler_config.pooling_type]
64 + if pooler_config.pooling_type is not None else pooling_type,
65 + normalize=pooler_config.pooling_norm
66 + if pooler_config.pooling_norm is not None else normalize,
67 + softmax=pooler_config.pooling_softmax
68 + if pooler_config.pooling_softmax is not None else softmax,
69 + step_tag_id=pooler_config.pooling_step_tag_id
70 + if pooler_config.pooling_step_tag_id is not None else step_tag_id,
71 + returned_token_ids=pooler_config.pooling_returned_token_ids
72 + if pooler_config.pooling_returned_token_ids is not None else
73 + returned_token_ids,
74 + )
75
76 def forward(
77 self,
@@ -62,6 +97,25 @@ class Pooler(nn.Module):
97 for prompt_len in prompt_lens:
98 pooled_data.append(hidden_states[offset:offset + prompt_len])
99 offset += prompt_len
100 + elif self.pooling_type == PoolingType.STEP:
101 + if self.returned_token_ids is not None and len(
102 + self.returned_token_ids) > 0:
103 + logits = hidden_states[:,
104 + self.returned_token_ids].softmax(dim=-1)
105 + else:
106 + logits = hidden_states.softmax(dim=-1)
107 + offset = 0
108 + pooled_data = []
109 + for prompt_len, seq_data_i in zip(
110 + prompt_lens, pooling_metadata.seq_data.values()):
111 + if self.step_tag_id is None:
112 + pooled_data.append(logits[offset:offset + prompt_len])
113 + else:
114 + step_idxs = torch.tensor(
115 + seq_data_i.prompt_token_ids) == self.step_tag_id
116 + pooled_data.append(logits[offset:offset +
117 + prompt_len][step_idxs])
118 + offset += prompt_len
119 else:
120 raise ValueError(f"Invalid pooling type: {self.pooling_type}")
121
vllm/model_executor/model_loader/loader.py
+10 -5
@@ -23,7 +23,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
23
24 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
25 LoRAConfig, ModelConfig, MultiModalConfig,
26 - ParallelConfig, SchedulerConfig)
26 + ParallelConfig, PoolerConfig, SchedulerConfig)
27 from vllm.distributed import (get_tensor_model_parallel_rank,
28 get_tensor_model_parallel_world_size)
29 from vllm.envs import VLLM_USE_MODELSCOPE
@@ -122,7 +122,8 @@ def _get_model_initialization_kwargs(
122 model_class: Type[nn.Module],
123 lora_config: Optional[LoRAConfig],
124 multimodal_config: Optional[MultiModalConfig],
125 - scheduler_config: Optional[SchedulerConfig] = None) -> Dict[str, Any]:
125 + scheduler_config: Optional[SchedulerConfig] = None,
126 + pooler_config: Optional[PoolerConfig] = None) -> Dict[str, Any]:
127 """Get extra kwargs for model initialization."""
128 extra_kwargs: Dict[str, Any] = {}
129
@@ -143,7 +144,8 @@ def _get_model_initialization_kwargs(
144
145 if has_inner_state(model_class) and scheduler_config:
146 extra_kwargs["scheduler_config"] = scheduler_config
146 -
147 + if pooler_config:
148 + extra_kwargs["pooler_config"] = pooler_config
149 return extra_kwargs
150
151
@@ -155,10 +157,12 @@ def build_model(model_class: Type[nn.Module],
157 lora_config: Optional[LoRAConfig],
158 multimodal_config: Optional[MultiModalConfig],
159 scheduler_config: Optional[SchedulerConfig],
158 - prefix: Optional[str] = None) -> nn.Module:
160 + prefix: Optional[str] = None,
161 + pooler_config: Optional[PoolerConfig] = None) -> nn.Module:
162 extra_kwargs = _get_model_initialization_kwargs(model_class, lora_config,
163 multimodal_config,
161 - scheduler_config)
164 + scheduler_config,
165 + pooler_config)
166 if prefix:
167 extra_kwargs["prefix"] = prefix
168
@@ -185,6 +189,7 @@ def _initialize_model(
189 lora_config=lora_config,
190 multimodal_config=model_config.multimodal_config,
191 scheduler_config=scheduler_config,
192 + pooler_config=model_config.pooler_config,
193 )
194
195
vllm/model_executor/models/bert.py
+7 -2
@@ -6,7 +6,7 @@ from transformers import BertConfig
6
7 from vllm.attention import Attention, AttentionMetadata, AttentionType
8 from vllm.attention.backends.xformers import XFormersImpl
9 -from vllm.config import CacheConfig
9 +from vllm.config import CacheConfig, PoolerConfig
10 from vllm.distributed import get_tensor_model_parallel_world_size
11 from vllm.model_executor.layers.activation import get_act_fn
12 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -387,10 +387,15 @@ class BertEmbeddingModel(nn.Module):
387 config: BertConfig,
388 cache_config: Optional[CacheConfig] = None,
389 quant_config: Optional[QuantizationConfig] = None,
390 + pooler_config: Optional[PoolerConfig] = None,
391 ) -> None:
392 super().__init__()
393 self.model = BertModel(config, cache_config, quant_config)
393 - self._pooler = Pooler(pooling_type=PoolingType.CLS, normalize=True)
394 + self._pooler = Pooler.from_config_with_defaults(
395 + pooler_config,
396 + pooling_type=PoolingType.CLS,
397 + normalize=True,
398 + softmax=False)
399
400 def forward(
401 self,
vllm/model_executor/models/gemma2.py
+7 -3
@@ -22,7 +22,7 @@ from transformers import Gemma2Config
22
23 from vllm.attention import Attention, AttentionMetadata
24 from vllm.compilation.decorators import support_torch_compile
25 -from vllm.config import CacheConfig, LoRAConfig
25 +from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
26 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
27 from vllm.logger import init_logger
28 from vllm.model_executor.layers.activation import GeluAndMul
@@ -473,13 +473,17 @@ class Gemma2EmbeddingModel(nn.Module, SupportsPP):
473
474 def __init__(
475 self,
476 + pooler_config: Optional[PoolerConfig] = None,
477 **kwargs,
478 ) -> None:
479 super().__init__()
480
481 self.model = Gemma2Model(**kwargs)
481 - self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
482 -
482 + self._pooler = Pooler.from_config_with_defaults(
483 + pooler_config,
484 + pooling_type=PoolingType.LAST,
485 + normalize=True,
486 + softmax=False)
487 self.make_empty_intermediate_tensors = (
488 self.model.make_empty_intermediate_tensors)
489
vllm/model_executor/models/llama.py
+21 -2
@@ -29,7 +29,7 @@ from transformers import LlamaConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 -from vllm.config import CacheConfig, LoRAConfig
32 +from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
35 from vllm.model_executor.layers.activation import SiluAndMul
@@ -502,6 +502,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
502 quant_config: Optional[QuantizationConfig] = None,
503 lora_config: Optional[LoRAConfig] = None,
504 prefix: str = "",
505 + pooler_config: Optional[PoolerConfig] = None,
506 ) -> None:
507 super().__init__()
508
@@ -543,6 +544,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
544 self.lm_head = PPMissingLayer()
545 self.make_empty_intermediate_tensors = (
546 self.model.make_empty_intermediate_tensors)
547 + self._pooler = Pooler.from_config_with_defaults(
548 + pooler_config,
549 + pooling_type=PoolingType.STEP,
550 + normalize=False,
551 + softmax=False)
552
553 def forward(
554 self,
@@ -565,6 +571,14 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
571 sampling_metadata)
572 return logits
573
574 + def pooler(
575 + self,
576 + hidden_states: torch.Tensor,
577 + pooling_metadata: PoolingMetadata,
578 + ) -> Optional[PoolerOutput]:
579 + logits = self.compute_logits(hidden_states, None)
580 + return self._pooler(logits, pooling_metadata)
581 +
582 def sample(self, logits: torch.Tensor,
583 sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]:
584 next_tokens = self.sampler(logits, sampling_metadata)
@@ -630,12 +644,17 @@ class LlamaEmbeddingModel(nn.Module, SupportsPP):
644
645 def __init__(
646 self,
647 + pooler_config: Optional[PoolerConfig] = None,
648 **kwargs,
649 ) -> None:
650 super().__init__()
651
652 self.model = LlamaModel(**kwargs)
638 - self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
653 + self._pooler = Pooler.from_config_with_defaults(
654 + pooler_config,
655 + pooling_type=PoolingType.LAST,
656 + normalize=True,
657 + softmax=False)
658 self.make_empty_intermediate_tensors = (
659 self.model.make_empty_intermediate_tensors)
660
vllm/model_executor/models/llava_next.py
+8 -4
@@ -11,7 +11,7 @@ from transformers.models.llava_next.modeling_llava_next import (
11 from typing_extensions import NotRequired
12
13 from vllm.attention import AttentionMetadata
14 -from vllm.config import CacheConfig, MultiModalConfig
14 +from vllm.config import CacheConfig, MultiModalConfig, PoolerConfig
15 from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
16 from vllm.model_executor.layers.pooler import Pooler, PoolingType
17 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -285,7 +285,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
285 config: LlavaNextConfig,
286 multimodal_config: MultiModalConfig,
287 cache_config: Optional[CacheConfig] = None,
288 - quant_config: Optional[QuantizationConfig] = None) -> None:
288 + quant_config: Optional[QuantizationConfig] = None,
289 + pooler_config: Optional[PoolerConfig] = None) -> None:
290 super().__init__()
291
292 self.config = config
@@ -312,8 +313,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
313
314 # The same model class supports both language generation and embedding
315 # because the architecture name is the same
315 - self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
316 -
316 + self._pooler = Pooler.from_config_with_defaults(
317 + pooler_config,
318 + pooling_type=PoolingType.LAST,
319 + normalize=True,
320 + softmax=False)
321 self.make_empty_intermediate_tensors = (
322 self.language_model.make_empty_intermediate_tensors)
323
vllm/model_executor/models/phi3v.py
+9 -4
@@ -26,7 +26,8 @@ from PIL import Image
26 from transformers import CLIPVisionConfig, PretrainedConfig
27
28 from vllm.attention import AttentionMetadata
29 -from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
29 +from vllm.config import (CacheConfig, ModelConfig, MultiModalConfig,
30 + PoolerConfig)
31 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
32 token_inputs)
33 from vllm.logger import init_logger
@@ -530,7 +531,8 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
531 config: PretrainedConfig,
532 multimodal_config: MultiModalConfig,
533 cache_config: Optional[CacheConfig] = None,
533 - quant_config: Optional[QuantizationConfig] = None) -> None:
534 + quant_config: Optional[QuantizationConfig] = None,
535 + pooler_config: Optional[PoolerConfig] = None) -> None:
536 super().__init__()
537
538 self.config = config
@@ -556,8 +558,11 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
558
559 # The same model class supports both language generation and embedding
560 # because the architecture name is the same
559 - self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
560 -
561 + self._pooler = Pooler.from_config_with_defaults(
562 + pooler_config,
563 + pooling_type=PoolingType.LAST,
564 + normalize=True,
565 + softmax=False)
566 self.make_empty_intermediate_tensors = (
567 self.language_model.make_empty_intermediate_tensors)
568
vllm/model_executor/models/qwen2_cls.py
+7 -4
@@ -12,7 +12,7 @@ from torch import nn
12 from transformers import Qwen2Config
13
14 from vllm.attention import AttentionMetadata
15 -from vllm.config import CacheConfig, LoRAConfig
15 +from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
16 from vllm.model_executor.layers.linear import RowParallelLinear
17 from vllm.model_executor.layers.pooler import Pooler, PoolingType
18 from vllm.model_executor.layers.quantization.base_config import (
@@ -53,6 +53,7 @@ class Qwen2ForSequenceClassification(nn.Module):
53 cache_config: Optional[CacheConfig] = None,
54 quant_config: Optional[QuantizationConfig] = None,
55 lora_config: Optional[LoRAConfig] = None,
56 + pooler_config: Optional[PoolerConfig] = None,
57 ) -> None:
58 # TODO (@robertgshaw2): see if this can be moved out
59 if (cache_config.sliding_window is not None
@@ -77,9 +78,11 @@ class Qwen2ForSequenceClassification(nn.Module):
78 self.score = RowParallelLinear(config.hidden_size,
79 config.num_labels,
80 quant_config=quant_config)
80 - self._pooler = Pooler(pooling_type=PoolingType.LAST,
81 - normalize=False,
82 - softmax=True)
81 + self._pooler = Pooler.from_config_with_defaults(
82 + pooler_config,
83 + pooling_type=PoolingType.LAST,
84 + normalize=False,
85 + softmax=True)
86
87 def forward(
88 self,
vllm/model_executor/models/qwen2_rm.py
+7 -3
@@ -11,7 +11,7 @@ from torch import nn
11 from transformers import Qwen2Config
12
13 from vllm.attention import AttentionMetadata
14 -from vllm.config import CacheConfig, LoRAConfig
14 +from vllm.config import CacheConfig, LoRAConfig, PoolerConfig
15 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 RowParallelLinear)
17 from vllm.model_executor.layers.pooler import Pooler, PoolingType
@@ -64,6 +64,7 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
64 cache_config: Optional[CacheConfig] = None,
65 quant_config: Optional[QuantizationConfig] = None,
66 lora_config: Optional[LoRAConfig] = None,
67 + pooler_config: Optional[PoolerConfig] = None,
68 ) -> None:
69 # TODO (@robertgshaw2): see if this can be moved out
70 if (cache_config.sliding_window is not None
@@ -93,8 +94,11 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
94 RowParallelLinear(config.hidden_size, 1,
95 quant_config=quant_config),
96 )
96 - self._pooler = Pooler(pooling_type=PoolingType.ALL, normalize=False)
97 -
97 + self._pooler = Pooler.from_config_with_defaults(
98 + pooler_config,
99 + pooling_type=PoolingType.ALL,
100 + normalize=False,
101 + softmax=False)
102 self.make_empty_intermediate_tensors = (
103 self.model.make_empty_intermediate_tensors)
104
vllm/model_executor/models/registry.py
+16
@@ -100,11 +100,27 @@ _EMBEDDING_MODELS = {
100 "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
101 "Qwen2ForSequenceClassification": (
102 "qwen2_cls", "Qwen2ForSequenceClassification"),
103 + "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
104 + "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
105 + "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
106 # [Multimodal]
107 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
108 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
109 }
110
111 +def add_embedding_models(base_models, embedding_models):
112 + with_pooler_method_models = {}
113 + embedding_models_name = embedding_models.keys()
114 + for name, (path, arch) in base_models.items():
115 + if arch in embedding_models_name:
116 + with_pooler_method_models[name] = (path, arch)
117 + return with_pooler_method_models
118 +
119 +_EMBEDDING_MODELS = {
120 + **add_embedding_models(_TEXT_GENERATION_MODELS, _EMBEDDING_MODELS),
121 + **_EMBEDDING_MODELS,
122 +}
123 +
124 _MULTIMODAL_MODELS = {
125 # [Decoder-only]
126 "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),