@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 343f8e0905
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

Support `BERTModel` (first `encoder-only` embedding model) (#9056)

Signed-off-by: Max de Bayser <maxdebayser@gmail.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Andrew Feldman <afeldman@neuralmagic.com> Co-authored-by: afeldman-nm <156691304+afeldman-nm@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: laishzh <laishengzhang@gmail.com> Co-authored-by: Max de Bayser <maxdebayser@gmail.com> Co-authored-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Robert Shaw committed Oct 17, 2024 at 19:21 UTC 343f8e09055b67a000023fc9ae7254905090de9e
6 files changed +497 -15
tests/models/embedding/language/test_embedding.py
+12 -2
@@ -6,21 +6,31 @@ import pytest
6
7 from ..utils import check_embeddings_close
8
9 +# Model, Guard
10 MODELS = [
11 "intfloat/e5-mistral-7b-instruct",
12 + "BAAI/bge-base-en-v1.5",
13 "BAAI/bge-multilingual-gemma2",
14 ]
15
16 +ENCODER_ONLY = [
17 + "BAAI/bge-base-en-v1.5",
18 +]
19 +
20
21 @pytest.mark.parametrize("model", MODELS)
22 @pytest.mark.parametrize("dtype", ["half"])
23 def test_models(
24 + monkeypatch,
25 hf_runner,
26 vllm_runner,
27 example_prompts,
21 - model: str,
28 + model,
29 dtype: str,
30 ) -> None:
31 + if model in ENCODER_ONLY:
32 + monkeypatch.setenv("VLLM_ATTENTION_BACKEND", "XFORMERS")
33 +
34 # The example_prompts has ending "\n", for example:
35 # "Write a short story about a robot that dreams for the first time.\n"
36 # sentence_transformers will strip the input texts, see:
@@ -33,7 +43,7 @@ def test_models(
43 is_sentence_transformer=True) as hf_model:
44 hf_outputs = hf_model.encode(example_prompts)
45
36 - with vllm_runner(model, dtype=dtype) as vllm_model:
46 + with vllm_runner(model, dtype=dtype, max_model_len=None) as vllm_model:
47 vllm_outputs = vllm_model.encode(example_prompts)
48
49 check_embeddings_close(
vllm/attention/backends/abstract.py
+5 -2
@@ -15,8 +15,11 @@ if TYPE_CHECKING:
15
16 class AttentionType(Enum):
17 DECODER = auto() # Decoder attention between previous layer Q/K/V
18 - ENCODER = auto() # Encoder attention between previous layer Q/K/V
19 - ENCODER_DECODER = auto() # Attention between dec. Q and enc. K/V
18 + ENCODER = auto(
19 + ) # Encoder attention between previous layer Q/K/V for encoder-decoder
20 + ENCODER_ONLY = auto() # Encoder attention between previous layer Q/K/V
21 + ENCODER_DECODER = auto(
22 + ) # Attention between dec. Q and enc. K/V for encoder-decoder
23
24
25 class AttentionBackend(ABC):
vllm/attention/backends/xformers.py
+50 -9
@@ -287,13 +287,15 @@ def _get_attn_bias(
287 * Appropriate attention bias value given the attention type
288 '''
289
290 - if attn_type == AttentionType.DECODER:
290 + if (attn_type == AttentionType.DECODER
291 + or attn_type == AttentionType.ENCODER_ONLY):
292 return attn_metadata.attn_bias
293 elif attn_type == AttentionType.ENCODER:
294 return attn_metadata.encoder_attn_bias
294 - else:
295 - # attn_type == AttentionType.ENCODER_DECODER
295 + elif attn_type == AttentionType.ENCODER_DECODER:
296 return attn_metadata.cross_attn_bias
297 + else:
298 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
299
300
301 def _set_attn_bias(
@@ -313,7 +315,8 @@ def _set_attn_bias(
315 encoder/decoder cross-attention
316 '''
317
316 - if attn_type == AttentionType.DECODER:
318 + if (attn_type == AttentionType.DECODER
319 + or attn_type == AttentionType.ENCODER_ONLY):
320 attn_metadata.attn_bias = attn_bias
321 elif attn_type == AttentionType.ENCODER:
322 attn_metadata.encoder_attn_bias = attn_bias
@@ -371,6 +374,12 @@ def _get_seq_len_block_table_args(
374 # No block tables associated with encoder attention
375 return (attn_metadata.encoder_seq_lens_tensor,
376 attn_metadata.max_encoder_seq_len, None)
377 + elif attn_type == AttentionType.ENCODER_ONLY:
378 + assert is_prompt, "Should not have decode for encoder only model."
379 +
380 + # No block tables associated with encoder attention
381 + return (attn_metadata.seq_lens_tensor,
382 + attn_metadata.max_prefill_seq_len, None)
383 else:
384 raise AttributeError(f"Invalid attention type {str(attn_type)}")
385
@@ -479,7 +488,10 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
488 * ENCODER: no KV caching; pass encoder sequence
489 attributes (encoder_seq_lens/encoder_seq_lens_tensor/
490 max_encoder_seq_len) to kernel, in lieu of decoder
482 - sequence attributes (seq_lens/seq_lens_tensor/max_seq_len)
491 + sequence attributes (seq_lens/seq_lens_tensor/max_seq_len).
492 + Used for encoder branch of encoder-decoder models.
493 + * ENCODER_ONLY: no kv_caching, uses the normal attention
494 + attributes (seq_lens/seq_lens_tensor/max_seq_len).
495 * ENCODER_DECODER: cross-attention behavior;
496 use cross-attention block table for caching KVs derived
497 from encoder hidden states; since KV sequence lengths
@@ -509,6 +521,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
521 and (not attn_metadata.is_all_encoder_attn_metadata_set)):
522 raise AttributeError("Encoder attention requires setting "
523 "encoder metadata attributes.")
524 +
525 elif (attn_type == AttentionType.ENCODER_DECODER
526 and (not attn_metadata.is_all_cross_attn_metadata_set)):
527 raise AttributeError("Encoder/decoder cross-attention "
@@ -609,6 +622,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
622 assert out.shape == output[:num_prefill_tokens].shape
623 output[:num_prefill_tokens] = out
624 else:
625 + assert attn_type != AttentionType.ENCODER_ONLY, (
626 + "Encoder-only models should not have prefix attention.")
627
628 assert prefill_meta.query_start_loc is not None
629 assert prefill_meta.max_query_len is not None
@@ -638,6 +653,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
653 output[:num_prefill_tokens] = out
654
655 if decode_meta := attn_metadata.decode_metadata:
656 + assert attn_type != AttentionType.ENCODER_ONLY, (
657 + "Encoder-only models should not have decode metadata.")
658
659 (
660 seq_lens_arg,
@@ -703,36 +720,60 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
720 None, :].expand(value.shape[0], self.num_kv_heads,
721 self.num_queries_per_kv,
722 value.shape[-1])
723 +
724 # Set attention bias if not provided. This typically happens at
725 # the very attention layer of every iteration.
726 # FIXME(woosuk): This is a hack.
727 attn_bias = _get_attn_bias(attn_metadata, attn_type)
728 if attn_bias is None:
729 if self.alibi_slopes is None:
730 +
731 + # Cross attention block of decoder branch of encoder-decoder
732 + # model uses seq_lens for dec / encoder_seq_lens for enc
733 if (attn_type == AttentionType.ENCODER_DECODER):
734 assert attn_metadata.seq_lens is not None
735 assert attn_metadata.encoder_seq_lens is not None
736
716 - # Default enc/dec cross-attention mask is non-causal
737 + # Cross-attention mask is non-causal
738 attn_bias = BlockDiagonalMask.from_seqlens(
739 attn_metadata.seq_lens, attn_metadata.encoder_seq_lens)
740 +
741 + # Encoder branch of encoder-decoder model uses
742 + # attn_metadata.encoder_seq_lens
743 elif attn_type == AttentionType.ENCODER:
744 +
745 assert attn_metadata.encoder_seq_lens is not None
746
722 - # Default encoder self-attention mask is non-causal
747 + # Encoder self-attention mask is non-causal
748 attn_bias = BlockDiagonalMask.from_seqlens(
749 attn_metadata.encoder_seq_lens)
725 - else:
750 +
751 + # Self-attention block of encoder-only model just
752 + # uses the seq_lens directly.
753 + elif attn_type == AttentionType.ENCODER_ONLY:
754 assert attn_metadata.seq_lens is not None
755
728 - # Default decoder self-attention mask is causal
756 + # Encoder self-attention mask is non-causal
757 + attn_bias = BlockDiagonalMask.from_seqlens(
758 + attn_metadata.seq_lens)
759 +
760 + # Self-attention block of decoder branch just
761 + # uses the seq_lens directly
762 + elif attn_type == AttentionType.DECODER:
763 + assert attn_metadata.seq_lens is not None
764 +
765 + # Decoder self-attention mask is causal
766 attn_bias = BlockDiagonalCausalMask.from_seqlens(
767 attn_metadata.seq_lens)
768 + else:
769 + raise ValueError("Unknown AttentionType: %s", attn_type)
770 +
771 if self.sliding_window is not None:
772 attn_bias = attn_bias.make_local_attention(
773 self.sliding_window)
774 attn_bias = [attn_bias]
775 else:
776 + assert attn_type == AttentionType.DECODER
777 assert attn_metadata.seq_lens is not None
778 attn_bias = _make_alibi_bias(self.alibi_slopes,
779 self.num_kv_heads, query.dtype,
vllm/model_executor/layers/pooler.py
+10 -2
@@ -12,6 +12,7 @@ class PoolingType(IntEnum):
12 """Enumeration for different types of pooling methods."""
13 LAST = 0
14 ALL = 1
15 + CLS = 2
16
17
18 class Pooler(nn.Module):
@@ -23,12 +24,13 @@ class Pooler(nn.Module):
24 3. Returns structured results as `PoolerOutput`.
25
26 Attributes:
26 - pooling_type: The type of pooling to use (LAST, AVERAGE, MAX).
27 + pooling_type: The type of pooling to use (LAST, ALL, CLS).
28 normalize: Whether to normalize the pooled data.
29 """
30
31 def __init__(self, pooling_type: PoolingType, normalize: bool):
32 super().__init__()
33 +
34 self.pooling_type = pooling_type
35 self.normalize = normalize
36
@@ -38,10 +40,16 @@ class Pooler(nn.Module):
40 pooling_metadata: PoolingMetadata,
41 ) -> PoolerOutput:
42 """Pools specific information from hidden states based on metadata."""
43 +
44 prompt_lens = PoolingTensors.from_pooling_metadata(
45 pooling_metadata, hidden_states.device).prompt_lens
46
44 - if self.pooling_type == PoolingType.LAST:
47 + if self.pooling_type is PoolingType.CLS:
48 + first_token_flat_indices = torch.zeros_like(prompt_lens)
49 + first_token_flat_indices[1:] += torch.cumsum(prompt_lens,
50 + dim=0)[:-1]
51 + pooled_data = hidden_states[first_token_flat_indices]
52 + elif self.pooling_type == PoolingType.LAST:
53 last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
54 pooled_data = hidden_states[last_token_flat_indices]
55 elif self.pooling_type == PoolingType.ALL:
vllm/model_executor/models/bert.py new
+419
@@ -0,0 +1,419 @@
1 +from typing import Iterable, List, Optional, Tuple
2 +
3 +import torch
4 +from torch import nn
5 +from transformers import BertConfig
6 +
7 +from vllm.attention import Attention, AttentionMetadata, AttentionType
8 +from vllm.attention.backends.xformers import XFormersImpl
9 +from vllm.config import CacheConfig
10 +from vllm.distributed import get_tensor_model_parallel_world_size
11 +from vllm.model_executor.layers.activation import get_act_fn
12 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13 + QKVParallelLinear,
14 + RowParallelLinear)
15 +from vllm.model_executor.layers.pooler import Pooler, PoolingType
16 +from vllm.model_executor.layers.quantization.base_config import (
17 + QuantizationConfig)
18 +from vllm.model_executor.layers.vocab_parallel_embedding import (
19 + VocabParallelEmbedding)
20 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 +from vllm.model_executor.pooling_metadata import PoolingMetadata
22 +from vllm.sequence import IntermediateTensors, PoolerOutput
23 +
24 +
25 +class BertEmbedding(nn.Module):
26 +
27 + def __init__(self, config: BertConfig):
28 +
29 + super().__init__()
30 + self.size = config.hidden_size
31 + self.word_embeddings = VocabParallelEmbedding(config.vocab_size,
32 + config.hidden_size)
33 + self.position_embeddings = VocabParallelEmbedding(
34 + config.max_position_embeddings, config.hidden_size)
35 + self.token_type_embeddings = VocabParallelEmbedding(
36 + config.type_vocab_size, config.hidden_size)
37 + self.LayerNorm = nn.LayerNorm(config.hidden_size,
38 + eps=config.layer_norm_eps)
39 + self.position_ids = nn.Parameter(
40 + torch.empty((1, config.max_position_embeddings)), )
41 +
42 + self.position_embedding_type = config.position_embedding_type
43 + if self.position_embedding_type != "absolute":
44 + raise ValueError("Only 'absolute' position_embedding_type" +
45 + " is supported")
46 +
47 + def forward(
48 + self,
49 + input_ids: torch.Tensor,
50 + position_ids: Optional[torch.Tensor] = None,
51 + ) -> torch.Tensor:
52 + input_shape = input_ids.size()
53 +
54 + # Input embeddings.
55 + inputs_embeds = self.word_embeddings(input_ids)
56 +
57 + # Position embeddings.
58 + position_embeddings = self.position_embeddings(position_ids)
59 +
60 + # Token type embeddings. (TODO: move off hotpath?)
61 + token_type_embeddings = self.token_type_embeddings(
62 + torch.zeros(input_shape,
63 + dtype=torch.long,
64 + device=inputs_embeds.device))
65 +
66 + embeddings = inputs_embeds + token_type_embeddings + position_embeddings
67 + embeddings = self.LayerNorm(embeddings)
68 + return embeddings
69 +
70 +
71 +class BertEncoder(nn.Module):
72 +
73 + def __init__(self,
74 + config: BertConfig,
75 + cache_config: Optional[CacheConfig] = None,
76 + quant_config: Optional[QuantizationConfig] = None,
77 + prefix: str = ""):
78 + super().__init__()
79 + self.layer = nn.ModuleList([
80 + BertLayer(config=config,
81 + cache_config=cache_config,
82 + quant_config=quant_config,
83 + prefix=f"{prefix}.layer.{layer_idx}")
84 + for layer_idx in range(config.num_hidden_layers)
85 + ])
86 +
87 + def forward(
88 + self,
89 + hidden_states: torch.Tensor,
90 + kv_caches: List[torch.Tensor],
91 + attn_metadata: AttentionMetadata,
92 + ) -> torch.Tensor:
93 + for i in range(len(self.layer)):
94 + layer = self.layer[i]
95 + hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
96 + return hidden_states
97 +
98 +
99 +class BertLayer(nn.Module):
100 +
101 + def __init__(self,
102 + config: BertConfig,
103 + cache_config: Optional[CacheConfig] = None,
104 + quant_config: Optional[QuantizationConfig] = None,
105 + prefix: str = ""):
106 + super().__init__()
107 +
108 + self.attention = BertAttention(
109 + hidden_size=config.hidden_size,
110 + num_attention_heads=config.num_attention_heads,
111 + layer_norm_eps=config.layer_norm_eps,
112 + cache_config=cache_config,
113 + quant_config=quant_config,
114 + prefix=f"{prefix}.attention")
115 +
116 + self.intermediate = BertIntermediate(
117 + hidden_size=config.hidden_size,
118 + intermediate_size=config.intermediate_size,
119 + hidden_act=config.hidden_act,
120 + quant_config=quant_config,
121 + prefix=f"{prefix}.intermediate")
122 +
123 + self.output = BertOutput(hidden_size=config.hidden_size,
124 + intermediate_size=config.intermediate_size,
125 + layer_norm_eps=config.layer_norm_eps,
126 + quant_config=quant_config,
127 + prefix=f"{prefix}.output")
128 +
129 + def forward(
130 + self,
131 + hidden_states: torch.Tensor,
132 + kv_cache: Optional[torch.Tensor],
133 + attn_metadata: AttentionMetadata,
134 + ):
135 + attn_output = self.attention(hidden_states, kv_cache, attn_metadata)
136 + intermediate_output = self.intermediate(attn_output)
137 + output = self.output(intermediate_output, attn_output)
138 + return output
139 +
140 +
141 +class BertAttention(nn.Module):
142 +
143 + def __init__(
144 + self,
145 + hidden_size: int,
146 + num_attention_heads: int,
147 + layer_norm_eps: float,
148 + cache_config: Optional[CacheConfig] = None,
149 + quant_config: Optional[QuantizationConfig] = None,
150 + prefix: str = "",
151 + ):
152 + super().__init__()
153 +
154 + self.self = BertSelfAttention(hidden_size=hidden_size,
155 + num_attention_heads=num_attention_heads,
156 + cache_config=cache_config,
157 + quant_config=quant_config,
158 + prefix=f"{prefix}.output")
159 +
160 + self.output = BertSelfOutput(hidden_size=hidden_size,
161 + layer_norm_eps=layer_norm_eps,
162 + quant_config=quant_config,
163 + prefix=f"{prefix}.output")
164 +
165 + def forward(
166 + self,
167 + hidden_states: torch.Tensor,
168 + kv_cache: torch.Tensor,
169 + attn_metadata: AttentionMetadata,
170 + ) -> torch.Tensor:
171 + self_output = self.self(hidden_states, kv_cache, attn_metadata)
172 + return self.output(self_output, hidden_states)
173 +
174 +
175 +class BertSelfAttention(nn.Module):
176 +
177 + def __init__(
178 + self,
179 + hidden_size: int,
180 + num_attention_heads: int,
181 + cache_config: Optional[CacheConfig] = None,
182 + quant_config: Optional[QuantizationConfig] = None,
183 + prefix: str = "",
184 + ):
185 + super().__init__()
186 + self.hidden_size = hidden_size
187 + tp_size = get_tensor_model_parallel_world_size()
188 +
189 + self.total_num_heads = num_attention_heads
190 + assert self.total_num_heads % tp_size == 0
191 +
192 + self.num_heads = self.total_num_heads // tp_size
193 + self.total_num_kv_heads = self.total_num_heads
194 + self.head_dim = self.hidden_size // self.total_num_heads
195 + assert self.head_dim * self.total_num_heads == self.hidden_size
196 +
197 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
198 +
199 + self.q_size = self.num_heads * self.head_dim
200 + self.kv_size = self.num_kv_heads * self.head_dim
201 + self.scaling = self.head_dim**-0.5
202 + self.qkv_proj = QKVParallelLinear(
203 + hidden_size=self.hidden_size,
204 + head_size=self.head_dim,
205 + total_num_heads=self.total_num_heads,
206 + total_num_kv_heads=self.total_num_kv_heads,
207 + bias=True,
208 + quant_config=quant_config,
209 + prefix=f"{prefix}.qkv_proj")
210 +
211 + self.attn = Attention(num_heads=self.num_heads,
212 + head_size=self.head_dim,
213 + scale=self.scaling,
214 + num_kv_heads=self.num_kv_heads,
215 + cache_config=cache_config,
216 + quant_config=quant_config,
217 + prefix=f"{prefix}.attn")
218 +
219 + if not isinstance(self.attn.impl, XFormersImpl):
220 + raise ValueError(
221 + "Encoder-only models currently require XFORMERS attention "
222 + "backend. Set VLLM_ATTENTION_BACKEND=XFORMERS to use BERT.")
223 +
224 + def forward(
225 + self,
226 + hidden_states: torch.Tensor,
227 + kv_cache: torch.Tensor,
228 + attn_metadata: AttentionMetadata,
229 + ) -> torch.Tensor:
230 + qkv, _ = self.qkv_proj(hidden_states)
231 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
232 + output = self.attn(q,
233 + k,
234 + v,
235 + kv_cache,
236 + attn_metadata,
237 + attn_type=AttentionType.ENCODER_ONLY)
238 + return output
239 +
240 +
241 +class BertSelfOutput(nn.Module):
242 +
243 + def __init__(self,
244 + hidden_size: int,
245 + layer_norm_eps: float,
246 + quant_config: Optional[QuantizationConfig] = None,
247 + prefix: str = ""):
248 + super().__init__()
249 + self.dense = RowParallelLinear(input_size=hidden_size,
250 + output_size=hidden_size,
251 + bias=True,
252 + quant_config=quant_config,
253 + prefix=f"{prefix}.dense")
254 + self.LayerNorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps)
255 +
256 + def forward(self, hidden_states: torch.Tensor,
257 + input_tensor: torch.Tensor) -> torch.Tensor:
258 + hidden_states, _ = self.dense(hidden_states)
259 + hidden_states = self.LayerNorm(hidden_states + input_tensor)
260 + return hidden_states
261 +
262 +
263 +class BertIntermediate(nn.Module):
264 +
265 + def __init__(self,
266 + hidden_size: int,
267 + intermediate_size: int,
268 + hidden_act: str,
269 + quant_config: Optional[QuantizationConfig] = None,
270 + prefix: str = ""):
271 + super().__init__()
272 + self.dense = ColumnParallelLinear(input_size=hidden_size,
273 + output_size=intermediate_size,
274 + bias=True,
275 + quant_config=quant_config,
276 + prefix=f"{prefix}.dense")
277 + self.intermediate_act_fn = get_act_fn(hidden_act)
278 +
279 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
280 + hidden_states, _ = self.dense(hidden_states)
281 + hidden_states = self.intermediate_act_fn(hidden_states)
282 + return hidden_states
283 +
284 +
285 +class BertOutput(nn.Module):
286 +
287 + def __init__(self,
288 + hidden_size: int,
289 + intermediate_size: int,
290 + layer_norm_eps: float,
291 + quant_config: Optional[QuantizationConfig] = None,
292 + prefix: str = ""):
293 + super().__init__()
294 +
295 + self.dense = RowParallelLinear(input_size=intermediate_size,
296 + output_size=hidden_size,
297 + bias=True,
298 + quant_config=quant_config,
299 + prefix=f"{prefix}.dense")
300 +
301 + self.LayerNorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps)
302 +
303 + def forward(self, hidden_states: torch.Tensor,
304 + input_tensor: torch.Tensor) -> torch.Tensor:
305 + hidden_states, _ = self.dense(hidden_states)
306 + hidden_states = self.LayerNorm(hidden_states + input_tensor)
307 + return hidden_states
308 +
309 +
310 +class BertModel(nn.Module):
311 +
312 + def __init__(self,
313 + config: BertConfig,
314 + cache_config: Optional[CacheConfig] = None,
315 + quant_config: Optional[QuantizationConfig] = None,
316 + prefix: str = ""):
317 + super().__init__()
318 + self.embeddings = BertEmbedding(config)
319 + self.encoder = BertEncoder(config,
320 + cache_config,
321 + quant_config,
322 + prefix=f"{prefix}.encoder")
323 +
324 + def forward(
325 + self,
326 + input_ids: torch.Tensor,
327 + position_ids: torch.Tensor,
328 + kv_caches: List[torch.Tensor],
329 + attn_metadata: AttentionMetadata,
330 + intermediate_tensors: Optional[IntermediateTensors] = None,
331 + inputs_embeds: Optional[torch.Tensor] = None,
332 + ) -> torch.Tensor:
333 + if inputs_embeds is not None:
334 + hidden_states = inputs_embeds
335 + else:
336 + hidden_states = self.embeddings(input_ids=input_ids,
337 + position_ids=position_ids)
338 +
339 + return self.encoder(hidden_states, kv_caches, attn_metadata)
340 +
341 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
342 + stacked_params_mapping = [
343 + # (param_name, shard_name, shard_id)
344 + ("qkv_proj", "query", "q"),
345 + ("qkv_proj", "key", "k"),
346 + ("qkv_proj", "value", "v"),
347 + ]
348 +
349 + params_dict = dict(self.named_parameters())
350 + for name, loaded_weight in weights:
351 + if "pooler" in name:
352 + continue
353 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
354 + if weight_name not in name:
355 + continue
356 + name = name.replace(weight_name, param_name)
357 + # Skip loading extra bias for GPTQ models.
358 + if name.endswith(".bias") and name not in params_dict:
359 + continue
360 + param = params_dict[name]
361 + weight_loader = param.weight_loader
362 + weight_loader(param, loaded_weight, shard_id)
363 + break
364 + else:
365 + # Skip loading extra bias for GPTQ models.
366 + if name.endswith(".bias") and name not in params_dict:
367 + continue
368 + param = params_dict[name]
369 + weight_loader = getattr(param, "weight_loader",
370 + default_weight_loader)
371 + weight_loader(param, loaded_weight)
372 +
373 +
374 +class BertEmbeddingModel(nn.Module):
375 + """A model that uses Bert to provide embedding functionalities.
376 +
377 + This class encapsulates the BertModel and provides an interface for
378 + embedding operations and customized pooling functions.
379 +
380 + Attributes:
381 + model: An instance of BertModel used for forward operations.
382 + _pooler: An instance of Pooler used for pooling operations.
383 + """
384 +
385 + def __init__(
386 + self,
387 + config: BertConfig,
388 + cache_config: Optional[CacheConfig] = None,
389 + quant_config: Optional[QuantizationConfig] = None,
390 + ) -> None:
391 + super().__init__()
392 + self.model = BertModel(config, cache_config, quant_config)
393 + self._pooler = Pooler(pooling_type=PoolingType.CLS, normalize=True)
394 +
395 + def forward(
396 + self,
397 + input_ids: Optional[torch.Tensor],
398 + positions: torch.Tensor,
399 + kv_caches: List[torch.Tensor],
400 + attn_metadata: AttentionMetadata,
401 + intermediate_tensors: Optional[IntermediateTensors] = None,
402 + inputs_embeds: Optional[torch.Tensor] = None,
403 + ) -> torch.Tensor:
404 + return self.model(input_ids=input_ids,
405 + position_ids=positions,
406 + kv_caches=kv_caches,
407 + inputs_embeds=inputs_embeds,
408 + intermediate_tensors=intermediate_tensors,
409 + attn_metadata=attn_metadata)
410 +
411 + def pooler(
412 + self,
413 + hidden_states: torch.Tensor,
414 + pooling_metadata: PoolingMetadata,
415 + ) -> Optional[PoolerOutput]:
416 + return self._pooler(hidden_states, pooling_metadata)
417 +
418 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
419 + self.model.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -87,6 +87,7 @@ _TEXT_GENERATION_MODELS = {
87
88 _EMBEDDING_MODELS = {
89 # [Text-only]
90 + "BertModel": ("bert", "BertEmbeddingModel"),
91 "Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
92 "MistralModel": ("llama", "LlamaEmbeddingModel"),
93 "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),