A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
Support `BERTModel` (first `encoder-only` embedding model) (#9056)
Signed-off-by: Max de Bayser <maxdebayser@gmail.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Andrew Feldman <afeldman@neuralmagic.com> Co-authored-by: afeldman-nm <156691304+afeldman-nm@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: laishzh <laishengzhang@gmail.com> Co-authored-by: Max de Bayser <maxdebayser@gmail.com> Co-authored-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Robert Shaw committed
Oct 17, 2024 at 19:21 UTC
343f8e09055b67a000023fc9ae7254905090de9e
6 files changed
+497
-15
tests/models/embedding/language/test_embedding.py
+12
-2
@@ -6,21 +6,31 @@ import pytest
6
7
from ..utils import check_embeddings_close
8
9
+# Model, Guard
10
MODELS = [
11
"intfloat/e5-mistral-7b-instruct",
12
+ "BAAI/bge-base-en-v1.5",
13
"BAAI/bge-multilingual-gemma2",
14
]
15
16
+ENCODER_ONLY = [
17
+ "BAAI/bge-base-en-v1.5",
18
+]
19
+
20
21
@pytest.mark.parametrize("model", MODELS)
22
@pytest.mark.parametrize("dtype", ["half"])
23
def test_models(
24
+ monkeypatch,
25
hf_runner,
26
vllm_runner,
27
example_prompts,
21
- model: str,
28
+ model,
29
dtype: str,
30
) -> None:
31
+ if model in ENCODER_ONLY:
32
+ monkeypatch.setenv("VLLM_ATTENTION_BACKEND", "XFORMERS")
33
+
34
# The example_prompts has ending "\n", for example:
35
# "Write a short story about a robot that dreams for the first time.\n"
36
# sentence_transformers will strip the input texts, see:
@@ -33,7 +43,7 @@ def test_models(
43
is_sentence_transformer=True) as hf_model:
44
hf_outputs = hf_model.encode(example_prompts)
45
36
- with vllm_runner(model, dtype=dtype) as vllm_model:
46
+ with vllm_runner(model, dtype=dtype, max_model_len=None) as vllm_model:
47
vllm_outputs = vllm_model.encode(example_prompts)
48
49
check_embeddings_close(
vllm/attention/backends/abstract.py
+5
-2
@@ -15,8 +15,11 @@ if TYPE_CHECKING:
15
16
class AttentionType(Enum):
17
DECODER = auto() # Decoder attention between previous layer Q/K/V
18
- ENCODER = auto() # Encoder attention between previous layer Q/K/V
19
- ENCODER_DECODER = auto() # Attention between dec. Q and enc. K/V
18
+ ENCODER = auto(
19
+ ) # Encoder attention between previous layer Q/K/V for encoder-decoder
20
+ ENCODER_ONLY = auto() # Encoder attention between previous layer Q/K/V
21
+ ENCODER_DECODER = auto(
22
+ ) # Attention between dec. Q and enc. K/V for encoder-decoder
23
24
25
class AttentionBackend(ABC):
vllm/attention/backends/xformers.py
+50
-9
@@ -287,13 +287,15 @@ def _get_attn_bias(
287
* Appropriate attention bias value given the attention type
288
'''
289
290
- if attn_type == AttentionType.DECODER:
290
+ if (attn_type == AttentionType.DECODER
291
+ or attn_type == AttentionType.ENCODER_ONLY):
292
return attn_metadata.attn_bias
293
elif attn_type == AttentionType.ENCODER:
294
return attn_metadata.encoder_attn_bias
294
- else:
295
- # attn_type == AttentionType.ENCODER_DECODER
295
+ elif attn_type == AttentionType.ENCODER_DECODER:
296
return attn_metadata.cross_attn_bias
297
+ else:
298
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
299
300
301
def _set_attn_bias(
@@ -313,7 +315,8 @@ def _set_attn_bias(
315
encoder/decoder cross-attention
316
'''
317
316
- if attn_type == AttentionType.DECODER:
318
+ if (attn_type == AttentionType.DECODER
319
+ or attn_type == AttentionType.ENCODER_ONLY):
320
attn_metadata.attn_bias = attn_bias
321
elif attn_type == AttentionType.ENCODER:
322
attn_metadata.encoder_attn_bias = attn_bias
@@ -371,6 +374,12 @@ def _get_seq_len_block_table_args(
374
# No block tables associated with encoder attention
375
return (attn_metadata.encoder_seq_lens_tensor,
376
attn_metadata.max_encoder_seq_len, None)
377
+ elif attn_type == AttentionType.ENCODER_ONLY:
378
+ assert is_prompt, "Should not have decode for encoder only model."
379
+
380
+ # No block tables associated with encoder attention
381
+ return (attn_metadata.seq_lens_tensor,
382
+ attn_metadata.max_prefill_seq_len, None)
383
else:
384
raise AttributeError(f"Invalid attention type {str(attn_type)}")
385
@@ -479,7 +488,10 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
488
* ENCODER: no KV caching; pass encoder sequence
489
attributes (encoder_seq_lens/encoder_seq_lens_tensor/
490
max_encoder_seq_len) to kernel, in lieu of decoder
482
- sequence attributes (seq_lens/seq_lens_tensor/max_seq_len)
491
+ sequence attributes (seq_lens/seq_lens_tensor/max_seq_len).
492
+ Used for encoder branch of encoder-decoder models.
493
+ * ENCODER_ONLY: no kv_caching, uses the normal attention
494
+ attributes (seq_lens/seq_lens_tensor/max_seq_len).
495
* ENCODER_DECODER: cross-attention behavior;
496
use cross-attention block table for caching KVs derived
497
from encoder hidden states; since KV sequence lengths
@@ -509,6 +521,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
521
and (not attn_metadata.is_all_encoder_attn_metadata_set)):
522
raise AttributeError("Encoder attention requires setting "
523
"encoder metadata attributes.")
524
+
525
elif (attn_type == AttentionType.ENCODER_DECODER
526
and (not attn_metadata.is_all_cross_attn_metadata_set)):
527
raise AttributeError("Encoder/decoder cross-attention "
@@ -609,6 +622,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
622
assert out.shape == output[:num_prefill_tokens].shape
623
output[:num_prefill_tokens] = out
624
else:
625
+ assert attn_type != AttentionType.ENCODER_ONLY, (
626
+ "Encoder-only models should not have prefix attention.")
627
628
assert prefill_meta.query_start_loc is not None
629
assert prefill_meta.max_query_len is not None
@@ -638,6 +653,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
653
output[:num_prefill_tokens] = out
654
655
if decode_meta := attn_metadata.decode_metadata:
656
+ assert attn_type != AttentionType.ENCODER_ONLY, (
657
+ "Encoder-only models should not have decode metadata.")
658
659
(
660
seq_lens_arg,
@@ -703,36 +720,60 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
720
None, :].expand(value.shape[0], self.num_kv_heads,
721
self.num_queries_per_kv,
722
value.shape[-1])
723
+
724
# Set attention bias if not provided. This typically happens at
725
# the very attention layer of every iteration.
726
# FIXME(woosuk): This is a hack.
727
attn_bias = _get_attn_bias(attn_metadata, attn_type)
728
if attn_bias is None:
729
if self.alibi_slopes is None:
730
+
731
+ # Cross attention block of decoder branch of encoder-decoder
732
+ # model uses seq_lens for dec / encoder_seq_lens for enc
733
if (attn_type == AttentionType.ENCODER_DECODER):
734
assert attn_metadata.seq_lens is not None
735
assert attn_metadata.encoder_seq_lens is not None
736
716
- # Default enc/dec cross-attention mask is non-causal
737
+ # Cross-attention mask is non-causal
738
attn_bias = BlockDiagonalMask.from_seqlens(
739
attn_metadata.seq_lens, attn_metadata.encoder_seq_lens)
740
+
741
+ # Encoder branch of encoder-decoder model uses
742
+ # attn_metadata.encoder_seq_lens
743
elif attn_type == AttentionType.ENCODER:
744
+
745
assert attn_metadata.encoder_seq_lens is not None
746
722
- # Default encoder self-attention mask is non-causal
747
+ # Encoder self-attention mask is non-causal
748
attn_bias = BlockDiagonalMask.from_seqlens(
749
attn_metadata.encoder_seq_lens)
725
- else:
750
+
751
+ # Self-attention block of encoder-only model just
752
+ # uses the seq_lens directly.
753
+ elif attn_type == AttentionType.ENCODER_ONLY:
754
assert attn_metadata.seq_lens is not None
755
728
- # Default decoder self-attention mask is causal
756
+ # Encoder self-attention mask is non-causal
757
+ attn_bias = BlockDiagonalMask.from_seqlens(
758
+ attn_metadata.seq_lens)
759
+
760
+ # Self-attention block of decoder branch just
761
+ # uses the seq_lens directly
762
+ elif attn_type == AttentionType.DECODER:
763
+ assert attn_metadata.seq_lens is not None
764
+
765
+ # Decoder self-attention mask is causal
766
attn_bias = BlockDiagonalCausalMask.from_seqlens(
767
attn_metadata.seq_lens)
768
+ else:
769
+ raise ValueError("Unknown AttentionType: %s", attn_type)
770
+
771
if self.sliding_window is not None:
772
attn_bias = attn_bias.make_local_attention(
773
self.sliding_window)
774
attn_bias = [attn_bias]
775
else:
776
+ assert attn_type == AttentionType.DECODER
777
assert attn_metadata.seq_lens is not None
778
attn_bias = _make_alibi_bias(self.alibi_slopes,
779
self.num_kv_heads, query.dtype,
vllm/model_executor/layers/pooler.py
+10
-2
@@ -12,6 +12,7 @@ class PoolingType(IntEnum):
12
"""Enumeration for different types of pooling methods."""
13
LAST = 0
14
ALL = 1
15
+ CLS = 2
16
17
18
class Pooler(nn.Module):
@@ -23,12 +24,13 @@ class Pooler(nn.Module):
24
3. Returns structured results as `PoolerOutput`.
25
26
Attributes:
26
- pooling_type: The type of pooling to use (LAST, AVERAGE, MAX).
27
+ pooling_type: The type of pooling to use (LAST, ALL, CLS).
28
normalize: Whether to normalize the pooled data.
29
"""
30
31
def __init__(self, pooling_type: PoolingType, normalize: bool):
32
super().__init__()
33
+
34
self.pooling_type = pooling_type
35
self.normalize = normalize
36
@@ -38,10 +40,16 @@ class Pooler(nn.Module):
40
pooling_metadata: PoolingMetadata,
41
) -> PoolerOutput:
42
"""Pools specific information from hidden states based on metadata."""
43
+
44
prompt_lens = PoolingTensors.from_pooling_metadata(
45
pooling_metadata, hidden_states.device).prompt_lens
46
44
- if self.pooling_type == PoolingType.LAST:
47
+ if self.pooling_type is PoolingType.CLS:
48
+ first_token_flat_indices = torch.zeros_like(prompt_lens)
49
+ first_token_flat_indices[1:] += torch.cumsum(prompt_lens,
50
+ dim=0)[:-1]
51
+ pooled_data = hidden_states[first_token_flat_indices]
52
+ elif self.pooling_type == PoolingType.LAST:
53
last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
54
pooled_data = hidden_states[last_token_flat_indices]
55
elif self.pooling_type == PoolingType.ALL:
vllm/model_executor/models/bert.py
new
+419
@@ -0,0 +1,419 @@
1
+from typing import Iterable, List, Optional, Tuple
2
+
3
+import torch
4
+from torch import nn
5
+from transformers import BertConfig
6
+
7
+from vllm.attention import Attention, AttentionMetadata, AttentionType
8
+from vllm.attention.backends.xformers import XFormersImpl
9
+from vllm.config import CacheConfig
10
+from vllm.distributed import get_tensor_model_parallel_world_size
11
+from vllm.model_executor.layers.activation import get_act_fn
12
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13
+ QKVParallelLinear,
14
+ RowParallelLinear)
15
+from vllm.model_executor.layers.pooler import Pooler, PoolingType
16
+from vllm.model_executor.layers.quantization.base_config import (
17
+ QuantizationConfig)
18
+from vllm.model_executor.layers.vocab_parallel_embedding import (
19
+ VocabParallelEmbedding)
20
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
+from vllm.model_executor.pooling_metadata import PoolingMetadata
22
+from vllm.sequence import IntermediateTensors, PoolerOutput
23
+
24
+
25
+class BertEmbedding(nn.Module):
26
+
27
+ def __init__(self, config: BertConfig):
28
+
29
+ super().__init__()
30
+ self.size = config.hidden_size
31
+ self.word_embeddings = VocabParallelEmbedding(config.vocab_size,
32
+ config.hidden_size)
33
+ self.position_embeddings = VocabParallelEmbedding(
34
+ config.max_position_embeddings, config.hidden_size)
35
+ self.token_type_embeddings = VocabParallelEmbedding(
36
+ config.type_vocab_size, config.hidden_size)
37
+ self.LayerNorm = nn.LayerNorm(config.hidden_size,
38
+ eps=config.layer_norm_eps)
39
+ self.position_ids = nn.Parameter(
40
+ torch.empty((1, config.max_position_embeddings)), )
41
+
42
+ self.position_embedding_type = config.position_embedding_type
43
+ if self.position_embedding_type != "absolute":
44
+ raise ValueError("Only 'absolute' position_embedding_type" +
45
+ " is supported")
46
+
47
+ def forward(
48
+ self,
49
+ input_ids: torch.Tensor,
50
+ position_ids: Optional[torch.Tensor] = None,
51
+ ) -> torch.Tensor:
52
+ input_shape = input_ids.size()
53
+
54
+ # Input embeddings.
55
+ inputs_embeds = self.word_embeddings(input_ids)
56
+
57
+ # Position embeddings.
58
+ position_embeddings = self.position_embeddings(position_ids)
59
+
60
+ # Token type embeddings. (TODO: move off hotpath?)
61
+ token_type_embeddings = self.token_type_embeddings(
62
+ torch.zeros(input_shape,
63
+ dtype=torch.long,
64
+ device=inputs_embeds.device))
65
+
66
+ embeddings = inputs_embeds + token_type_embeddings + position_embeddings
67
+ embeddings = self.LayerNorm(embeddings)
68
+ return embeddings
69
+
70
+
71
+class BertEncoder(nn.Module):
72
+
73
+ def __init__(self,
74
+ config: BertConfig,
75
+ cache_config: Optional[CacheConfig] = None,
76
+ quant_config: Optional[QuantizationConfig] = None,
77
+ prefix: str = ""):
78
+ super().__init__()
79
+ self.layer = nn.ModuleList([
80
+ BertLayer(config=config,
81
+ cache_config=cache_config,
82
+ quant_config=quant_config,
83
+ prefix=f"{prefix}.layer.{layer_idx}")
84
+ for layer_idx in range(config.num_hidden_layers)
85
+ ])
86
+
87
+ def forward(
88
+ self,
89
+ hidden_states: torch.Tensor,
90
+ kv_caches: List[torch.Tensor],
91
+ attn_metadata: AttentionMetadata,
92
+ ) -> torch.Tensor:
93
+ for i in range(len(self.layer)):
94
+ layer = self.layer[i]
95
+ hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
96
+ return hidden_states
97
+
98
+
99
+class BertLayer(nn.Module):
100
+
101
+ def __init__(self,
102
+ config: BertConfig,
103
+ cache_config: Optional[CacheConfig] = None,
104
+ quant_config: Optional[QuantizationConfig] = None,
105
+ prefix: str = ""):
106
+ super().__init__()
107
+
108
+ self.attention = BertAttention(
109
+ hidden_size=config.hidden_size,
110
+ num_attention_heads=config.num_attention_heads,
111
+ layer_norm_eps=config.layer_norm_eps,
112
+ cache_config=cache_config,
113
+ quant_config=quant_config,
114
+ prefix=f"{prefix}.attention")
115
+
116
+ self.intermediate = BertIntermediate(
117
+ hidden_size=config.hidden_size,
118
+ intermediate_size=config.intermediate_size,
119
+ hidden_act=config.hidden_act,
120
+ quant_config=quant_config,
121
+ prefix=f"{prefix}.intermediate")
122
+
123
+ self.output = BertOutput(hidden_size=config.hidden_size,
124
+ intermediate_size=config.intermediate_size,
125
+ layer_norm_eps=config.layer_norm_eps,
126
+ quant_config=quant_config,
127
+ prefix=f"{prefix}.output")
128
+
129
+ def forward(
130
+ self,
131
+ hidden_states: torch.Tensor,
132
+ kv_cache: Optional[torch.Tensor],
133
+ attn_metadata: AttentionMetadata,
134
+ ):
135
+ attn_output = self.attention(hidden_states, kv_cache, attn_metadata)
136
+ intermediate_output = self.intermediate(attn_output)
137
+ output = self.output(intermediate_output, attn_output)
138
+ return output
139
+
140
+
141
+class BertAttention(nn.Module):
142
+
143
+ def __init__(
144
+ self,
145
+ hidden_size: int,
146
+ num_attention_heads: int,
147
+ layer_norm_eps: float,
148
+ cache_config: Optional[CacheConfig] = None,
149
+ quant_config: Optional[QuantizationConfig] = None,
150
+ prefix: str = "",
151
+ ):
152
+ super().__init__()
153
+
154
+ self.self = BertSelfAttention(hidden_size=hidden_size,
155
+ num_attention_heads=num_attention_heads,
156
+ cache_config=cache_config,
157
+ quant_config=quant_config,
158
+ prefix=f"{prefix}.output")
159
+
160
+ self.output = BertSelfOutput(hidden_size=hidden_size,
161
+ layer_norm_eps=layer_norm_eps,
162
+ quant_config=quant_config,
163
+ prefix=f"{prefix}.output")
164
+
165
+ def forward(
166
+ self,
167
+ hidden_states: torch.Tensor,
168
+ kv_cache: torch.Tensor,
169
+ attn_metadata: AttentionMetadata,
170
+ ) -> torch.Tensor:
171
+ self_output = self.self(hidden_states, kv_cache, attn_metadata)
172
+ return self.output(self_output, hidden_states)
173
+
174
+
175
+class BertSelfAttention(nn.Module):
176
+
177
+ def __init__(
178
+ self,
179
+ hidden_size: int,
180
+ num_attention_heads: int,
181
+ cache_config: Optional[CacheConfig] = None,
182
+ quant_config: Optional[QuantizationConfig] = None,
183
+ prefix: str = "",
184
+ ):
185
+ super().__init__()
186
+ self.hidden_size = hidden_size
187
+ tp_size = get_tensor_model_parallel_world_size()
188
+
189
+ self.total_num_heads = num_attention_heads
190
+ assert self.total_num_heads % tp_size == 0
191
+
192
+ self.num_heads = self.total_num_heads // tp_size
193
+ self.total_num_kv_heads = self.total_num_heads
194
+ self.head_dim = self.hidden_size // self.total_num_heads
195
+ assert self.head_dim * self.total_num_heads == self.hidden_size
196
+
197
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
198
+
199
+ self.q_size = self.num_heads * self.head_dim
200
+ self.kv_size = self.num_kv_heads * self.head_dim
201
+ self.scaling = self.head_dim**-0.5
202
+ self.qkv_proj = QKVParallelLinear(
203
+ hidden_size=self.hidden_size,
204
+ head_size=self.head_dim,
205
+ total_num_heads=self.total_num_heads,
206
+ total_num_kv_heads=self.total_num_kv_heads,
207
+ bias=True,
208
+ quant_config=quant_config,
209
+ prefix=f"{prefix}.qkv_proj")
210
+
211
+ self.attn = Attention(num_heads=self.num_heads,
212
+ head_size=self.head_dim,
213
+ scale=self.scaling,
214
+ num_kv_heads=self.num_kv_heads,
215
+ cache_config=cache_config,
216
+ quant_config=quant_config,
217
+ prefix=f"{prefix}.attn")
218
+
219
+ if not isinstance(self.attn.impl, XFormersImpl):
220
+ raise ValueError(
221
+ "Encoder-only models currently require XFORMERS attention "
222
+ "backend. Set VLLM_ATTENTION_BACKEND=XFORMERS to use BERT.")
223
+
224
+ def forward(
225
+ self,
226
+ hidden_states: torch.Tensor,
227
+ kv_cache: torch.Tensor,
228
+ attn_metadata: AttentionMetadata,
229
+ ) -> torch.Tensor:
230
+ qkv, _ = self.qkv_proj(hidden_states)
231
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
232
+ output = self.attn(q,
233
+ k,
234
+ v,
235
+ kv_cache,
236
+ attn_metadata,
237
+ attn_type=AttentionType.ENCODER_ONLY)
238
+ return output
239
+
240
+
241
+class BertSelfOutput(nn.Module):
242
+
243
+ def __init__(self,
244
+ hidden_size: int,
245
+ layer_norm_eps: float,
246
+ quant_config: Optional[QuantizationConfig] = None,
247
+ prefix: str = ""):
248
+ super().__init__()
249
+ self.dense = RowParallelLinear(input_size=hidden_size,
250
+ output_size=hidden_size,
251
+ bias=True,
252
+ quant_config=quant_config,
253
+ prefix=f"{prefix}.dense")
254
+ self.LayerNorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps)
255
+
256
+ def forward(self, hidden_states: torch.Tensor,
257
+ input_tensor: torch.Tensor) -> torch.Tensor:
258
+ hidden_states, _ = self.dense(hidden_states)
259
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
260
+ return hidden_states
261
+
262
+
263
+class BertIntermediate(nn.Module):
264
+
265
+ def __init__(self,
266
+ hidden_size: int,
267
+ intermediate_size: int,
268
+ hidden_act: str,
269
+ quant_config: Optional[QuantizationConfig] = None,
270
+ prefix: str = ""):
271
+ super().__init__()
272
+ self.dense = ColumnParallelLinear(input_size=hidden_size,
273
+ output_size=intermediate_size,
274
+ bias=True,
275
+ quant_config=quant_config,
276
+ prefix=f"{prefix}.dense")
277
+ self.intermediate_act_fn = get_act_fn(hidden_act)
278
+
279
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
280
+ hidden_states, _ = self.dense(hidden_states)
281
+ hidden_states = self.intermediate_act_fn(hidden_states)
282
+ return hidden_states
283
+
284
+
285
+class BertOutput(nn.Module):
286
+
287
+ def __init__(self,
288
+ hidden_size: int,
289
+ intermediate_size: int,
290
+ layer_norm_eps: float,
291
+ quant_config: Optional[QuantizationConfig] = None,
292
+ prefix: str = ""):
293
+ super().__init__()
294
+
295
+ self.dense = RowParallelLinear(input_size=intermediate_size,
296
+ output_size=hidden_size,
297
+ bias=True,
298
+ quant_config=quant_config,
299
+ prefix=f"{prefix}.dense")
300
+
301
+ self.LayerNorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps)
302
+
303
+ def forward(self, hidden_states: torch.Tensor,
304
+ input_tensor: torch.Tensor) -> torch.Tensor:
305
+ hidden_states, _ = self.dense(hidden_states)
306
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
307
+ return hidden_states
308
+
309
+
310
+class BertModel(nn.Module):
311
+
312
+ def __init__(self,
313
+ config: BertConfig,
314
+ cache_config: Optional[CacheConfig] = None,
315
+ quant_config: Optional[QuantizationConfig] = None,
316
+ prefix: str = ""):
317
+ super().__init__()
318
+ self.embeddings = BertEmbedding(config)
319
+ self.encoder = BertEncoder(config,
320
+ cache_config,
321
+ quant_config,
322
+ prefix=f"{prefix}.encoder")
323
+
324
+ def forward(
325
+ self,
326
+ input_ids: torch.Tensor,
327
+ position_ids: torch.Tensor,
328
+ kv_caches: List[torch.Tensor],
329
+ attn_metadata: AttentionMetadata,
330
+ intermediate_tensors: Optional[IntermediateTensors] = None,
331
+ inputs_embeds: Optional[torch.Tensor] = None,
332
+ ) -> torch.Tensor:
333
+ if inputs_embeds is not None:
334
+ hidden_states = inputs_embeds
335
+ else:
336
+ hidden_states = self.embeddings(input_ids=input_ids,
337
+ position_ids=position_ids)
338
+
339
+ return self.encoder(hidden_states, kv_caches, attn_metadata)
340
+
341
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
342
+ stacked_params_mapping = [
343
+ # (param_name, shard_name, shard_id)
344
+ ("qkv_proj", "query", "q"),
345
+ ("qkv_proj", "key", "k"),
346
+ ("qkv_proj", "value", "v"),
347
+ ]
348
+
349
+ params_dict = dict(self.named_parameters())
350
+ for name, loaded_weight in weights:
351
+ if "pooler" in name:
352
+ continue
353
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
354
+ if weight_name not in name:
355
+ continue
356
+ name = name.replace(weight_name, param_name)
357
+ # Skip loading extra bias for GPTQ models.
358
+ if name.endswith(".bias") and name not in params_dict:
359
+ continue
360
+ param = params_dict[name]
361
+ weight_loader = param.weight_loader
362
+ weight_loader(param, loaded_weight, shard_id)
363
+ break
364
+ else:
365
+ # Skip loading extra bias for GPTQ models.
366
+ if name.endswith(".bias") and name not in params_dict:
367
+ continue
368
+ param = params_dict[name]
369
+ weight_loader = getattr(param, "weight_loader",
370
+ default_weight_loader)
371
+ weight_loader(param, loaded_weight)
372
+
373
+
374
+class BertEmbeddingModel(nn.Module):
375
+ """A model that uses Bert to provide embedding functionalities.
376
+
377
+ This class encapsulates the BertModel and provides an interface for
378
+ embedding operations and customized pooling functions.
379
+
380
+ Attributes:
381
+ model: An instance of BertModel used for forward operations.
382
+ _pooler: An instance of Pooler used for pooling operations.
383
+ """
384
+
385
+ def __init__(
386
+ self,
387
+ config: BertConfig,
388
+ cache_config: Optional[CacheConfig] = None,
389
+ quant_config: Optional[QuantizationConfig] = None,
390
+ ) -> None:
391
+ super().__init__()
392
+ self.model = BertModel(config, cache_config, quant_config)
393
+ self._pooler = Pooler(pooling_type=PoolingType.CLS, normalize=True)
394
+
395
+ def forward(
396
+ self,
397
+ input_ids: Optional[torch.Tensor],
398
+ positions: torch.Tensor,
399
+ kv_caches: List[torch.Tensor],
400
+ attn_metadata: AttentionMetadata,
401
+ intermediate_tensors: Optional[IntermediateTensors] = None,
402
+ inputs_embeds: Optional[torch.Tensor] = None,
403
+ ) -> torch.Tensor:
404
+ return self.model(input_ids=input_ids,
405
+ position_ids=positions,
406
+ kv_caches=kv_caches,
407
+ inputs_embeds=inputs_embeds,
408
+ intermediate_tensors=intermediate_tensors,
409
+ attn_metadata=attn_metadata)
410
+
411
+ def pooler(
412
+ self,
413
+ hidden_states: torch.Tensor,
414
+ pooling_metadata: PoolingMetadata,
415
+ ) -> Optional[PoolerOutput]:
416
+ return self._pooler(hidden_states, pooling_metadata)
417
+
418
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
419
+ self.model.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -87,6 +87,7 @@ _TEXT_GENERATION_MODELS = {
87
88
_EMBEDDING_MODELS = {
89
# [Text-only]
90
+ "BertModel": ("bert", "BertEmbeddingModel"),
91
"Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
92
"MistralModel": ("llama", "LlamaEmbeddingModel"),
93
"Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),