A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
Support Roberta embedding models (#9387)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Signed-off-by: Flavia Beo <flavia.beo@ibm.com> Co-authored-by: Flavia Beo <flavia.beo@ibm.com>
Maximilien de Bayser committed
Nov 14, 2024 at 18:23 UTC
4a18fd14ba4a349291c798a16bf62fa8a9af0b6b
10 files changed
+202
-14
csrc/attention/paged_attention_v1.cu
+3
@@ -98,6 +98,9 @@ void paged_attention_v1_launcher(
98
// NOTE(woosuk): To reduce the compilation time, we only compile for the
99
// head sizes that we use in the model. However, we can easily extend this
100
// to support any head size which is a multiple of 16.
101
+ case 32:
102
+ LAUNCH_PAGED_ATTENTION_V1(32);
103
+ break;
104
case 64:
105
LAUNCH_PAGED_ATTENTION_V1(64);
106
break;
csrc/attention/paged_attention_v2.cu
+3
@@ -104,6 +104,9 @@ void paged_attention_v2_launcher(
104
// NOTE(woosuk): To reduce the compilation time, we only compile for the
105
// head sizes that we use in the model. However, we can easily extend this
106
// to support any head size which is a multiple of 16.
107
+ case 32:
108
+ LAUNCH_PAGED_ATTENTION_V2(32);
109
+ break;
110
case 64:
111
LAUNCH_PAGED_ATTENTION_V2(64);
112
break;
csrc/cpu/attention.cpp
+6
@@ -385,6 +385,9 @@ void paged_attention_v1_impl_launcher(
385
int* seq_lens_ptr = seq_lens.data_ptr<int>();
386
387
switch (head_size) {
388
+ case 32:
389
+ LAUNCH_V1_ATTENTION_KERNEL(T, 32, BLOCK_SIZE);
390
+ break;
391
case 64:
392
LAUNCH_V1_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
393
break;
@@ -702,6 +705,9 @@ void paged_attention_v2_impl_launcher(
705
int* seq_lens_ptr = seq_lens.data_ptr<int>();
706
707
switch (head_size) {
708
+ case 32:
709
+ LAUNCH_V2_ATTENTION_KERNEL(T, 32, BLOCK_SIZE);
710
+ break;
711
case 64:
712
LAUNCH_V2_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
713
break;
tests/model_executor/test_model_load_with_params.py
+44
@@ -4,12 +4,17 @@ import pytest
4
5
from vllm.model_executor.layers.pooler import PoolingType
6
from vllm.model_executor.models.bert import BertEmbeddingModel
7
+from vllm.model_executor.models.roberta import RobertaEmbeddingModel
8
from vllm.platforms import current_platform
9
10
MAX_MODEL_LEN = 128
11
MODEL_NAME = os.environ.get("MODEL_NAME", "BAAI/bge-base-en-v1.5")
12
REVISION = os.environ.get("REVISION", "main")
13
14
+MODEL_NAME_ROBERTA = os.environ.get("MODEL_NAME",
15
+ "intfloat/multilingual-e5-large")
16
+REVISION_ROBERTA = os.environ.get("REVISION", "main")
17
+
18
19
@pytest.mark.skipif(current_platform.is_rocm(),
20
reason="Xformers backend is not supported on ROCm.")
@@ -48,3 +53,42 @@ def test_model_loading_with_params(vllm_runner):
53
assert model._pooler.normalize
54
# assert output
55
assert output
56
+
57
+
58
+@pytest.mark.skipif(current_platform.is_rocm(),
59
+ reason="Xformers backend is not supported on ROCm.")
60
+def test_roberta_model_loading_with_params(vllm_runner):
61
+ """
62
+ Test parameter weight loading with tp>1.
63
+ """
64
+ with vllm_runner(model_name=MODEL_NAME_ROBERTA,
65
+ revision=REVISION_ROBERTA,
66
+ dtype="float16",
67
+ max_model_len=MAX_MODEL_LEN) as model:
68
+ output = model.encode("Write a short story about a robot that"
69
+ " dreams for the first time.\n")
70
+
71
+ model_config = model.model.llm_engine.model_config
72
+
73
+ model_tokenizer = model.model.llm_engine.tokenizer
74
+
75
+ # asserts on the bert model config file
76
+ assert model_config.encoder_config["max_seq_length"] == 512
77
+ assert not model_config.encoder_config["do_lower_case"]
78
+
79
+ # asserts on the pooling config files
80
+ assert model_config.pooler_config.pooling_type == PoolingType.MEAN.name
81
+ assert model_config.pooler_config.pooling_norm
82
+
83
+ # asserts on the tokenizer loaded
84
+ assert model_tokenizer.tokenizer_id == "intfloat/multilingual-e5-large"
85
+ assert not model_tokenizer.tokenizer_config["do_lower_case"]
86
+
87
+ model = model.model.llm_engine.model_executor\
88
+ .driver_worker.model_runner.model
89
+ assert isinstance(model, RobertaEmbeddingModel)
90
+ assert model._pooler.pooling_type == PoolingType.MEAN
91
+ assert model._pooler.normalize
92
+
93
+ # assert output
94
+ assert output
tests/models/embedding/language/test_embedding.py
+2
@@ -13,10 +13,12 @@ MODELS = [
13
"intfloat/e5-mistral-7b-instruct",
14
"BAAI/bge-base-en-v1.5",
15
"BAAI/bge-multilingual-gemma2",
16
+ "intfloat/multilingual-e5-large",
17
]
18
19
ENCODER_ONLY = [
20
"BAAI/bge-base-en-v1.5",
21
+ "intfloat/multilingual-e5-large",
22
]
23
24
vllm/attention/ops/ipex_attn.py
+1
-1
@@ -10,7 +10,7 @@ class PagedAttention:
10
11
@staticmethod
12
def get_supported_head_sizes() -> List[int]:
13
- return [64, 80, 96, 112, 128, 256]
13
+ return [32, 64, 80, 96, 112, 128, 256]
14
15
@staticmethod
16
def get_kv_cache_shape(
vllm/attention/ops/paged_attn.py
+1
-1
@@ -34,7 +34,7 @@ class PagedAttention:
34
35
@staticmethod
36
def get_supported_head_sizes() -> List[int]:
37
- return [64, 80, 96, 112, 120, 128, 192, 256]
37
+ return [32, 64, 80, 96, 112, 120, 128, 192, 256]
38
39
@staticmethod
40
def get_kv_cache_shape(
vllm/model_executor/models/bert.py
+23
-12
@@ -5,7 +5,7 @@ from torch import nn
5
from transformers import BertConfig
6
7
from vllm.attention import Attention, AttentionMetadata, AttentionType
8
-from vllm.config import CacheConfig, VllmConfig
8
+from vllm.config import CacheConfig, PoolerConfig, VllmConfig
9
from vllm.distributed import get_tensor_model_parallel_world_size
10
from vllm.model_executor.layers.activation import get_act_fn
11
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -305,14 +305,16 @@ class BertOutput(nn.Module):
305
306
class BertModel(nn.Module):
307
308
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
308
+ def __init__(self,
309
+ *,
310
+ vllm_config: VllmConfig,
311
+ prefix: str = "",
312
+ embedding_class: type = BertEmbedding):
313
super().__init__()
310
-
314
config = vllm_config.model_config.hf_config
315
cache_config = vllm_config.cache_config
316
quant_config = vllm_config.quant_config
314
-
315
- self.embeddings = BertEmbedding(config)
317
+ self.embeddings = embedding_class(config)
318
self.encoder = BertEncoder(config,
319
cache_config,
320
quant_config,
@@ -382,13 +384,9 @@ class BertEmbeddingModel(nn.Module):
384
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
385
super().__init__()
386
pooler_config = vllm_config.model_config.pooler_config
385
- self.model = BertModel(vllm_config=vllm_config,
386
- prefix=maybe_prefix(prefix, "model"))
387
- self._pooler = Pooler.from_config_with_defaults(
388
- pooler_config,
389
- pooling_type=PoolingType.CLS,
390
- normalize=True,
391
- softmax=False)
387
+ self.model = self._build_model(vllm_config=vllm_config,
388
+ prefix=maybe_prefix(prefix, "model"))
389
+ self._pooler = self._build_pooler(pooler_config)
390
391
def forward(
392
self,
@@ -415,3 +413,16 @@ class BertEmbeddingModel(nn.Module):
413
414
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
415
self.model.load_weights(weights)
416
+
417
+ def _build_model(self,
418
+ vllm_config: VllmConfig,
419
+ prefix: str = "") -> BertModel:
420
+ return BertModel(vllm_config=vllm_config,
421
+ prefix=prefix,
422
+ embedding_class=BertEmbedding)
423
+
424
+ def _build_pooler(self, pooler_config: PoolerConfig) -> Pooler:
425
+ return Pooler.from_config_with_defaults(pooler_config,
426
+ pooling_type=PoolingType.CLS,
427
+ normalize=True,
428
+ softmax=False)
vllm/model_executor/models/registry.py
+2
@@ -94,6 +94,8 @@ _TEXT_GENERATION_MODELS = {
94
_EMBEDDING_MODELS = {
95
# [Text-only]
96
"BertModel": ("bert", "BertEmbeddingModel"),
97
+ "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
98
+ "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
99
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
100
"Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
101
"LlamaModel": ("llama", "LlamaEmbeddingModel"),
vllm/model_executor/models/roberta.py
new
+117
@@ -0,0 +1,117 @@
1
+from typing import List, Optional
2
+
3
+import torch
4
+from torch import nn
5
+from transformers import RobertaConfig
6
+
7
+from vllm.attention import AttentionMetadata
8
+from vllm.config import VllmConfig
9
+from vllm.model_executor.layers.vocab_parallel_embedding import (
10
+ VocabParallelEmbedding)
11
+from vllm.model_executor.models.bert import BertEmbeddingModel, BertModel
12
+from vllm.sequence import IntermediateTensors
13
+
14
+
15
+class RobertaEmbedding(nn.Module):
16
+
17
+ def __init__(self, config: RobertaConfig):
18
+ super().__init__()
19
+ self.size = config.hidden_size
20
+ self.word_embeddings = VocabParallelEmbedding(config.vocab_size,
21
+ config.hidden_size)
22
+ self.padding_idx = config.pad_token_id
23
+ self.position_embeddings = nn.Embedding(config.max_position_embeddings,
24
+ config.hidden_size,
25
+ padding_idx=self.padding_idx)
26
+
27
+ self.token_type_embeddings = nn.Embedding(config.type_vocab_size,
28
+ config.hidden_size)
29
+ self.LayerNorm = nn.LayerNorm(config.hidden_size,
30
+ eps=config.layer_norm_eps)
31
+ self.position_ids = nn.Parameter(
32
+ torch.empty((1, config.max_position_embeddings)), )
33
+
34
+ self.position_embedding_type = config.position_embedding_type
35
+ if self.position_embedding_type != "absolute":
36
+ raise ValueError("Only 'absolute' position_embedding_type" +
37
+ " is supported")
38
+
39
+ def forward(
40
+ self,
41
+ input_ids: torch.Tensor,
42
+ position_ids: Optional[torch.Tensor] = None,
43
+ ) -> torch.Tensor:
44
+ input_shape = input_ids.size()
45
+
46
+ # Input embeddings.
47
+ inputs_embeds = self.word_embeddings(input_ids)
48
+
49
+ # TODO: figure out if there is a better way
50
+ # to make to make position ids start at padding_idx + 1
51
+ # References:
52
+ # - https://github.com/huggingface/transformers/blob/a3d69a8994d673899608a7c17fbf4f953f50474e/src/transformers/models/roberta/modeling_roberta.py#L133
53
+ # - https://github.com/huggingface/transformers/blob/a3d69a8994d673899608a7c17fbf4f953f50474e/src/transformers/models/roberta/modeling_roberta.py#L1669
54
+ position_ids += self.padding_idx + 1
55
+
56
+ # Position embeddings.
57
+ position_embeddings = self.position_embeddings(position_ids)
58
+
59
+ # Token type embeddings. (TODO: move off hotpath?)
60
+ token_type_embeddings = self.token_type_embeddings(
61
+ torch.zeros(input_shape,
62
+ dtype=torch.long,
63
+ device=inputs_embeds.device))
64
+
65
+ embeddings = inputs_embeds + token_type_embeddings + position_embeddings
66
+ embeddings = self.LayerNorm(embeddings)
67
+ return embeddings
68
+
69
+
70
+class RobertaEmbeddingModel(BertEmbeddingModel):
71
+ """A model that uses Roberta to provide embedding functionalities.
72
+
73
+ This class encapsulates the BertModel and provides an interface for
74
+ embedding operations and customized pooling functions.
75
+
76
+ Attributes:
77
+ model: An instance of BertModel used for forward operations.
78
+ _pooler: An instance of Pooler used for pooling operations.
79
+ """
80
+
81
+ def _build_model(self,
82
+ vllm_config: VllmConfig,
83
+ prefix: str = "") -> BertModel:
84
+ return BertModel(vllm_config=vllm_config,
85
+ prefix=prefix,
86
+ embedding_class=RobertaEmbedding)
87
+
88
+ def forward(
89
+ self,
90
+ input_ids: Optional[torch.Tensor],
91
+ positions: torch.Tensor,
92
+ kv_caches: List[torch.Tensor],
93
+ attn_metadata: AttentionMetadata,
94
+ intermediate_tensors: Optional[IntermediateTensors] = None,
95
+ inputs_embeds: Optional[torch.Tensor] = None,
96
+ ) -> torch.Tensor:
97
+
98
+ # Verify assumption that position are always a sequence from
99
+ # 0 to N. (Actually here we just check 0 and N to simplify).
100
+ # This is important to fix the position which are assumed to
101
+ # start from padding_idx + 1 instead of 0 in the Roberta models.
102
+ assert hasattr(attn_metadata, "seq_lens_tensor")
103
+ cumulative = attn_metadata.seq_lens_tensor.cumsum(dim=0)
104
+ start_pos = torch.cat(
105
+ (torch.tensor([0], device=attn_metadata.seq_lens_tensor.device),
106
+ cumulative[:-1]))
107
+ assert len(torch.nonzero(positions[start_pos])) == 0
108
+ end_pos = cumulative - 1
109
+ last_tokens = attn_metadata.seq_lens_tensor - 1
110
+ assert len(torch.nonzero(positions[end_pos] - last_tokens)) == 0
111
+
112
+ return super().forward(input_ids=input_ids,
113
+ positions=positions,
114
+ kv_caches=kv_caches,
115
+ attn_metadata=attn_metadata,
116
+ intermediate_tensors=intermediate_tensors,
117
+ inputs_embeds=inputs_embeds)