@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4a18fd14ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

Support Roberta embedding models (#9387)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Signed-off-by: Flavia Beo <flavia.beo@ibm.com> Co-authored-by: Flavia Beo <flavia.beo@ibm.com>

Maximilien de Bayser committed Nov 14, 2024 at 18:23 UTC 4a18fd14ba4a349291c798a16bf62fa8a9af0b6b
10 files changed +202 -14
csrc/attention/paged_attention_v1.cu
+3
@@ -98,6 +98,9 @@ void paged_attention_v1_launcher(
98 // NOTE(woosuk): To reduce the compilation time, we only compile for the
99 // head sizes that we use in the model. However, we can easily extend this
100 // to support any head size which is a multiple of 16.
101 + case 32:
102 + LAUNCH_PAGED_ATTENTION_V1(32);
103 + break;
104 case 64:
105 LAUNCH_PAGED_ATTENTION_V1(64);
106 break;
csrc/attention/paged_attention_v2.cu
+3
@@ -104,6 +104,9 @@ void paged_attention_v2_launcher(
104 // NOTE(woosuk): To reduce the compilation time, we only compile for the
105 // head sizes that we use in the model. However, we can easily extend this
106 // to support any head size which is a multiple of 16.
107 + case 32:
108 + LAUNCH_PAGED_ATTENTION_V2(32);
109 + break;
110 case 64:
111 LAUNCH_PAGED_ATTENTION_V2(64);
112 break;
csrc/cpu/attention.cpp
+6
@@ -385,6 +385,9 @@ void paged_attention_v1_impl_launcher(
385 int* seq_lens_ptr = seq_lens.data_ptr<int>();
386
387 switch (head_size) {
388 + case 32:
389 + LAUNCH_V1_ATTENTION_KERNEL(T, 32, BLOCK_SIZE);
390 + break;
391 case 64:
392 LAUNCH_V1_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
393 break;
@@ -702,6 +705,9 @@ void paged_attention_v2_impl_launcher(
705 int* seq_lens_ptr = seq_lens.data_ptr<int>();
706
707 switch (head_size) {
708 + case 32:
709 + LAUNCH_V2_ATTENTION_KERNEL(T, 32, BLOCK_SIZE);
710 + break;
711 case 64:
712 LAUNCH_V2_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
713 break;
tests/model_executor/test_model_load_with_params.py
+44
@@ -4,12 +4,17 @@ import pytest
4
5 from vllm.model_executor.layers.pooler import PoolingType
6 from vllm.model_executor.models.bert import BertEmbeddingModel
7 +from vllm.model_executor.models.roberta import RobertaEmbeddingModel
8 from vllm.platforms import current_platform
9
10 MAX_MODEL_LEN = 128
11 MODEL_NAME = os.environ.get("MODEL_NAME", "BAAI/bge-base-en-v1.5")
12 REVISION = os.environ.get("REVISION", "main")
13
14 +MODEL_NAME_ROBERTA = os.environ.get("MODEL_NAME",
15 + "intfloat/multilingual-e5-large")
16 +REVISION_ROBERTA = os.environ.get("REVISION", "main")
17 +
18
19 @pytest.mark.skipif(current_platform.is_rocm(),
20 reason="Xformers backend is not supported on ROCm.")
@@ -48,3 +53,42 @@ def test_model_loading_with_params(vllm_runner):
53 assert model._pooler.normalize
54 # assert output
55 assert output
56 +
57 +
58 +@pytest.mark.skipif(current_platform.is_rocm(),
59 + reason="Xformers backend is not supported on ROCm.")
60 +def test_roberta_model_loading_with_params(vllm_runner):
61 + """
62 + Test parameter weight loading with tp>1.
63 + """
64 + with vllm_runner(model_name=MODEL_NAME_ROBERTA,
65 + revision=REVISION_ROBERTA,
66 + dtype="float16",
67 + max_model_len=MAX_MODEL_LEN) as model:
68 + output = model.encode("Write a short story about a robot that"
69 + " dreams for the first time.\n")
70 +
71 + model_config = model.model.llm_engine.model_config
72 +
73 + model_tokenizer = model.model.llm_engine.tokenizer
74 +
75 + # asserts on the bert model config file
76 + assert model_config.encoder_config["max_seq_length"] == 512
77 + assert not model_config.encoder_config["do_lower_case"]
78 +
79 + # asserts on the pooling config files
80 + assert model_config.pooler_config.pooling_type == PoolingType.MEAN.name
81 + assert model_config.pooler_config.pooling_norm
82 +
83 + # asserts on the tokenizer loaded
84 + assert model_tokenizer.tokenizer_id == "intfloat/multilingual-e5-large"
85 + assert not model_tokenizer.tokenizer_config["do_lower_case"]
86 +
87 + model = model.model.llm_engine.model_executor\
88 + .driver_worker.model_runner.model
89 + assert isinstance(model, RobertaEmbeddingModel)
90 + assert model._pooler.pooling_type == PoolingType.MEAN
91 + assert model._pooler.normalize
92 +
93 + # assert output
94 + assert output
tests/models/embedding/language/test_embedding.py
+2
@@ -13,10 +13,12 @@ MODELS = [
13 "intfloat/e5-mistral-7b-instruct",
14 "BAAI/bge-base-en-v1.5",
15 "BAAI/bge-multilingual-gemma2",
16 + "intfloat/multilingual-e5-large",
17 ]
18
19 ENCODER_ONLY = [
20 "BAAI/bge-base-en-v1.5",
21 + "intfloat/multilingual-e5-large",
22 ]
23
24
vllm/attention/ops/ipex_attn.py
+1 -1
@@ -10,7 +10,7 @@ class PagedAttention:
10
11 @staticmethod
12 def get_supported_head_sizes() -> List[int]:
13 - return [64, 80, 96, 112, 128, 256]
13 + return [32, 64, 80, 96, 112, 128, 256]
14
15 @staticmethod
16 def get_kv_cache_shape(
vllm/attention/ops/paged_attn.py
+1 -1
@@ -34,7 +34,7 @@ class PagedAttention:
34
35 @staticmethod
36 def get_supported_head_sizes() -> List[int]:
37 - return [64, 80, 96, 112, 120, 128, 192, 256]
37 + return [32, 64, 80, 96, 112, 120, 128, 192, 256]
38
39 @staticmethod
40 def get_kv_cache_shape(
vllm/model_executor/models/bert.py
+23 -12
@@ -5,7 +5,7 @@ from torch import nn
5 from transformers import BertConfig
6
7 from vllm.attention import Attention, AttentionMetadata, AttentionType
8 -from vllm.config import CacheConfig, VllmConfig
8 +from vllm.config import CacheConfig, PoolerConfig, VllmConfig
9 from vllm.distributed import get_tensor_model_parallel_world_size
10 from vllm.model_executor.layers.activation import get_act_fn
11 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -305,14 +305,16 @@ class BertOutput(nn.Module):
305
306 class BertModel(nn.Module):
307
308 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
308 + def __init__(self,
309 + *,
310 + vllm_config: VllmConfig,
311 + prefix: str = "",
312 + embedding_class: type = BertEmbedding):
313 super().__init__()
310 -
314 config = vllm_config.model_config.hf_config
315 cache_config = vllm_config.cache_config
316 quant_config = vllm_config.quant_config
314 -
315 - self.embeddings = BertEmbedding(config)
317 + self.embeddings = embedding_class(config)
318 self.encoder = BertEncoder(config,
319 cache_config,
320 quant_config,
@@ -382,13 +384,9 @@ class BertEmbeddingModel(nn.Module):
384 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
385 super().__init__()
386 pooler_config = vllm_config.model_config.pooler_config
385 - self.model = BertModel(vllm_config=vllm_config,
386 - prefix=maybe_prefix(prefix, "model"))
387 - self._pooler = Pooler.from_config_with_defaults(
388 - pooler_config,
389 - pooling_type=PoolingType.CLS,
390 - normalize=True,
391 - softmax=False)
387 + self.model = self._build_model(vllm_config=vllm_config,
388 + prefix=maybe_prefix(prefix, "model"))
389 + self._pooler = self._build_pooler(pooler_config)
390
391 def forward(
392 self,
@@ -415,3 +413,16 @@ class BertEmbeddingModel(nn.Module):
413
414 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
415 self.model.load_weights(weights)
416 +
417 + def _build_model(self,
418 + vllm_config: VllmConfig,
419 + prefix: str = "") -> BertModel:
420 + return BertModel(vllm_config=vllm_config,
421 + prefix=prefix,
422 + embedding_class=BertEmbedding)
423 +
424 + def _build_pooler(self, pooler_config: PoolerConfig) -> Pooler:
425 + return Pooler.from_config_with_defaults(pooler_config,
426 + pooling_type=PoolingType.CLS,
427 + normalize=True,
428 + softmax=False)
vllm/model_executor/models/registry.py
+2
@@ -94,6 +94,8 @@ _TEXT_GENERATION_MODELS = {
94 _EMBEDDING_MODELS = {
95 # [Text-only]
96 "BertModel": ("bert", "BertEmbeddingModel"),
97 + "RobertaModel": ("roberta", "RobertaEmbeddingModel"),
98 + "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
99 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
100 "Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
101 "LlamaModel": ("llama", "LlamaEmbeddingModel"),
vllm/model_executor/models/roberta.py new
+117
@@ -0,0 +1,117 @@
1 +from typing import List, Optional
2 +
3 +import torch
4 +from torch import nn
5 +from transformers import RobertaConfig
6 +
7 +from vllm.attention import AttentionMetadata
8 +from vllm.config import VllmConfig
9 +from vllm.model_executor.layers.vocab_parallel_embedding import (
10 + VocabParallelEmbedding)
11 +from vllm.model_executor.models.bert import BertEmbeddingModel, BertModel
12 +from vllm.sequence import IntermediateTensors
13 +
14 +
15 +class RobertaEmbedding(nn.Module):
16 +
17 + def __init__(self, config: RobertaConfig):
18 + super().__init__()
19 + self.size = config.hidden_size
20 + self.word_embeddings = VocabParallelEmbedding(config.vocab_size,
21 + config.hidden_size)
22 + self.padding_idx = config.pad_token_id
23 + self.position_embeddings = nn.Embedding(config.max_position_embeddings,
24 + config.hidden_size,
25 + padding_idx=self.padding_idx)
26 +
27 + self.token_type_embeddings = nn.Embedding(config.type_vocab_size,
28 + config.hidden_size)
29 + self.LayerNorm = nn.LayerNorm(config.hidden_size,
30 + eps=config.layer_norm_eps)
31 + self.position_ids = nn.Parameter(
32 + torch.empty((1, config.max_position_embeddings)), )
33 +
34 + self.position_embedding_type = config.position_embedding_type
35 + if self.position_embedding_type != "absolute":
36 + raise ValueError("Only 'absolute' position_embedding_type" +
37 + " is supported")
38 +
39 + def forward(
40 + self,
41 + input_ids: torch.Tensor,
42 + position_ids: Optional[torch.Tensor] = None,
43 + ) -> torch.Tensor:
44 + input_shape = input_ids.size()
45 +
46 + # Input embeddings.
47 + inputs_embeds = self.word_embeddings(input_ids)
48 +
49 + # TODO: figure out if there is a better way
50 + # to make to make position ids start at padding_idx + 1
51 + # References:
52 + # - https://github.com/huggingface/transformers/blob/a3d69a8994d673899608a7c17fbf4f953f50474e/src/transformers/models/roberta/modeling_roberta.py#L133
53 + # - https://github.com/huggingface/transformers/blob/a3d69a8994d673899608a7c17fbf4f953f50474e/src/transformers/models/roberta/modeling_roberta.py#L1669
54 + position_ids += self.padding_idx + 1
55 +
56 + # Position embeddings.
57 + position_embeddings = self.position_embeddings(position_ids)
58 +
59 + # Token type embeddings. (TODO: move off hotpath?)
60 + token_type_embeddings = self.token_type_embeddings(
61 + torch.zeros(input_shape,
62 + dtype=torch.long,
63 + device=inputs_embeds.device))
64 +
65 + embeddings = inputs_embeds + token_type_embeddings + position_embeddings
66 + embeddings = self.LayerNorm(embeddings)
67 + return embeddings
68 +
69 +
70 +class RobertaEmbeddingModel(BertEmbeddingModel):
71 + """A model that uses Roberta to provide embedding functionalities.
72 +
73 + This class encapsulates the BertModel and provides an interface for
74 + embedding operations and customized pooling functions.
75 +
76 + Attributes:
77 + model: An instance of BertModel used for forward operations.
78 + _pooler: An instance of Pooler used for pooling operations.
79 + """
80 +
81 + def _build_model(self,
82 + vllm_config: VllmConfig,
83 + prefix: str = "") -> BertModel:
84 + return BertModel(vllm_config=vllm_config,
85 + prefix=prefix,
86 + embedding_class=RobertaEmbedding)
87 +
88 + def forward(
89 + self,
90 + input_ids: Optional[torch.Tensor],
91 + positions: torch.Tensor,
92 + kv_caches: List[torch.Tensor],
93 + attn_metadata: AttentionMetadata,
94 + intermediate_tensors: Optional[IntermediateTensors] = None,
95 + inputs_embeds: Optional[torch.Tensor] = None,
96 + ) -> torch.Tensor:
97 +
98 + # Verify assumption that position are always a sequence from
99 + # 0 to N. (Actually here we just check 0 and N to simplify).
100 + # This is important to fix the position which are assumed to
101 + # start from padding_idx + 1 instead of 0 in the Roberta models.
102 + assert hasattr(attn_metadata, "seq_lens_tensor")
103 + cumulative = attn_metadata.seq_lens_tensor.cumsum(dim=0)
104 + start_pos = torch.cat(
105 + (torch.tensor([0], device=attn_metadata.seq_lens_tensor.device),
106 + cumulative[:-1]))
107 + assert len(torch.nonzero(positions[start_pos])) == 0
108 + end_pos = cumulative - 1
109 + last_tokens = attn_metadata.seq_lens_tensor - 1
110 + assert len(torch.nonzero(positions[end_pos] - last_tokens)) == 0
111 +
112 + return super().forward(input_ids=input_ids,
113 + positions=positions,
114 + kv_caches=kv_caches,
115 + attn_metadata=attn_metadata,
116 + intermediate_tensors=intermediate_tensors,
117 + inputs_embeds=inputs_embeds)