A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model][Pixtral] Use memory_efficient_attention for PixtralHFVision (#9520)
Michael Goin committed
Oct 20, 2024 at 01:29 UTC
962d2c63495e930cdd3b59479dce1de48be57ecd
1 file changed
+21
-41
vllm/model_executor/models/pixtral.py
+21
-41
@@ -13,8 +13,7 @@ from transformers import PixtralVisionConfig, PretrainedConfig
13
from transformers.models.pixtral.image_processing_pixtral import (
14
_num_image_tokens)
15
from transformers.models.pixtral.modeling_pixtral import (
16
- PixtralRotaryEmbedding, apply_rotary_pos_emb,
17
- generate_block_attention_mask, position_ids_in_meshgrid)
16
+ PixtralRotaryEmbedding, apply_rotary_pos_emb, position_ids_in_meshgrid)
17
from xformers.ops.fmha import memory_efficient_attention
18
from xformers.ops.fmha.attn_bias import BlockDiagonalMask
19
@@ -813,48 +812,30 @@ class PixtralHFAttention(nn.Module):
812
def forward(
813
self,
814
hidden_states: torch.Tensor,
816
- attention_mask: torch.Tensor,
815
+ attention_mask: BlockDiagonalMask,
816
position_embeddings: torch.Tensor,
817
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
819
- """Input shape: Batch x Time x Channel"""
818
+ batch, patches, _ = hidden_states.size()
819
821
- batch_size, patches, _ = hidden_states.size()
822
-
823
- query_states = self.q_proj(hidden_states)
824
- key_states = self.k_proj(hidden_states)
825
- value_states = self.v_proj(hidden_states)
826
-
827
- query_states = query_states.view(batch_size, patches, self.n_heads,
828
- self.head_dim).transpose(1, 2)
829
- key_states = key_states.view(batch_size, patches, self.n_heads,
830
- self.head_dim).transpose(1, 2)
831
- value_states = value_states.view(batch_size, patches, self.n_heads,
832
- self.head_dim).transpose(1, 2)
820
+ q = self.q_proj(hidden_states)
821
+ k = self.k_proj(hidden_states)
822
+ v = self.v_proj(hidden_states)
823
824
+ # Transpose q and k to apply HF's Rotary Position Embedding
825
+ q = q.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
826
+ k = k.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
827
cos, sin = position_embeddings
835
- query_states, key_states = apply_rotary_pos_emb(query_states,
836
- key_states,
837
- cos,
838
- sin,
839
- unsqueeze_dim=0)
840
-
841
- attn_weights = torch.matmul(query_states, key_states.transpose(
842
- 2, 3)) * self.scale
843
-
844
- if attention_mask is not None:
845
- attn_weights = attn_weights + attention_mask
828
+ q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=0)
829
847
- # upcast attention to fp32
848
- attn_weights = nn.functional.softmax(attn_weights,
849
- dim=-1,
850
- dtype=torch.float32).to(
851
- query_states.dtype)
852
- attn_output = torch.matmul(attn_weights, value_states)
830
+ # Transpose q and k back for attention
831
+ q = q.transpose(1, 2).contiguous()
832
+ k = k.transpose(1, 2).contiguous()
833
+ v = v.reshape(batch, patches, self.n_heads, self.head_dim)
834
854
- attn_output = attn_output.transpose(1, 2).contiguous()
855
- attn_output = attn_output.reshape(batch_size, patches, -1)
835
+ out = memory_efficient_attention(q, k, v, attn_bias=attention_mask)
836
+ out = out.reshape(batch, patches, self.n_heads * self.head_dim)
837
857
- return self.o_proj(attn_output)
838
+ return self.o_proj(out)
839
840
841
class PixtralHFTransformerBlock(nn.Module):
@@ -869,7 +850,7 @@ class PixtralHFTransformerBlock(nn.Module):
850
def forward(
851
self,
852
hidden_states: torch.Tensor,
872
- attention_mask: torch.Tensor,
853
+ attention_mask: BlockDiagonalMask,
854
position_embeddings: torch.Tensor,
855
) -> torch.Tensor:
856
r = self.attention.forward(self.attention_norm(hidden_states),
@@ -892,7 +873,7 @@ class PixtralHFTransformer(nn.Module):
873
def forward(
874
self,
875
x: torch.Tensor,
895
- attention_mask: torch.Tensor,
876
+ attention_mask: BlockDiagonalMask,
877
position_embeddings: torch.Tensor,
878
) -> torch.Tensor:
879
for layer in self.layers:
@@ -953,9 +934,8 @@ class PixtralHFVisionModel(nn.Module):
934
935
position_embedding = self.patch_positional_embedding(
936
patch_embeds, position_ids)
956
- attention_mask = generate_block_attention_mask(
957
- [p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
958
- patch_embeds)
937
+ attention_mask = BlockDiagonalMask.from_seqlens(
938
+ [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
939
out = self.transformer(patch_embeds, attention_mask,
940
position_embedding)
941