@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 962d2c6349
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model][Pixtral] Use memory_efficient_attention for PixtralHFVision (#9520)

Michael Goin committed Oct 20, 2024 at 01:29 UTC 962d2c63495e930cdd3b59479dce1de48be57ecd
1 file changed +21 -41
vllm/model_executor/models/pixtral.py
+21 -41
@@ -13,8 +13,7 @@ from transformers import PixtralVisionConfig, PretrainedConfig
13 from transformers.models.pixtral.image_processing_pixtral import (
14 _num_image_tokens)
15 from transformers.models.pixtral.modeling_pixtral import (
16 - PixtralRotaryEmbedding, apply_rotary_pos_emb,
17 - generate_block_attention_mask, position_ids_in_meshgrid)
16 + PixtralRotaryEmbedding, apply_rotary_pos_emb, position_ids_in_meshgrid)
17 from xformers.ops.fmha import memory_efficient_attention
18 from xformers.ops.fmha.attn_bias import BlockDiagonalMask
19
@@ -813,48 +812,30 @@ class PixtralHFAttention(nn.Module):
812 def forward(
813 self,
814 hidden_states: torch.Tensor,
816 - attention_mask: torch.Tensor,
815 + attention_mask: BlockDiagonalMask,
816 position_embeddings: torch.Tensor,
817 ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
819 - """Input shape: Batch x Time x Channel"""
818 + batch, patches, _ = hidden_states.size()
819
821 - batch_size, patches, _ = hidden_states.size()
822 -
823 - query_states = self.q_proj(hidden_states)
824 - key_states = self.k_proj(hidden_states)
825 - value_states = self.v_proj(hidden_states)
826 -
827 - query_states = query_states.view(batch_size, patches, self.n_heads,
828 - self.head_dim).transpose(1, 2)
829 - key_states = key_states.view(batch_size, patches, self.n_heads,
830 - self.head_dim).transpose(1, 2)
831 - value_states = value_states.view(batch_size, patches, self.n_heads,
832 - self.head_dim).transpose(1, 2)
820 + q = self.q_proj(hidden_states)
821 + k = self.k_proj(hidden_states)
822 + v = self.v_proj(hidden_states)
823
824 + # Transpose q and k to apply HF's Rotary Position Embedding
825 + q = q.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
826 + k = k.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
827 cos, sin = position_embeddings
835 - query_states, key_states = apply_rotary_pos_emb(query_states,
836 - key_states,
837 - cos,
838 - sin,
839 - unsqueeze_dim=0)
840 -
841 - attn_weights = torch.matmul(query_states, key_states.transpose(
842 - 2, 3)) * self.scale
843 -
844 - if attention_mask is not None:
845 - attn_weights = attn_weights + attention_mask
828 + q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=0)
829
847 - # upcast attention to fp32
848 - attn_weights = nn.functional.softmax(attn_weights,
849 - dim=-1,
850 - dtype=torch.float32).to(
851 - query_states.dtype)
852 - attn_output = torch.matmul(attn_weights, value_states)
830 + # Transpose q and k back for attention
831 + q = q.transpose(1, 2).contiguous()
832 + k = k.transpose(1, 2).contiguous()
833 + v = v.reshape(batch, patches, self.n_heads, self.head_dim)
834
854 - attn_output = attn_output.transpose(1, 2).contiguous()
855 - attn_output = attn_output.reshape(batch_size, patches, -1)
835 + out = memory_efficient_attention(q, k, v, attn_bias=attention_mask)
836 + out = out.reshape(batch, patches, self.n_heads * self.head_dim)
837
857 - return self.o_proj(attn_output)
838 + return self.o_proj(out)
839
840
841 class PixtralHFTransformerBlock(nn.Module):
@@ -869,7 +850,7 @@ class PixtralHFTransformerBlock(nn.Module):
850 def forward(
851 self,
852 hidden_states: torch.Tensor,
872 - attention_mask: torch.Tensor,
853 + attention_mask: BlockDiagonalMask,
854 position_embeddings: torch.Tensor,
855 ) -> torch.Tensor:
856 r = self.attention.forward(self.attention_norm(hidden_states),
@@ -892,7 +873,7 @@ class PixtralHFTransformer(nn.Module):
873 def forward(
874 self,
875 x: torch.Tensor,
895 - attention_mask: torch.Tensor,
876 + attention_mask: BlockDiagonalMask,
877 position_embeddings: torch.Tensor,
878 ) -> torch.Tensor:
879 for layer in self.layers:
@@ -953,9 +934,8 @@ class PixtralHFVisionModel(nn.Module):
934
935 position_embedding = self.patch_positional_embedding(
936 patch_embeds, position_ids)
956 - attention_mask = generate_block_attention_mask(
957 - [p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
958 - patch_embeds)
937 + attention_mask = BlockDiagonalMask.from_seqlens(
938 + [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
939 out = self.transformer(patch_embeds, attention_mask,
940 position_embedding)
941