A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Remove xformers requirement for Pixtral (#9597)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Oct 24, 2024 at 18:38 UTC
c91ed47c436f2d45299bed5eacd257e8cbc7c312
1 file changed
+46
-19
vllm/model_executor/models/pixtral.py
+46
-19
@@ -14,8 +14,6 @@ from transformers.models.pixtral.image_processing_pixtral import (
14
_num_image_tokens)
15
from transformers.models.pixtral.modeling_pixtral import (
16
PixtralRotaryEmbedding, apply_rotary_pos_emb, position_ids_in_meshgrid)
17
-from xformers.ops.fmha import memory_efficient_attention
18
-from xformers.ops.fmha.attn_bias import BlockDiagonalMask
17
18
from vllm.attention import AttentionMetadata
19
from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
@@ -38,6 +36,12 @@ from vllm.utils import is_list_of
36
from .interfaces import SupportsMultiModal, SupportsPP
37
from .utils import init_vllm_registered_model
38
39
+try:
40
+ from xformers import ops as xops
41
+ USE_XFORMERS_OPS = True
42
+except ImportError:
43
+ USE_XFORMERS_OPS = False
44
+
45
46
def get_max_pixtral_image_tokens(ctx: InputContext):
47
tokenizer = cached_get_tokenizer(
@@ -416,7 +420,7 @@ class Attention(nn.Module):
420
def forward(
421
self,
422
x: torch.Tensor,
419
- mask: BlockDiagonalMask,
423
+ mask: torch.Tensor,
424
freqs_cis: torch.Tensor,
425
) -> torch.Tensor:
426
batch, patches, _ = x.shape
@@ -427,7 +431,7 @@ class Attention(nn.Module):
431
v = v.reshape(batch, patches, self.n_heads, self.head_dim)
432
433
q, k = apply_rotary_emb_vit(q, k, freqs_cis=freqs_cis)
430
- out = memory_efficient_attention(q, k, v, attn_bias=mask)
434
+ out = xops.memory_efficient_attention(q, k, v, attn_bias=mask)
435
out = out.reshape(batch, patches, self.n_heads * self.head_dim)
436
return self.wo(out)
437
@@ -444,7 +448,7 @@ class TransformerBlock(nn.Module):
448
def forward(
449
self,
450
x: torch.Tensor,
447
- mask: BlockDiagonalMask,
451
+ mask: torch.Tensor,
452
freqs_cis: torch.Tensor,
453
) -> torch.Tensor:
454
r = self.attention.forward(self.attention_norm(x),
@@ -467,7 +471,7 @@ class Transformer(nn.Module):
471
def forward(
472
self,
473
x: torch.Tensor,
470
- mask: BlockDiagonalMask,
474
+ mask: torch.Tensor,
475
freqs_cis: Optional[torch.Tensor],
476
) -> torch.Tensor:
477
for layer in self.layers:
@@ -562,8 +566,12 @@ class VisionTransformer(nn.Module):
566
freqs_cis = self.freqs_cis[positions[:, 0], positions[:, 1]]
567
568
# pass through Transformer with a block diagonal mask delimiting images
565
- mask = BlockDiagonalMask.from_seqlens(
566
- [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
569
+ if USE_XFORMERS_OPS:
570
+ mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(
571
+ [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
572
+ else:
573
+ raise ImportError("Xformers is required for Pixtral inference "
574
+ "with the Mistral format")
575
out = self.transformer(patch_embeds, mask=mask, freqs_cis=freqs_cis)
576
577
# remove batch dimension of the single sequence
@@ -828,7 +836,7 @@ class PixtralHFAttention(nn.Module):
836
def forward(
837
self,
838
hidden_states: torch.Tensor,
831
- attention_mask: BlockDiagonalMask,
839
+ attention_mask: torch.Tensor,
840
position_embeddings: torch.Tensor,
841
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
842
batch, patches, _ = hidden_states.size()
@@ -843,12 +851,23 @@ class PixtralHFAttention(nn.Module):
851
cos, sin = position_embeddings
852
q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=0)
853
846
- # Transpose q and k back for attention
847
- q = q.transpose(1, 2).contiguous()
848
- k = k.transpose(1, 2).contiguous()
849
- v = v.reshape(batch, patches, self.n_heads, self.head_dim)
854
+ if USE_XFORMERS_OPS:
855
+ # Transpose q and k back for attention
856
+ q = q.transpose(1, 2).contiguous()
857
+ k = k.transpose(1, 2).contiguous()
858
+ v = v.reshape(batch, patches, self.n_heads, self.head_dim)
859
+
860
+ out = xops.memory_efficient_attention(q,
861
+ k,
862
+ v,
863
+ attn_bias=attention_mask)
864
+ else:
865
+ v = v.reshape(batch, patches, self.n_heads,
866
+ self.head_dim).transpose(1, 2)
867
+ out = nn.functional.scaled_dot_product_attention(
868
+ q, k, v, attn_mask=attention_mask)
869
+ out = out.transpose(1, 2)
870
851
- out = memory_efficient_attention(q, k, v, attn_bias=attention_mask)
871
out = out.reshape(batch, patches, self.n_heads * self.head_dim)
872
873
return self.o_proj(out)
@@ -877,7 +896,7 @@ class PixtralHFTransformerBlock(nn.Module):
896
def forward(
897
self,
898
hidden_states: torch.Tensor,
880
- attention_mask: BlockDiagonalMask,
899
+ attention_mask: torch.Tensor,
900
position_embeddings: torch.Tensor,
901
) -> torch.Tensor:
902
r = self.attention.forward(self.attention_norm(hidden_states),
@@ -916,7 +935,7 @@ class PixtralHFTransformer(nn.Module):
935
def forward(
936
self,
937
x: torch.Tensor,
919
- attention_mask: BlockDiagonalMask,
938
+ attention_mask: torch.Tensor,
939
position_embeddings: torch.Tensor,
940
) -> torch.Tensor:
941
for layer in self.layers:
@@ -1000,11 +1019,19 @@ class PixtralHFVisionModel(nn.Module):
1019
patch_embeds_list,
1020
max_width=self.config.image_size // self.config.patch_size).to(
1021
self.device)
1003
-
1022
position_embedding = self.patch_positional_embedding(
1023
patch_embeds, position_ids)
1006
- attention_mask = BlockDiagonalMask.from_seqlens(
1007
- [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
1024
+
1025
+ if USE_XFORMERS_OPS:
1026
+ attention_mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(
1027
+ [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
1028
+ else:
1029
+ from transformers.models.pixtral.modeling_pixtral import (
1030
+ generate_block_attention_mask)
1031
+ attention_mask = generate_block_attention_mask(
1032
+ [p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
1033
+ patch_embeds)
1034
+
1035
out = self.transformer(patch_embeds, attention_mask,
1036
position_embedding)
1037