@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c91ed47c43
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Remove xformers requirement for Pixtral (#9597)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Oct 24, 2024 at 18:38 UTC c91ed47c436f2d45299bed5eacd257e8cbc7c312
1 file changed +46 -19
vllm/model_executor/models/pixtral.py
+46 -19
@@ -14,8 +14,6 @@ from transformers.models.pixtral.image_processing_pixtral import (
14 _num_image_tokens)
15 from transformers.models.pixtral.modeling_pixtral import (
16 PixtralRotaryEmbedding, apply_rotary_pos_emb, position_ids_in_meshgrid)
17 -from xformers.ops.fmha import memory_efficient_attention
18 -from xformers.ops.fmha.attn_bias import BlockDiagonalMask
17
18 from vllm.attention import AttentionMetadata
19 from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
@@ -38,6 +36,12 @@ from vllm.utils import is_list_of
36 from .interfaces import SupportsMultiModal, SupportsPP
37 from .utils import init_vllm_registered_model
38
39 +try:
40 + from xformers import ops as xops
41 + USE_XFORMERS_OPS = True
42 +except ImportError:
43 + USE_XFORMERS_OPS = False
44 +
45
46 def get_max_pixtral_image_tokens(ctx: InputContext):
47 tokenizer = cached_get_tokenizer(
@@ -416,7 +420,7 @@ class Attention(nn.Module):
420 def forward(
421 self,
422 x: torch.Tensor,
419 - mask: BlockDiagonalMask,
423 + mask: torch.Tensor,
424 freqs_cis: torch.Tensor,
425 ) -> torch.Tensor:
426 batch, patches, _ = x.shape
@@ -427,7 +431,7 @@ class Attention(nn.Module):
431 v = v.reshape(batch, patches, self.n_heads, self.head_dim)
432
433 q, k = apply_rotary_emb_vit(q, k, freqs_cis=freqs_cis)
430 - out = memory_efficient_attention(q, k, v, attn_bias=mask)
434 + out = xops.memory_efficient_attention(q, k, v, attn_bias=mask)
435 out = out.reshape(batch, patches, self.n_heads * self.head_dim)
436 return self.wo(out)
437
@@ -444,7 +448,7 @@ class TransformerBlock(nn.Module):
448 def forward(
449 self,
450 x: torch.Tensor,
447 - mask: BlockDiagonalMask,
451 + mask: torch.Tensor,
452 freqs_cis: torch.Tensor,
453 ) -> torch.Tensor:
454 r = self.attention.forward(self.attention_norm(x),
@@ -467,7 +471,7 @@ class Transformer(nn.Module):
471 def forward(
472 self,
473 x: torch.Tensor,
470 - mask: BlockDiagonalMask,
474 + mask: torch.Tensor,
475 freqs_cis: Optional[torch.Tensor],
476 ) -> torch.Tensor:
477 for layer in self.layers:
@@ -562,8 +566,12 @@ class VisionTransformer(nn.Module):
566 freqs_cis = self.freqs_cis[positions[:, 0], positions[:, 1]]
567
568 # pass through Transformer with a block diagonal mask delimiting images
565 - mask = BlockDiagonalMask.from_seqlens(
566 - [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
569 + if USE_XFORMERS_OPS:
570 + mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(
571 + [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
572 + else:
573 + raise ImportError("Xformers is required for Pixtral inference "
574 + "with the Mistral format")
575 out = self.transformer(patch_embeds, mask=mask, freqs_cis=freqs_cis)
576
577 # remove batch dimension of the single sequence
@@ -828,7 +836,7 @@ class PixtralHFAttention(nn.Module):
836 def forward(
837 self,
838 hidden_states: torch.Tensor,
831 - attention_mask: BlockDiagonalMask,
839 + attention_mask: torch.Tensor,
840 position_embeddings: torch.Tensor,
841 ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
842 batch, patches, _ = hidden_states.size()
@@ -843,12 +851,23 @@ class PixtralHFAttention(nn.Module):
851 cos, sin = position_embeddings
852 q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=0)
853
846 - # Transpose q and k back for attention
847 - q = q.transpose(1, 2).contiguous()
848 - k = k.transpose(1, 2).contiguous()
849 - v = v.reshape(batch, patches, self.n_heads, self.head_dim)
854 + if USE_XFORMERS_OPS:
855 + # Transpose q and k back for attention
856 + q = q.transpose(1, 2).contiguous()
857 + k = k.transpose(1, 2).contiguous()
858 + v = v.reshape(batch, patches, self.n_heads, self.head_dim)
859 +
860 + out = xops.memory_efficient_attention(q,
861 + k,
862 + v,
863 + attn_bias=attention_mask)
864 + else:
865 + v = v.reshape(batch, patches, self.n_heads,
866 + self.head_dim).transpose(1, 2)
867 + out = nn.functional.scaled_dot_product_attention(
868 + q, k, v, attn_mask=attention_mask)
869 + out = out.transpose(1, 2)
870
851 - out = memory_efficient_attention(q, k, v, attn_bias=attention_mask)
871 out = out.reshape(batch, patches, self.n_heads * self.head_dim)
872
873 return self.o_proj(out)
@@ -877,7 +896,7 @@ class PixtralHFTransformerBlock(nn.Module):
896 def forward(
897 self,
898 hidden_states: torch.Tensor,
880 - attention_mask: BlockDiagonalMask,
899 + attention_mask: torch.Tensor,
900 position_embeddings: torch.Tensor,
901 ) -> torch.Tensor:
902 r = self.attention.forward(self.attention_norm(hidden_states),
@@ -916,7 +935,7 @@ class PixtralHFTransformer(nn.Module):
935 def forward(
936 self,
937 x: torch.Tensor,
919 - attention_mask: BlockDiagonalMask,
938 + attention_mask: torch.Tensor,
939 position_embeddings: torch.Tensor,
940 ) -> torch.Tensor:
941 for layer in self.layers:
@@ -1000,11 +1019,19 @@ class PixtralHFVisionModel(nn.Module):
1019 patch_embeds_list,
1020 max_width=self.config.image_size // self.config.patch_size).to(
1021 self.device)
1003 -
1022 position_embedding = self.patch_positional_embedding(
1023 patch_embeds, position_ids)
1006 - attention_mask = BlockDiagonalMask.from_seqlens(
1007 - [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
1024 +
1025 + if USE_XFORMERS_OPS:
1026 + attention_mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(
1027 + [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
1028 + else:
1029 + from transformers.models.pixtral.modeling_pixtral import (
1030 + generate_block_attention_mask)
1031 + attention_mask = generate_block_attention_mask(
1032 + [p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
1033 + patch_embeds)
1034 +
1035 out = self.transformer(patch_embeds, attention_mask,
1036 position_embedding)
1037