A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix][Model] Fix Mllama SDPA illegal memory access for batched multi-image (#9626)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Oct 23, 2024 at 22:03 UTC
bb01f2915eb3ade94b086033d7f2a6fe7de3c067
1 file changed
+5
-3
vllm/model_executor/models/mllama.py
+5
-3
@@ -795,17 +795,19 @@ class MllamaTextCrossAttention(nn.Module):
795
kv_len = k.shape[0]
796
q = q.transpose(0, 1).view(self.num_local_key_value_heads,
797
self.num_key_value_groups, q_len,
798
- self.head_dim)
798
+ self.head_dim).contiguous()
799
k = k.transpose(0,
800
1)[:,
801
None, :, :].expand(self.num_local_key_value_heads,
802
self.num_key_value_groups,
803
- kv_len, self.head_dim)
803
+ kv_len,
804
+ self.head_dim).contiguous()
805
v = v.transpose(0,
806
1)[:,
807
None, :, :].expand(self.num_local_key_value_heads,
808
self.num_key_value_groups,
808
- kv_len, self.head_dim)
809
+ kv_len,
810
+ self.head_dim).contiguous()
811
attention_mask = attention_mask.view(1, 1, q_len, kv_len)
812
output = F.scaled_dot_product_attention(q,
813
k,