@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bb01f2915e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix][Model] Fix Mllama SDPA illegal memory access for batched multi-image (#9626)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Oct 23, 2024 at 22:03 UTC bb01f2915eb3ade94b086033d7f2a6fe7de3c067
1 file changed +5 -3
vllm/model_executor/models/mllama.py
+5 -3
@@ -795,17 +795,19 @@ class MllamaTextCrossAttention(nn.Module):
795 kv_len = k.shape[0]
796 q = q.transpose(0, 1).view(self.num_local_key_value_heads,
797 self.num_key_value_groups, q_len,
798 - self.head_dim)
798 + self.head_dim).contiguous()
799 k = k.transpose(0,
800 1)[:,
801 None, :, :].expand(self.num_local_key_value_heads,
802 self.num_key_value_groups,
803 - kv_len, self.head_dim)
803 + kv_len,
804 + self.head_dim).contiguous()
805 v = v.transpose(0,
806 1)[:,
807 None, :, :].expand(self.num_local_key_value_heads,
808 self.num_key_value_groups,
808 - kv_len, self.head_dim)
809 + kv_len,
810 + self.head_dim).contiguous()
811 attention_mask = attention_mask.view(1, 1, q_len, kv_len)
812 output = F.scaled_dot_product_attention(q,
813 k,