@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 72f4880425
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix/CI] Fix broken kernels/test_mha.py (#12450)

Tyler Michael Smith committed Jan 26, 2025 at 13:39 UTC 72f4880425edf06f105863b2389f9c46025e08ee
2 files changed +10 -2
tests/kernels/test_mha_attn.py
+2 -2
@@ -26,7 +26,7 @@ def clear_cache():
26 @pytest.mark.parametrize("device", ["cpu", "hip", "cuda"])
27 def test_mha_attn_platform(device: str):
28 """
29 - Test that the attention selector between different platform and device.
29 + Test the attention selector between different platform and device.
30 """
31 torch.set_default_dtype(torch.float16)
32
@@ -41,7 +41,7 @@ def test_mha_attn_platform(device: str):
41 else:
42 with patch("vllm.attention.selector.current_platform", CudaPlatform()):
43 attn = MultiHeadAttention(16, 64, scale=1)
44 - assert attn.attn_backend == _Backend.FLASH_ATTN
44 + assert attn.attn_backend == _Backend.XFORMERS
45
46 with patch("vllm.attention.selector.current_platform", CudaPlatform()):
47 attn = MultiHeadAttention(16, 72, scale=1)
vllm/attention/layer.py
+8
@@ -210,6 +210,9 @@ class MultiHeadAttention(nn.Module):
210 self.scale = scale
211 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
212
213 + assert self.num_heads % self.num_kv_heads == 0
214 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
215 +
216 dtype = torch.get_default_dtype()
217 attn_backend = get_attn_backend(head_size,
218 dtype,
@@ -240,6 +243,11 @@ class MultiHeadAttention(nn.Module):
243 key = key.view(bsz, kv_len, self.num_kv_heads, self.head_size)
244 value = value.view(bsz, kv_len, self.num_kv_heads, self.head_size)
245
246 + if (num_repeat := self.num_queries_per_kv) > 1:
247 + # Handle MQA and GQA
248 + key = torch.repeat_interleave(key, num_repeat, dim=2)
249 + value = torch.repeat_interleave(value, num_repeat, dim=2)
250 +
251 if self.attn_backend == _Backend.XFORMERS:
252 from xformers import ops as xops
253