A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
Fix: MI100 Support By Bypassing Custom Paged Attention (#9560)
ErkinSagiroglu committed
Oct 26, 2024 at 13:12 UTC
55137e8ee32509b2fa3b83d5caaee018a929f82d
1 file changed
+6
-2
vllm/attention/backends/rocm_flash_attn.py
+6
-2
@@ -21,7 +21,10 @@ if TYPE_CHECKING:
21
logger = init_logger(__name__)
22
23
_PARTITION_SIZE_ROCM = 512
24
-_ON_NAVI = "gfx1" in torch.cuda.get_device_properties("cuda").gcnArchName
24
+_GPU_ARCH = torch.cuda.get_device_properties("cuda").gcnArchName
25
+_ON_NAVI = "gfx1" in _GPU_ARCH
26
+_ON_MI250_MI300 = any(arch in _GPU_ARCH
27
+ for arch in ["gfx90a", "gfx940", "gfx941", "gfx942"])
28
29
30
class ROCmFlashAttentionBackend(AttentionBackend):
@@ -662,7 +665,8 @@ def _use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int,
665
block_size: int, gqa_ratio: int,
666
max_seq_len: int) -> bool:
667
# rocm custom page attention not support on navi (gfx1*)
665
- return (not _ON_NAVI and (qtype == torch.half or qtype == torch.bfloat16)
668
+ return (_ON_MI250_MI300 and not _ON_NAVI
669
+ and (qtype == torch.half or qtype == torch.bfloat16)
670
and (head_size == 64 or head_size == 128)
671
and (block_size == 16 or block_size == 32)
672
and (gqa_ratio >= 1 and gqa_ratio <= 16) and max_seq_len <= 32768)