@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55137e8ee3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

Fix: MI100 Support By Bypassing Custom Paged Attention (#9560)

ErkinSagiroglu committed Oct 26, 2024 at 13:12 UTC 55137e8ee32509b2fa3b83d5caaee018a929f82d
1 file changed +6 -2
vllm/attention/backends/rocm_flash_attn.py
+6 -2
@@ -21,7 +21,10 @@ if TYPE_CHECKING:
21 logger = init_logger(__name__)
22
23 _PARTITION_SIZE_ROCM = 512
24 -_ON_NAVI = "gfx1" in torch.cuda.get_device_properties("cuda").gcnArchName
24 +_GPU_ARCH = torch.cuda.get_device_properties("cuda").gcnArchName
25 +_ON_NAVI = "gfx1" in _GPU_ARCH
26 +_ON_MI250_MI300 = any(arch in _GPU_ARCH
27 + for arch in ["gfx90a", "gfx940", "gfx941", "gfx942"])
28
29
30 class ROCmFlashAttentionBackend(AttentionBackend):
@@ -662,7 +665,8 @@ def _use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int,
665 block_size: int, gqa_ratio: int,
666 max_seq_len: int) -> bool:
667 # rocm custom page attention not support on navi (gfx1*)
665 - return (not _ON_NAVI and (qtype == torch.half or qtype == torch.bfloat16)
668 + return (_ON_MI250_MI300 and not _ON_NAVI
669 + and (qtype == torch.half or qtype == torch.bfloat16)
670 and (head_size == 64 or head_size == 128)
671 and (block_size == 16 or block_size == 32)
672 and (gqa_ratio >= 1 and gqa_ratio <= 16) and max_seq_len <= 32768)