@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ab5bbf5ae3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix][Kernel] Fix CUDA 11.8 being broken by FA3 build (#12375)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>

Lucas Wilkinson committed Jan 24, 2025 at 10:27 UTC ab5bbf5ae32bc438803ced4c2a021bb161e00050
6 files changed +42 -22
CMakeLists.txt
+1 -1
@@ -576,7 +576,7 @@ else()
576 FetchContent_Declare(
577 vllm-flash-attn
578 GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
579 - GIT_TAG 90eacc1af2a7c3de62ea249e929ed5faccf38954
579 + GIT_TAG 0aff05f577e8a10086066a00618609199b25231d
580 GIT_PROGRESS TRUE
581 # Don't share the vllm-flash-attn build between build types
582 BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
setup.py
+4 -1
@@ -598,7 +598,10 @@ if _is_hip():
598
599 if _is_cuda():
600 ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
601 - ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
601 + if envs.VLLM_USE_PRECOMPILED or get_nvcc_cuda_version() >= Version("12.0"):
602 + # FA3 requires CUDA 12.0 or later
603 + ext_modules.append(
604 + CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
605 ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
606
607 if _build_custom_ops():
tests/kernels/test_cascade_flash_attn.py
+6 -5
@@ -6,7 +6,9 @@ import torch
6 from vllm.platforms import current_platform
7 from vllm.v1.attention.backends.flash_attn import (cascade_attention,
8 merge_attn_states)
9 -from vllm.vllm_flash_attn import flash_attn_varlen_func
9 +from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
10 + flash_attn_varlen_func,
11 + is_fa_version_supported)
12
13 NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
14 HEAD_SIZES = [128, 192, 256]
@@ -91,10 +93,9 @@ def test_cascade(
93 fa_version: int,
94 ) -> None:
95 torch.set_default_device("cuda")
94 - if fa_version == 3 and (torch.cuda.get_device_capability() == (8, 6)
95 - or torch.cuda.get_device_capability() == (8, 9)):
96 - pytest.skip("Flash attention version 3 fails on 8.6 and 8.9 due to "
97 - "insufficient shared memory for some shapes")
96 + if not is_fa_version_supported(fa_version):
97 + pytest.skip(f"Flash attention version {fa_version} not supported due "
98 + f"to: \"{fa_version_unsupported_reason(fa_version)}\"")
99
100 current_platform.seed_everything(0)
101
tests/kernels/test_flash_attn.py
+10 -11
@@ -4,8 +4,10 @@ import pytest
4 import torch
5
6 from vllm.platforms import current_platform
7 -from vllm.vllm_flash_attn import (flash_attn_varlen_func,
8 - flash_attn_with_kvcache)
7 +from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
8 + flash_attn_varlen_func,
9 + flash_attn_with_kvcache,
10 + is_fa_version_supported)
11
12 NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
13 HEAD_SIZES = [128, 256]
@@ -95,10 +97,9 @@ def test_flash_attn_with_paged_kv(
97 fa_version: int,
98 ) -> None:
99 torch.set_default_device("cuda")
98 - if fa_version == 3 and (torch.cuda.get_device_capability() == (8, 6)
99 - or torch.cuda.get_device_capability() == (8, 9)):
100 - pytest.skip("Flash attention version 3 fails on 8.6 and 8.9 due to "
101 - "insufficient shared memory for some shapes")
100 + if not is_fa_version_supported(fa_version):
101 + pytest.skip(f"Flash attention version {fa_version} not supported due "
102 + f"to: \"{fa_version_unsupported_reason(fa_version)}\"")
103
104 current_platform.seed_everything(0)
105 num_seqs = len(kv_lens)
@@ -182,11 +183,9 @@ def test_varlen_with_paged_kv(
183 fa_version: int,
184 ) -> None:
185 torch.set_default_device("cuda")
185 - if fa_version == 3 and (torch.cuda.get_device_capability() == (8, 6)
186 - or torch.cuda.get_device_capability() == (8, 9)):
187 - pytest.skip("Flash attention version 3 fails on 8.6 and 8.9 due to "
188 - "insufficient shared memory for some shapes")
189 -
186 + if not is_fa_version_supported(fa_version):
187 + pytest.skip(f"Flash attention version {fa_version} not supported due "
188 + f"to: \"{fa_version_unsupported_reason(fa_version)}\"")
189 current_platform.seed_everything(0)
190 num_seqs = len(seq_lens)
191 query_lens = [x[0] for x in seq_lens]
vllm/attention/backends/flash_attn.py
+11 -3
@@ -18,17 +18,20 @@ from vllm.attention.backends.utils import (
18 get_seq_len_block_table_args, is_all_cross_attn_metadata_set,
19 is_all_encoder_attn_metadata_set, is_block_tables_empty)
20 from vllm.envs import VLLM_FLASH_ATTN_VERSION
21 +from vllm.logger import init_logger
22 from vllm.multimodal import MultiModalPlaceholderMap
23 from vllm.platforms import current_platform
24 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
25 +from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
26 + flash_attn_varlen_func,
27 + flash_attn_with_kvcache,
28 + is_fa_version_supported)
29
30 if TYPE_CHECKING:
31 from vllm.worker.model_runner import (ModelInputForGPUBuilder,
32 ModelInputForGPUWithSamplingMetadata)
33
29 -from vllm.vllm_flash_attn import (flash_attn_varlen_func,
30 - flash_attn_with_kvcache,
31 - is_fa_version_supported)
34 +logger = init_logger(__name__)
35
36
37 class FlashAttentionBackend(AttentionBackend):
@@ -652,6 +655,11 @@ class FlashAttentionImpl(AttentionImpl):
655 assert VLLM_FLASH_ATTN_VERSION in [2, 3]
656 self.fa_version = VLLM_FLASH_ATTN_VERSION
657
658 + if not is_fa_version_supported(self.fa_version):
659 + logger.error("Cannot use FA version %d is not supported due to %s",
660 + self.fa_version,
661 + fa_version_unsupported_reason(self.fa_version))
662 +
663 assert is_fa_version_supported(self.fa_version)
664
665 def forward(
vllm/v1/attention/backends/flash_attn.py
+10 -1
@@ -10,11 +10,15 @@ import triton.language as tl
10 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11 AttentionMetadata, AttentionType)
12 from vllm.envs import VLLM_FLASH_ATTN_VERSION
13 +from vllm.logger import init_logger
14 from vllm.platforms import current_platform
15 from vllm.utils import cdiv
15 -from vllm.vllm_flash_attn import (flash_attn_varlen_func,
16 +from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
17 + flash_attn_varlen_func,
18 is_fa_version_supported)
19
20 +logger = init_logger(__name__)
21 +
22
23 class FlashAttentionBackend(AttentionBackend):
24
@@ -143,6 +147,11 @@ class FlashAttentionImpl(AttentionImpl):
147 assert VLLM_FLASH_ATTN_VERSION in [2, 3]
148 self.fa_version = VLLM_FLASH_ATTN_VERSION
149
150 + if not is_fa_version_supported(self.fa_version):
151 + logger.error("Cannot use FA version %d is not supported due to %s",
152 + self.fa_version,
153 + fa_version_unsupported_reason(self.fa_version))
154 +
155 assert is_fa_version_supported(self.fa_version)
156
157 def forward(