A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix][Kernel] Fix CUDA 11.8 being broken by FA3 build (#12375)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Lucas Wilkinson committed
Jan 24, 2025 at 10:27 UTC
ab5bbf5ae32bc438803ced4c2a021bb161e00050
6 files changed
+42
-22
CMakeLists.txt
+1
-1
@@ -576,7 +576,7 @@ else()
576
FetchContent_Declare(
577
vllm-flash-attn
578
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
579
- GIT_TAG 90eacc1af2a7c3de62ea249e929ed5faccf38954
579
+ GIT_TAG 0aff05f577e8a10086066a00618609199b25231d
580
GIT_PROGRESS TRUE
581
# Don't share the vllm-flash-attn build between build types
582
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
setup.py
+4
-1
@@ -598,7 +598,10 @@ if _is_hip():
598
599
if _is_cuda():
600
ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
601
- ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
601
+ if envs.VLLM_USE_PRECOMPILED or get_nvcc_cuda_version() >= Version("12.0"):
602
+ # FA3 requires CUDA 12.0 or later
603
+ ext_modules.append(
604
+ CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
605
ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
606
607
if _build_custom_ops():
tests/kernels/test_cascade_flash_attn.py
+6
-5
@@ -6,7 +6,9 @@ import torch
6
from vllm.platforms import current_platform
7
from vllm.v1.attention.backends.flash_attn import (cascade_attention,
8
merge_attn_states)
9
-from vllm.vllm_flash_attn import flash_attn_varlen_func
9
+from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
10
+ flash_attn_varlen_func,
11
+ is_fa_version_supported)
12
13
NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
14
HEAD_SIZES = [128, 192, 256]
@@ -91,10 +93,9 @@ def test_cascade(
93
fa_version: int,
94
) -> None:
95
torch.set_default_device("cuda")
94
- if fa_version == 3 and (torch.cuda.get_device_capability() == (8, 6)
95
- or torch.cuda.get_device_capability() == (8, 9)):
96
- pytest.skip("Flash attention version 3 fails on 8.6 and 8.9 due to "
97
- "insufficient shared memory for some shapes")
96
+ if not is_fa_version_supported(fa_version):
97
+ pytest.skip(f"Flash attention version {fa_version} not supported due "
98
+ f"to: \"{fa_version_unsupported_reason(fa_version)}\"")
99
100
current_platform.seed_everything(0)
101
tests/kernels/test_flash_attn.py
+10
-11
@@ -4,8 +4,10 @@ import pytest
4
import torch
5
6
from vllm.platforms import current_platform
7
-from vllm.vllm_flash_attn import (flash_attn_varlen_func,
8
- flash_attn_with_kvcache)
7
+from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
8
+ flash_attn_varlen_func,
9
+ flash_attn_with_kvcache,
10
+ is_fa_version_supported)
11
12
NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
13
HEAD_SIZES = [128, 256]
@@ -95,10 +97,9 @@ def test_flash_attn_with_paged_kv(
97
fa_version: int,
98
) -> None:
99
torch.set_default_device("cuda")
98
- if fa_version == 3 and (torch.cuda.get_device_capability() == (8, 6)
99
- or torch.cuda.get_device_capability() == (8, 9)):
100
- pytest.skip("Flash attention version 3 fails on 8.6 and 8.9 due to "
101
- "insufficient shared memory for some shapes")
100
+ if not is_fa_version_supported(fa_version):
101
+ pytest.skip(f"Flash attention version {fa_version} not supported due "
102
+ f"to: \"{fa_version_unsupported_reason(fa_version)}\"")
103
104
current_platform.seed_everything(0)
105
num_seqs = len(kv_lens)
@@ -182,11 +183,9 @@ def test_varlen_with_paged_kv(
183
fa_version: int,
184
) -> None:
185
torch.set_default_device("cuda")
185
- if fa_version == 3 and (torch.cuda.get_device_capability() == (8, 6)
186
- or torch.cuda.get_device_capability() == (8, 9)):
187
- pytest.skip("Flash attention version 3 fails on 8.6 and 8.9 due to "
188
- "insufficient shared memory for some shapes")
189
-
186
+ if not is_fa_version_supported(fa_version):
187
+ pytest.skip(f"Flash attention version {fa_version} not supported due "
188
+ f"to: \"{fa_version_unsupported_reason(fa_version)}\"")
189
current_platform.seed_everything(0)
190
num_seqs = len(seq_lens)
191
query_lens = [x[0] for x in seq_lens]
vllm/attention/backends/flash_attn.py
+11
-3
@@ -18,17 +18,20 @@ from vllm.attention.backends.utils import (
18
get_seq_len_block_table_args, is_all_cross_attn_metadata_set,
19
is_all_encoder_attn_metadata_set, is_block_tables_empty)
20
from vllm.envs import VLLM_FLASH_ATTN_VERSION
21
+from vllm.logger import init_logger
22
from vllm.multimodal import MultiModalPlaceholderMap
23
from vllm.platforms import current_platform
24
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
25
+from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
26
+ flash_attn_varlen_func,
27
+ flash_attn_with_kvcache,
28
+ is_fa_version_supported)
29
30
if TYPE_CHECKING:
31
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
32
ModelInputForGPUWithSamplingMetadata)
33
29
-from vllm.vllm_flash_attn import (flash_attn_varlen_func,
30
- flash_attn_with_kvcache,
31
- is_fa_version_supported)
34
+logger = init_logger(__name__)
35
36
37
class FlashAttentionBackend(AttentionBackend):
@@ -652,6 +655,11 @@ class FlashAttentionImpl(AttentionImpl):
655
assert VLLM_FLASH_ATTN_VERSION in [2, 3]
656
self.fa_version = VLLM_FLASH_ATTN_VERSION
657
658
+ if not is_fa_version_supported(self.fa_version):
659
+ logger.error("Cannot use FA version %d is not supported due to %s",
660
+ self.fa_version,
661
+ fa_version_unsupported_reason(self.fa_version))
662
+
663
assert is_fa_version_supported(self.fa_version)
664
665
def forward(
vllm/v1/attention/backends/flash_attn.py
+10
-1
@@ -10,11 +10,15 @@ import triton.language as tl
10
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11
AttentionMetadata, AttentionType)
12
from vllm.envs import VLLM_FLASH_ATTN_VERSION
13
+from vllm.logger import init_logger
14
from vllm.platforms import current_platform
15
from vllm.utils import cdiv
15
-from vllm.vllm_flash_attn import (flash_attn_varlen_func,
16
+from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
17
+ flash_attn_varlen_func,
18
is_fa_version_supported)
19
20
+logger = init_logger(__name__)
21
+
22
23
class FlashAttentionBackend(AttentionBackend):
24
@@ -143,6 +147,11 @@ class FlashAttentionImpl(AttentionImpl):
147
assert VLLM_FLASH_ATTN_VERSION in [2, 3]
148
self.fa_version = VLLM_FLASH_ATTN_VERSION
149
150
+ if not is_fa_version_supported(self.fa_version):
151
+ logger.error("Cannot use FA version %d is not supported due to %s",
152
+ self.fa_version,
153
+ fa_version_unsupported_reason(self.fa_version))
154
+
155
assert is_fa_version_supported(self.fa_version)
156
157
def forward(