A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Attention] Use FA3 for MLA on Hopper (#12807)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Lucas Wilkinson committed
Feb 6, 2025 at 06:43 UTC
c786e757fae4519256e4ef88a7d4f56c3339d14d
4 files changed
+51
-59
vllm/attention/backends/flash_attn.py
+11
-33
@@ -14,19 +14,16 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
AttentionMetadataBuilder,
15
AttentionType)
16
from vllm.attention.backends.utils import (
17
- PAD_SLOT_ID, CommonAttentionState, compute_slot_mapping,
18
- compute_slot_mapping_start_idx, get_num_prefill_decode_query_kv_tokens,
19
- get_seq_len_block_table_args, is_all_cross_attn_metadata_set,
20
- is_all_encoder_attn_metadata_set, is_block_tables_empty)
21
-from vllm.envs import VLLM_FLASH_ATTN_VERSION
17
+ PAD_SLOT_ID, VLLM_FLASH_ATTN_VERSION, CommonAttentionState,
18
+ compute_slot_mapping, compute_slot_mapping_start_idx,
19
+ get_num_prefill_decode_query_kv_tokens, get_seq_len_block_table_args,
20
+ is_all_cross_attn_metadata_set, is_all_encoder_attn_metadata_set,
21
+ is_block_tables_empty)
22
from vllm.logger import init_logger
23
from vllm.multimodal import MultiModalPlaceholderMap
24
-from vllm.platforms import current_platform
24
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
26
-from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
27
- flash_attn_varlen_func,
28
- flash_attn_with_kvcache,
29
- is_fa_version_supported)
25
+from vllm.vllm_flash_attn import (flash_attn_varlen_func,
26
+ flash_attn_with_kvcache)
27
28
if TYPE_CHECKING:
29
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -644,25 +641,6 @@ class FlashAttentionImpl(AttentionImpl):
641
f"Supported head sizes are: {support_head_sizes}.")
642
self.attn_type = attn_type
643
647
- # if hopper default to FA3, otherwise stick to FA2 for now
648
- # TODO(lucas): profile FA3 on ampere to see if it makes sense to
649
- # use FA3 as default for both
650
- if current_platform.get_device_capability()[0] >= 9:
651
- self.fa_version = 3 if is_fa_version_supported(3) else 2
652
- else:
653
- self.fa_version = 2
654
-
655
- if VLLM_FLASH_ATTN_VERSION is not None:
656
- assert VLLM_FLASH_ATTN_VERSION in [2, 3]
657
- self.fa_version = VLLM_FLASH_ATTN_VERSION
658
-
659
- if not is_fa_version_supported(self.fa_version):
660
- logger.error("Cannot use FA version %d is not supported due to %s",
661
- self.fa_version,
662
- fa_version_unsupported_reason(self.fa_version))
663
-
664
- assert is_fa_version_supported(self.fa_version)
665
-
644
def forward(
645
self,
646
layer: AttentionLayer,
@@ -781,7 +759,7 @@ class FlashAttentionImpl(AttentionImpl):
759
alibi_slopes=alibi_slopes,
760
softcap=logits_soft_cap,
761
out=prefill_output,
784
- fa_version=self.fa_version,
762
+ fa_version=VLLM_FLASH_ATTN_VERSION,
763
)
764
else:
765
# prefix-enabled attention
@@ -804,7 +782,7 @@ class FlashAttentionImpl(AttentionImpl):
782
block_table=prefill_meta.block_tables,
783
softcap=logits_soft_cap,
784
out=prefill_output,
807
- fa_version=self.fa_version,
785
+ fa_version=VLLM_FLASH_ATTN_VERSION,
786
)
787
788
if decode_meta := attn_metadata.decode_metadata:
@@ -833,7 +811,7 @@ class FlashAttentionImpl(AttentionImpl):
811
softcap=logits_soft_cap,
812
block_table=decode_meta.block_tables,
813
out=decode_output,
836
- fa_version=self.fa_version,
814
+ fa_version=VLLM_FLASH_ATTN_VERSION,
815
)
816
else:
817
# Use flash_attn_with_kvcache for normal decoding.
@@ -854,7 +832,7 @@ class FlashAttentionImpl(AttentionImpl):
832
alibi_slopes=alibi_slopes,
833
softcap=logits_soft_cap,
834
out=decode_output.unsqueeze(1),
857
- fa_version=self.fa_version,
835
+ fa_version=VLLM_FLASH_ATTN_VERSION,
836
)
837
return output
838
vllm/attention/backends/mla/utils.py
+2
@@ -12,6 +12,7 @@ from vllm import envs
12
from vllm.attention.backends.abstract import (AttentionLayer,
13
AttentionMetadata,
14
MLAAttentionImpl, T)
15
+from vllm.attention.backends.utils import VLLM_FLASH_ATTN_VERSION
16
from vllm.distributed import (get_tensor_model_parallel_world_size,
17
tensor_model_parallel_all_reduce)
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -533,6 +534,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
534
max_seqlen_k=max_prefill_seq_len,
535
softmax_scale=self.scale,
536
causal=True,
537
+ fa_version=VLLM_FLASH_ATTN_VERSION,
538
)
539
attn_output = attn_output\
540
.view(-1, self.num_heads, q.shape[-1])[..., :v.shape[-1]]\
vllm/attention/backends/utils.py
+34
@@ -8,12 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type, TypeVar, Union
8
import numpy as np
9
import torch
10
11
+from vllm import envs
12
from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder,
13
AttentionState)
14
from vllm.attention.backends.abstract import AttentionType
15
+from vllm.logger import logging
16
from vllm.multimodal import MultiModalPlaceholderMap
17
+from vllm.platforms import current_platform
18
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
19
20
+logger = logging.getLogger(__name__)
21
+
22
if TYPE_CHECKING:
23
from vllm.worker.model_runner_base import ModelRunnerBase
24
@@ -580,3 +585,32 @@ def get_num_prefill_decode_query_kv_tokens(
585
586
return (num_prefill_query_tokens, num_prefill_kv_tokens,
587
num_decode_query_tokens)
588
+
589
+
590
+try:
591
+ from vllm.vllm_flash_attn.flash_attn_interface import (
592
+ fa_version_unsupported_reason, is_fa_version_supported)
593
+
594
+ def flash_attn_version():
595
+ # if hopper default to FA3, otherwise stick to FA2 for now
596
+ # TODO(lucas): profile FA3 on ampere to see if it makes sense to
597
+ # use FA3 as default for both
598
+ if current_platform.get_device_capability()[0] >= 9:
599
+ fa_version = 3 if is_fa_version_supported(3) else 2
600
+ else:
601
+ fa_version = 2
602
+
603
+ if envs.VLLM_FLASH_ATTN_VERSION is not None:
604
+ assert envs.VLLM_FLASH_ATTN_VERSION in [2, 3]
605
+ fa_version = envs.VLLM_FLASH_ATTN_VERSION
606
+
607
+ if not is_fa_version_supported(fa_version):
608
+ logger.error("Cannot use FA version %d is not supported due to %s",
609
+ fa_version, fa_version_unsupported_reason(fa_version))
610
+
611
+ assert is_fa_version_supported(fa_version)
612
+ return fa_version
613
+
614
+ VLLM_FLASH_ATTN_VERSION = flash_attn_version()
615
+except ImportError:
616
+ VLLM_FLASH_ATTN_VERSION = None
vllm/v1/attention/backends/flash_attn.py
+4
-26
@@ -10,13 +10,10 @@ import triton.language as tl
10
11
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12
AttentionMetadata, AttentionType)
13
-from vllm.envs import VLLM_FLASH_ATTN_VERSION
13
+from vllm.attention.backends.utils import VLLM_FLASH_ATTN_VERSION
14
from vllm.logger import init_logger
15
-from vllm.platforms import current_platform
15
from vllm.utils import cdiv
17
-from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
18
- flash_attn_varlen_func,
19
- is_fa_version_supported)
16
+from vllm.vllm_flash_attn import flash_attn_varlen_func
17
18
logger = init_logger(__name__)
19
@@ -136,25 +133,6 @@ class FlashAttentionImpl(AttentionImpl):
133
"are not implemented for "
134
"FlashAttentionImpl")
135
139
- # if hopper default to FA3, otherwise stick to FA2 for now
140
- # TODO(lucas): profile FA3 on ampere to see if it makes sense to
141
- # use FA3 as default for both
142
- if current_platform.get_device_capability()[0] >= 9:
143
- self.fa_version = 3 if is_fa_version_supported(3) else 2
144
- else:
145
- self.fa_version = 2
146
-
147
- if VLLM_FLASH_ATTN_VERSION is not None:
148
- assert VLLM_FLASH_ATTN_VERSION in [2, 3]
149
- self.fa_version = VLLM_FLASH_ATTN_VERSION
150
-
151
- if not is_fa_version_supported(self.fa_version):
152
- logger.error("Cannot use FA version %d is not supported due to %s",
153
- self.fa_version,
154
- fa_version_unsupported_reason(self.fa_version))
155
-
156
- assert is_fa_version_supported(self.fa_version)
157
-
136
def forward(
137
self,
138
layer: torch.nn.Module,
@@ -227,7 +205,7 @@ class FlashAttentionImpl(AttentionImpl):
205
window_size=self.sliding_window,
206
block_table=attn_metadata.block_table,
207
softcap=self.logits_soft_cap,
230
- fa_version=self.fa_version,
208
+ fa_version=VLLM_FLASH_ATTN_VERSION,
209
)
210
return output
211
@@ -249,7 +227,7 @@ class FlashAttentionImpl(AttentionImpl):
227
logits_soft_cap=self.logits_soft_cap,
228
block_table=attn_metadata.block_table,
229
common_prefix_len=attn_metadata.common_prefix_len,
252
- fa_version=self.fa_version,
230
+ fa_version=VLLM_FLASH_ATTN_VERSION,
231
)
232
return output
233