@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c786e757fa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Attention] Use FA3 for MLA on Hopper (#12807)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>

Lucas Wilkinson committed Feb 6, 2025 at 06:43 UTC c786e757fae4519256e4ef88a7d4f56c3339d14d
4 files changed +51 -59
vllm/attention/backends/flash_attn.py
+11 -33
@@ -14,19 +14,16 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14 AttentionMetadataBuilder,
15 AttentionType)
16 from vllm.attention.backends.utils import (
17 - PAD_SLOT_ID, CommonAttentionState, compute_slot_mapping,
18 - compute_slot_mapping_start_idx, get_num_prefill_decode_query_kv_tokens,
19 - get_seq_len_block_table_args, is_all_cross_attn_metadata_set,
20 - is_all_encoder_attn_metadata_set, is_block_tables_empty)
21 -from vllm.envs import VLLM_FLASH_ATTN_VERSION
17 + PAD_SLOT_ID, VLLM_FLASH_ATTN_VERSION, CommonAttentionState,
18 + compute_slot_mapping, compute_slot_mapping_start_idx,
19 + get_num_prefill_decode_query_kv_tokens, get_seq_len_block_table_args,
20 + is_all_cross_attn_metadata_set, is_all_encoder_attn_metadata_set,
21 + is_block_tables_empty)
22 from vllm.logger import init_logger
23 from vllm.multimodal import MultiModalPlaceholderMap
24 -from vllm.platforms import current_platform
24 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
26 -from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
27 - flash_attn_varlen_func,
28 - flash_attn_with_kvcache,
29 - is_fa_version_supported)
25 +from vllm.vllm_flash_attn import (flash_attn_varlen_func,
26 + flash_attn_with_kvcache)
27
28 if TYPE_CHECKING:
29 from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -644,25 +641,6 @@ class FlashAttentionImpl(AttentionImpl):
641 f"Supported head sizes are: {support_head_sizes}.")
642 self.attn_type = attn_type
643
647 - # if hopper default to FA3, otherwise stick to FA2 for now
648 - # TODO(lucas): profile FA3 on ampere to see if it makes sense to
649 - # use FA3 as default for both
650 - if current_platform.get_device_capability()[0] >= 9:
651 - self.fa_version = 3 if is_fa_version_supported(3) else 2
652 - else:
653 - self.fa_version = 2
654 -
655 - if VLLM_FLASH_ATTN_VERSION is not None:
656 - assert VLLM_FLASH_ATTN_VERSION in [2, 3]
657 - self.fa_version = VLLM_FLASH_ATTN_VERSION
658 -
659 - if not is_fa_version_supported(self.fa_version):
660 - logger.error("Cannot use FA version %d is not supported due to %s",
661 - self.fa_version,
662 - fa_version_unsupported_reason(self.fa_version))
663 -
664 - assert is_fa_version_supported(self.fa_version)
665 -
644 def forward(
645 self,
646 layer: AttentionLayer,
@@ -781,7 +759,7 @@ class FlashAttentionImpl(AttentionImpl):
759 alibi_slopes=alibi_slopes,
760 softcap=logits_soft_cap,
761 out=prefill_output,
784 - fa_version=self.fa_version,
762 + fa_version=VLLM_FLASH_ATTN_VERSION,
763 )
764 else:
765 # prefix-enabled attention
@@ -804,7 +782,7 @@ class FlashAttentionImpl(AttentionImpl):
782 block_table=prefill_meta.block_tables,
783 softcap=logits_soft_cap,
784 out=prefill_output,
807 - fa_version=self.fa_version,
785 + fa_version=VLLM_FLASH_ATTN_VERSION,
786 )
787
788 if decode_meta := attn_metadata.decode_metadata:
@@ -833,7 +811,7 @@ class FlashAttentionImpl(AttentionImpl):
811 softcap=logits_soft_cap,
812 block_table=decode_meta.block_tables,
813 out=decode_output,
836 - fa_version=self.fa_version,
814 + fa_version=VLLM_FLASH_ATTN_VERSION,
815 )
816 else:
817 # Use flash_attn_with_kvcache for normal decoding.
@@ -854,7 +832,7 @@ class FlashAttentionImpl(AttentionImpl):
832 alibi_slopes=alibi_slopes,
833 softcap=logits_soft_cap,
834 out=decode_output.unsqueeze(1),
857 - fa_version=self.fa_version,
835 + fa_version=VLLM_FLASH_ATTN_VERSION,
836 )
837 return output
838
vllm/attention/backends/mla/utils.py
+2
@@ -12,6 +12,7 @@ from vllm import envs
12 from vllm.attention.backends.abstract import (AttentionLayer,
13 AttentionMetadata,
14 MLAAttentionImpl, T)
15 +from vllm.attention.backends.utils import VLLM_FLASH_ATTN_VERSION
16 from vllm.distributed import (get_tensor_model_parallel_world_size,
17 tensor_model_parallel_all_reduce)
18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -533,6 +534,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
534 max_seqlen_k=max_prefill_seq_len,
535 softmax_scale=self.scale,
536 causal=True,
537 + fa_version=VLLM_FLASH_ATTN_VERSION,
538 )
539 attn_output = attn_output\
540 .view(-1, self.num_heads, q.shape[-1])[..., :v.shape[-1]]\
vllm/attention/backends/utils.py
+34
@@ -8,12 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type, TypeVar, Union
8 import numpy as np
9 import torch
10
11 +from vllm import envs
12 from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder,
13 AttentionState)
14 from vllm.attention.backends.abstract import AttentionType
15 +from vllm.logger import logging
16 from vllm.multimodal import MultiModalPlaceholderMap
17 +from vllm.platforms import current_platform
18 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
19
20 +logger = logging.getLogger(__name__)
21 +
22 if TYPE_CHECKING:
23 from vllm.worker.model_runner_base import ModelRunnerBase
24
@@ -580,3 +585,32 @@ def get_num_prefill_decode_query_kv_tokens(
585
586 return (num_prefill_query_tokens, num_prefill_kv_tokens,
587 num_decode_query_tokens)
588 +
589 +
590 +try:
591 + from vllm.vllm_flash_attn.flash_attn_interface import (
592 + fa_version_unsupported_reason, is_fa_version_supported)
593 +
594 + def flash_attn_version():
595 + # if hopper default to FA3, otherwise stick to FA2 for now
596 + # TODO(lucas): profile FA3 on ampere to see if it makes sense to
597 + # use FA3 as default for both
598 + if current_platform.get_device_capability()[0] >= 9:
599 + fa_version = 3 if is_fa_version_supported(3) else 2
600 + else:
601 + fa_version = 2
602 +
603 + if envs.VLLM_FLASH_ATTN_VERSION is not None:
604 + assert envs.VLLM_FLASH_ATTN_VERSION in [2, 3]
605 + fa_version = envs.VLLM_FLASH_ATTN_VERSION
606 +
607 + if not is_fa_version_supported(fa_version):
608 + logger.error("Cannot use FA version %d is not supported due to %s",
609 + fa_version, fa_version_unsupported_reason(fa_version))
610 +
611 + assert is_fa_version_supported(fa_version)
612 + return fa_version
613 +
614 + VLLM_FLASH_ATTN_VERSION = flash_attn_version()
615 +except ImportError:
616 + VLLM_FLASH_ATTN_VERSION = None
vllm/v1/attention/backends/flash_attn.py
+4 -26
@@ -10,13 +10,10 @@ import triton.language as tl
10
11 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12 AttentionMetadata, AttentionType)
13 -from vllm.envs import VLLM_FLASH_ATTN_VERSION
13 +from vllm.attention.backends.utils import VLLM_FLASH_ATTN_VERSION
14 from vllm.logger import init_logger
15 -from vllm.platforms import current_platform
15 from vllm.utils import cdiv
17 -from vllm.vllm_flash_attn import (fa_version_unsupported_reason,
18 - flash_attn_varlen_func,
19 - is_fa_version_supported)
16 +from vllm.vllm_flash_attn import flash_attn_varlen_func
17
18 logger = init_logger(__name__)
19
@@ -136,25 +133,6 @@ class FlashAttentionImpl(AttentionImpl):
133 "are not implemented for "
134 "FlashAttentionImpl")
135
139 - # if hopper default to FA3, otherwise stick to FA2 for now
140 - # TODO(lucas): profile FA3 on ampere to see if it makes sense to
141 - # use FA3 as default for both
142 - if current_platform.get_device_capability()[0] >= 9:
143 - self.fa_version = 3 if is_fa_version_supported(3) else 2
144 - else:
145 - self.fa_version = 2
146 -
147 - if VLLM_FLASH_ATTN_VERSION is not None:
148 - assert VLLM_FLASH_ATTN_VERSION in [2, 3]
149 - self.fa_version = VLLM_FLASH_ATTN_VERSION
150 -
151 - if not is_fa_version_supported(self.fa_version):
152 - logger.error("Cannot use FA version %d is not supported due to %s",
153 - self.fa_version,
154 - fa_version_unsupported_reason(self.fa_version))
155 -
156 - assert is_fa_version_supported(self.fa_version)
157 -
136 def forward(
137 self,
138 layer: torch.nn.Module,
@@ -227,7 +205,7 @@ class FlashAttentionImpl(AttentionImpl):
205 window_size=self.sliding_window,
206 block_table=attn_metadata.block_table,
207 softcap=self.logits_soft_cap,
230 - fa_version=self.fa_version,
208 + fa_version=VLLM_FLASH_ATTN_VERSION,
209 )
210 return output
211
@@ -249,7 +227,7 @@ class FlashAttentionImpl(AttentionImpl):
227 logits_soft_cap=self.logits_soft_cap,
228 block_table=attn_metadata.block_table,
229 common_prefix_len=attn_metadata.common_prefix_len,
252 - fa_version=self.fa_version,
230 + fa_version=VLLM_FLASH_ATTN_VERSION,
231 )
232 return output
233