A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[bugfix] fix early import of flash attention (#12959)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Feb 9, 2025 at 00:06 UTC
fe743b798dfa56aea3e2cb7182365ba3495489ee
4 files changed
+20
-19
vllm/attention/backends/flash_attn.py
+7
-6
@@ -14,8 +14,8 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
AttentionMetadataBuilder,
15
AttentionType)
16
from vllm.attention.backends.utils import (
17
- PAD_SLOT_ID, VLLM_FLASH_ATTN_VERSION, CommonAttentionState,
18
- compute_slot_mapping, compute_slot_mapping_start_idx,
17
+ PAD_SLOT_ID, CommonAttentionState, compute_slot_mapping,
18
+ compute_slot_mapping_start_idx, get_flash_attn_version,
19
get_num_prefill_decode_query_kv_tokens, get_seq_len_block_table_args,
20
is_all_cross_attn_metadata_set, is_all_encoder_attn_metadata_set,
21
is_block_tables_empty)
@@ -640,6 +640,7 @@ class FlashAttentionImpl(AttentionImpl):
640
f"Head size {head_size} is not supported by FlashAttention. "
641
f"Supported head sizes are: {support_head_sizes}.")
642
self.attn_type = attn_type
643
+ self.vllm_flash_attn_version = get_flash_attn_version()
644
645
def forward(
646
self,
@@ -759,7 +760,7 @@ class FlashAttentionImpl(AttentionImpl):
760
alibi_slopes=alibi_slopes,
761
softcap=logits_soft_cap,
762
out=prefill_output,
762
- fa_version=VLLM_FLASH_ATTN_VERSION,
763
+ fa_version=self.vllm_flash_attn_version,
764
)
765
else:
766
# prefix-enabled attention
@@ -782,7 +783,7 @@ class FlashAttentionImpl(AttentionImpl):
783
block_table=prefill_meta.block_tables,
784
softcap=logits_soft_cap,
785
out=prefill_output,
785
- fa_version=VLLM_FLASH_ATTN_VERSION,
786
+ fa_version=self.vllm_flash_attn_version,
787
)
788
789
if decode_meta := attn_metadata.decode_metadata:
@@ -811,7 +812,7 @@ class FlashAttentionImpl(AttentionImpl):
812
softcap=logits_soft_cap,
813
block_table=decode_meta.block_tables,
814
out=decode_output,
814
- fa_version=VLLM_FLASH_ATTN_VERSION,
815
+ fa_version=self.vllm_flash_attn_version,
816
)
817
else:
818
# Use flash_attn_with_kvcache for normal decoding.
@@ -832,7 +833,7 @@ class FlashAttentionImpl(AttentionImpl):
833
alibi_slopes=alibi_slopes,
834
softcap=logits_soft_cap,
835
out=decode_output.unsqueeze(1),
835
- fa_version=VLLM_FLASH_ATTN_VERSION,
836
+ fa_version=self.vllm_flash_attn_version,
837
)
838
return output
839
vllm/attention/backends/mla/utils.py
+3
-2
@@ -12,7 +12,7 @@ from vllm import envs
12
from vllm.attention.backends.abstract import (AttentionLayer,
13
AttentionMetadata,
14
MLAAttentionImpl, T)
15
-from vllm.attention.backends.utils import VLLM_FLASH_ATTN_VERSION
15
+from vllm.attention.backends.utils import get_flash_attn_version
16
from vllm.distributed import (get_tensor_model_parallel_world_size,
17
tensor_model_parallel_all_reduce)
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -181,6 +181,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
181
self.q_proj = q_proj
182
self.kv_b_proj = kv_b_proj
183
self.o_proj = o_proj
184
+ self.vllm_flash_attn_version = get_flash_attn_version()
185
186
def _v_up_proj_and_o_proj(self, x):
187
if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
@@ -515,7 +516,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
516
max_seqlen_k=max_prefill_seq_len,
517
softmax_scale=self.scale,
518
causal=True,
518
- fa_version=VLLM_FLASH_ATTN_VERSION,
519
+ fa_version=self.vllm_flash_attn_version,
520
)
521
attn_output = attn_output\
522
.view(-1, self.num_heads, q.shape[-1])[..., :v.shape[-1]]\
vllm/attention/backends/utils.py
+6
-8
@@ -587,11 +587,11 @@ def get_num_prefill_decode_query_kv_tokens(
587
num_decode_query_tokens)
588
589
590
-try:
591
- from vllm.vllm_flash_attn.flash_attn_interface import (
592
- fa_version_unsupported_reason, is_fa_version_supported)
590
+def get_flash_attn_version():
591
+ try:
592
+ from vllm.vllm_flash_attn.flash_attn_interface import (
593
+ fa_version_unsupported_reason, is_fa_version_supported)
594
594
- def flash_attn_version():
595
# if hopper default to FA3, otherwise stick to FA2 for now
596
# TODO(lucas): profile FA3 on ampere to see if it makes sense to
597
# use FA3 as default for both
@@ -610,7 +610,5 @@ try:
610
611
assert is_fa_version_supported(fa_version)
612
return fa_version
613
-
614
- VLLM_FLASH_ATTN_VERSION = flash_attn_version()
615
-except (ImportError, AssertionError):
616
- VLLM_FLASH_ATTN_VERSION = None
613
+ except (ImportError, AssertionError):
614
+ return None
vllm/v1/attention/backends/flash_attn.py
+4
-3
@@ -10,7 +10,7 @@ import triton.language as tl
10
11
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12
AttentionMetadata, AttentionType)
13
-from vllm.attention.backends.utils import VLLM_FLASH_ATTN_VERSION
13
+from vllm.attention.backends.utils import get_flash_attn_version
14
from vllm.logger import init_logger
15
from vllm.utils import cdiv
16
from vllm.vllm_flash_attn import flash_attn_varlen_func
@@ -132,6 +132,7 @@ class FlashAttentionImpl(AttentionImpl):
132
"encoder/decoder cross-attention "
133
"are not implemented for "
134
"FlashAttentionImpl")
135
+ self.vllm_flash_attn_version = get_flash_attn_version()
136
137
def forward(
138
self,
@@ -205,7 +206,7 @@ class FlashAttentionImpl(AttentionImpl):
206
window_size=self.sliding_window,
207
block_table=attn_metadata.block_table,
208
softcap=self.logits_soft_cap,
208
- fa_version=VLLM_FLASH_ATTN_VERSION,
209
+ fa_version=self.vllm_flash_attn_version,
210
)
211
return output
212
@@ -227,7 +228,7 @@ class FlashAttentionImpl(AttentionImpl):
228
logits_soft_cap=self.logits_soft_cap,
229
block_table=attn_metadata.block_table,
230
common_prefix_len=attn_metadata.common_prefix_len,
230
- fa_version=VLLM_FLASH_ATTN_VERSION,
231
+ fa_version=self.vllm_flash_attn_version,
232
)
233
return output
234