@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fe743b798d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[bugfix] fix early import of flash attention (#12959)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Feb 9, 2025 at 00:06 UTC fe743b798dfa56aea3e2cb7182365ba3495489ee
4 files changed +20 -19
vllm/attention/backends/flash_attn.py
+7 -6
@@ -14,8 +14,8 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14 AttentionMetadataBuilder,
15 AttentionType)
16 from vllm.attention.backends.utils import (
17 - PAD_SLOT_ID, VLLM_FLASH_ATTN_VERSION, CommonAttentionState,
18 - compute_slot_mapping, compute_slot_mapping_start_idx,
17 + PAD_SLOT_ID, CommonAttentionState, compute_slot_mapping,
18 + compute_slot_mapping_start_idx, get_flash_attn_version,
19 get_num_prefill_decode_query_kv_tokens, get_seq_len_block_table_args,
20 is_all_cross_attn_metadata_set, is_all_encoder_attn_metadata_set,
21 is_block_tables_empty)
@@ -640,6 +640,7 @@ class FlashAttentionImpl(AttentionImpl):
640 f"Head size {head_size} is not supported by FlashAttention. "
641 f"Supported head sizes are: {support_head_sizes}.")
642 self.attn_type = attn_type
643 + self.vllm_flash_attn_version = get_flash_attn_version()
644
645 def forward(
646 self,
@@ -759,7 +760,7 @@ class FlashAttentionImpl(AttentionImpl):
760 alibi_slopes=alibi_slopes,
761 softcap=logits_soft_cap,
762 out=prefill_output,
762 - fa_version=VLLM_FLASH_ATTN_VERSION,
763 + fa_version=self.vllm_flash_attn_version,
764 )
765 else:
766 # prefix-enabled attention
@@ -782,7 +783,7 @@ class FlashAttentionImpl(AttentionImpl):
783 block_table=prefill_meta.block_tables,
784 softcap=logits_soft_cap,
785 out=prefill_output,
785 - fa_version=VLLM_FLASH_ATTN_VERSION,
786 + fa_version=self.vllm_flash_attn_version,
787 )
788
789 if decode_meta := attn_metadata.decode_metadata:
@@ -811,7 +812,7 @@ class FlashAttentionImpl(AttentionImpl):
812 softcap=logits_soft_cap,
813 block_table=decode_meta.block_tables,
814 out=decode_output,
814 - fa_version=VLLM_FLASH_ATTN_VERSION,
815 + fa_version=self.vllm_flash_attn_version,
816 )
817 else:
818 # Use flash_attn_with_kvcache for normal decoding.
@@ -832,7 +833,7 @@ class FlashAttentionImpl(AttentionImpl):
833 alibi_slopes=alibi_slopes,
834 softcap=logits_soft_cap,
835 out=decode_output.unsqueeze(1),
835 - fa_version=VLLM_FLASH_ATTN_VERSION,
836 + fa_version=self.vllm_flash_attn_version,
837 )
838 return output
839
vllm/attention/backends/mla/utils.py
+3 -2
@@ -12,7 +12,7 @@ from vllm import envs
12 from vllm.attention.backends.abstract import (AttentionLayer,
13 AttentionMetadata,
14 MLAAttentionImpl, T)
15 -from vllm.attention.backends.utils import VLLM_FLASH_ATTN_VERSION
15 +from vllm.attention.backends.utils import get_flash_attn_version
16 from vllm.distributed import (get_tensor_model_parallel_world_size,
17 tensor_model_parallel_all_reduce)
18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -181,6 +181,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
181 self.q_proj = q_proj
182 self.kv_b_proj = kv_b_proj
183 self.o_proj = o_proj
184 + self.vllm_flash_attn_version = get_flash_attn_version()
185
186 def _v_up_proj_and_o_proj(self, x):
187 if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
@@ -515,7 +516,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
516 max_seqlen_k=max_prefill_seq_len,
517 softmax_scale=self.scale,
518 causal=True,
518 - fa_version=VLLM_FLASH_ATTN_VERSION,
519 + fa_version=self.vllm_flash_attn_version,
520 )
521 attn_output = attn_output\
522 .view(-1, self.num_heads, q.shape[-1])[..., :v.shape[-1]]\
vllm/attention/backends/utils.py
+6 -8
@@ -587,11 +587,11 @@ def get_num_prefill_decode_query_kv_tokens(
587 num_decode_query_tokens)
588
589
590 -try:
591 - from vllm.vllm_flash_attn.flash_attn_interface import (
592 - fa_version_unsupported_reason, is_fa_version_supported)
590 +def get_flash_attn_version():
591 + try:
592 + from vllm.vllm_flash_attn.flash_attn_interface import (
593 + fa_version_unsupported_reason, is_fa_version_supported)
594
594 - def flash_attn_version():
595 # if hopper default to FA3, otherwise stick to FA2 for now
596 # TODO(lucas): profile FA3 on ampere to see if it makes sense to
597 # use FA3 as default for both
@@ -610,7 +610,5 @@ try:
610
611 assert is_fa_version_supported(fa_version)
612 return fa_version
613 -
614 - VLLM_FLASH_ATTN_VERSION = flash_attn_version()
615 -except (ImportError, AssertionError):
616 - VLLM_FLASH_ATTN_VERSION = None
613 + except (ImportError, AssertionError):
614 + return None
vllm/v1/attention/backends/flash_attn.py
+4 -3
@@ -10,7 +10,7 @@ import triton.language as tl
10
11 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12 AttentionMetadata, AttentionType)
13 -from vllm.attention.backends.utils import VLLM_FLASH_ATTN_VERSION
13 +from vllm.attention.backends.utils import get_flash_attn_version
14 from vllm.logger import init_logger
15 from vllm.utils import cdiv
16 from vllm.vllm_flash_attn import flash_attn_varlen_func
@@ -132,6 +132,7 @@ class FlashAttentionImpl(AttentionImpl):
132 "encoder/decoder cross-attention "
133 "are not implemented for "
134 "FlashAttentionImpl")
135 + self.vllm_flash_attn_version = get_flash_attn_version()
136
137 def forward(
138 self,
@@ -205,7 +206,7 @@ class FlashAttentionImpl(AttentionImpl):
206 window_size=self.sliding_window,
207 block_table=attn_metadata.block_table,
208 softcap=self.logits_soft_cap,
208 - fa_version=VLLM_FLASH_ATTN_VERSION,
209 + fa_version=self.vllm_flash_attn_version,
210 )
211 return output
212
@@ -227,7 +228,7 @@ class FlashAttentionImpl(AttentionImpl):
228 logits_soft_cap=self.logits_soft_cap,
229 block_table=attn_metadata.block_table,
230 common_prefix_len=attn_metadata.common_prefix_len,
230 - fa_version=VLLM_FLASH_ATTN_VERSION,
231 + fa_version=self.vllm_flash_attn_version,
232 )
233 return output
234