@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e20c92bb61
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Kernel] Move attn_type to Attention.__init__() (#11690)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>

Chen Zhang committed Jan 7, 2025 at 00:11 UTC e20c92bb618384ce8d0013e0c9ad273d0c23d65b
18 files changed +159 -201
tests/kernels/test_encoder_decoder_attn.py
+49 -51
@@ -13,8 +13,7 @@ import pytest
13 import torch
14
15 from tests.kernels.utils import *
16 -from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
17 - AttentionType)
16 +from vllm.attention import Attention, AttentionMetadata, AttentionType
17 from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
18 from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
19 global_force_attn_backend_context_manager)
@@ -64,6 +63,7 @@ class TestPoint(NamedTuple):
63 max_dec_seq_len: int
64 max_enc_seq_len: int
65 num_blocks: int
66 + attn_type: AttentionType
67
68
69 class TestResources(NamedTuple):
@@ -96,7 +96,6 @@ class TestResources(NamedTuple):
96 '''
97
98 scale: float
99 - attn_backend: AttentionBackend
99 attn: Attention
100 kv_cache: torch.Tensor
101
@@ -129,16 +128,17 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
128 '''
129
130 scale = float(1.0 / (test_pt.head_size**0.5))
132 - attn_backend = make_backend(test_pt.backend_name)
131 attn = Attention(
132 test_pt.num_heads,
133 test_pt.head_size,
134 scale=scale,
135 + prefix=f"{test_pt.attn_type}",
136 + attn_type=test_pt.attn_type,
137 )
138 if test_pt.num_blocks is None or test_pt.num_heads is None:
139 # Caller does not require a KV cache
140 return TestResources(
141 - scale, attn_backend, attn,
141 + scale, attn,
142 torch.tensor([], dtype=torch.float32, device=CUDA_DEVICE))
143
144 # Construct KV cache
@@ -148,7 +148,7 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
148 test_pt.block_size,
149 device=CUDA_DEVICE,
150 backend=test_pt.backend_name)
151 - return TestResources(scale, attn_backend, attn, kv_cache)
151 + return TestResources(scale, attn, kv_cache)
152
153
154 def _encoder_attn_setup(
@@ -193,6 +193,7 @@ def _encoder_attn_setup(
193 _,
194 max_q_seq_len,
195 _,
196 + _,
197 ) = test_pt
198
199 scale = test_rsrcs.scale
@@ -301,6 +302,7 @@ def _decoder_attn_setup(
302 max_q_seq_len,
303 _,
304 _,
305 + _,
306 ) = test_pt
307
308 scale = test_rsrcs.scale
@@ -488,6 +490,7 @@ def _enc_dec_cross_attn_setup_reuses_query(
490 max_decoder_seq_len,
491 max_encoder_seq_len,
492 _,
493 + _,
494 ) = test_pt
495
496 scale = test_rsrcs.scale
@@ -622,7 +625,6 @@ def _run_encoder_attention_test(
625 & attn_metadata
626 '''
627 assert attn_metadata.num_decode_tokens == 0
625 - attn_type = AttentionType.ENCODER
628 packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
629 assert packed_qkv is not None
630 with set_forward_context(attn_metadata, vllm_config):
@@ -635,14 +637,11 @@ def _run_encoder_attention_test(
637 # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
638 reshaped_query = packed_qkv.query.view(
639 -1, test_pt.num_heads * test_pt.head_size)
638 - return attn.forward(reshaped_query,
639 - packed_qkv.key,
640 - packed_qkv.value,
641 - torch.tensor([],
642 - dtype=torch.float32,
643 - device=packed_qkv.query.device),
644 - attn_metadata,
645 - attn_type=attn_type)
640 + return attn.forward(
641 + reshaped_query, packed_qkv.key, packed_qkv.value,
642 + torch.tensor([],
643 + dtype=torch.float32,
644 + device=packed_qkv.query.device), attn_metadata)
645
646
647 def _run_decoder_self_attention_test(
@@ -675,7 +674,6 @@ def _run_decoder_self_attention_test(
674 * Attention.forward() applied to packed_{query,key,value}, kv_cache
675 & attn_metadata
676 '''
678 - attn_type = AttentionType.DECODER
677 attn = test_rsrcs.attn
678 kv_cache = test_rsrcs.kv_cache
679 packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
@@ -690,12 +688,8 @@ def _run_decoder_self_attention_test(
688 # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
689 reshaped_query = packed_qkv.query.view(
690 -1, test_pt.num_heads * test_pt.head_size)
693 - return attn.forward(reshaped_query,
694 - packed_qkv.key,
695 - packed_qkv.value,
696 - kv_cache,
697 - attn_metadata,
698 - attn_type=attn_type)
691 + return attn.forward(reshaped_query, packed_qkv.key, packed_qkv.value,
692 + kv_cache, attn_metadata)
693
694
695 def _run_encoder_decoder_cross_attention_test(
@@ -742,7 +736,6 @@ def _run_encoder_decoder_cross_attention_test(
736 '''
737 assert decoder_test_params.packed_qkvo.packed_qkv is not None
738
745 - attn_type = AttentionType.ENCODER_DECODER
739 attn = test_rsrcs.attn
740 kv_cache = test_rsrcs.kv_cache
741 if cross_test_params is None:
@@ -762,12 +755,8 @@ def _run_encoder_decoder_cross_attention_test(
755 # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
756 reshaped_query = decoder_test_params.packed_qkvo.packed_qkv.query.view(
757 -1, test_pt.num_heads * test_pt.head_size)
765 - return attn.forward(reshaped_query,
766 - key,
767 - value,
768 - kv_cache,
769 - attn_metadata,
770 - attn_type=attn_type)
758 + return attn.forward(reshaped_query, key, value, kv_cache,
759 + attn_metadata)
760
761
762 @pytest.fixture(autouse=True)
@@ -839,7 +828,7 @@ def test_encoder_only(
828 # is not part of this test
829 test_pt = TestPoint(num_heads, head_size, attn_backend.name,
830 batch_size, block_size, max_dec_seq_len,
842 - max_enc_seq_len, 4096)
831 + max_enc_seq_len, 4096, AttentionType.ENCODER)
832
833 # Attention scale factor, attention backend instance, attention wrapper
834 # instance, KV cache init
@@ -855,7 +844,7 @@ def test_encoder_only(
844 # Shared prefill metadata structure
845
846 prephase_attn_metadata: AttentionMetadata = make_test_metadata(
858 - test_rsrcs.attn_backend,
847 + attn_backend,
848 True,
849 None,
850 decoder_test_params=None,
@@ -961,20 +950,29 @@ def test_e2e_enc_dec_attn(
950 # Note: KV cache size of 4096 is arbitrary & chosen intentionally
951 # to be more than necessary, since exceeding the kv cache size
952 # is not part of this test
964 - test_pt = TestPoint(num_heads, head_size, attn_backend.name,
965 - batch_size, block_size, max_dec_seq_len,
966 - max_enc_seq_len, 4096)
953 + enc_test_pt = TestPoint(num_heads, head_size, attn_backend.name,
954 + batch_size, block_size, max_dec_seq_len,
955 + max_enc_seq_len, 4096, AttentionType.ENCODER)
956 + enc_dec_test_pt = TestPoint(num_heads, head_size, attn_backend.name,
957 + batch_size, block_size, max_dec_seq_len,
958 + max_enc_seq_len, 4096,
959 + AttentionType.ENCODER_DECODER)
960 + dec_test_pt = TestPoint(num_heads, head_size, attn_backend.name,
961 + batch_size, block_size, max_dec_seq_len,
962 + max_enc_seq_len, 4096, AttentionType.DECODER)
963
964 # Attention scale factor, attention backend instance, attention wrapper
965 # instance, KV cache init
966 vllm_config = VllmConfig()
967 with set_current_vllm_config(vllm_config):
972 - test_rsrcs = _make_test_resources(test_pt)
968 + enc_test_rsrcs = _make_test_resources(enc_test_pt)
969 + enc_dec_test_rsrcs = _make_test_resources(enc_dec_test_pt)
970 + dec_test_rsrcs = _make_test_resources(dec_test_pt)
971
972 # Construct encoder attention test params (only used
973 # during prefill)
974
977 - enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
975 + enc_test_params = _encoder_attn_setup(enc_test_pt, enc_test_rsrcs)
976
977 # Construct Decoder self-attention prefill-phase & decode-phase
978 # test params, including query/key/value tensors, decoder self-attention
@@ -987,7 +985,7 @@ def test_e2e_enc_dec_attn(
985 prephase_dec_test_params,
986 decphase_dec_test_params,
987 cross_block_base_addr,
990 - ) = _decoder_attn_setup(test_pt, test_rsrcs)
988 + ) = _decoder_attn_setup(dec_test_pt, dec_test_rsrcs)
989
990 # Construct encoder/decoder cross-attention prefill-phase
991 # & decode-phase test params, including key/value tensors,
@@ -1000,14 +998,14 @@ def test_e2e_enc_dec_attn(
998 dec_qkv,
999 enc_test_params,
1000 prephase_dec_test_params,
1003 - test_pt,
1004 - test_rsrcs,
1001 + enc_dec_test_pt,
1002 + enc_dec_test_rsrcs,
1003 block_base_addr=cross_block_base_addr)
1004
1005 # Shared prefill metadata structure
1006 assert prephase_dec_test_params.packed_qkvo.packed_qkv is not None
1007 prephase_attn_metadata: AttentionMetadata = make_test_metadata(
1010 - test_rsrcs.attn_backend,
1008 + attn_backend,
1009 True,
1010 prephase_dec_test_params.packed_qkvo.packed_qkv.q_seq_lens,
1011 decoder_test_params=prephase_dec_test_params,
@@ -1017,10 +1015,10 @@ def test_e2e_enc_dec_attn(
1015
1016 # PREFILL: encoder attention
1017
1020 - enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
1018 + enc_pckd_act_out = _run_encoder_attention_test(enc_test_rsrcs.attn,
1019 enc_test_params,
1020 prephase_attn_metadata,
1023 - test_pt=test_pt,
1021 + test_pt=enc_test_pt,
1022 vllm_config=vllm_config)
1023
1024 # - Is encoder attention result correct?
@@ -1030,10 +1028,10 @@ def test_e2e_enc_dec_attn(
1028 # PREFILL: decoder self-attention test
1029
1030 prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
1033 - test_rsrcs,
1031 + dec_test_rsrcs,
1032 prephase_dec_test_params,
1033 prephase_attn_metadata,
1036 - test_pt=test_pt,
1034 + test_pt=dec_test_pt,
1035 vllm_config=vllm_config)
1036
1037 # - Is prefill decoder self-attention correct?
@@ -1044,11 +1042,11 @@ def test_e2e_enc_dec_attn(
1042 # PREFILL: encoder/decoder cross-attention test
1043
1044 prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
1047 - test_rsrcs,
1045 + enc_dec_test_rsrcs,
1046 prephase_dec_test_params,
1047 prephase_cross_test_params,
1048 prephase_attn_metadata,
1051 - test_pt=test_pt,
1049 + test_pt=enc_dec_test_pt,
1050 vllm_config=vllm_config)
1051
1052 # - Is prefill encoder/decoder cross-attention correct?
@@ -1059,7 +1057,7 @@ def test_e2e_enc_dec_attn(
1057 # DECODE: build decode-phase attention metadata
1058
1059 decphase_attn_metadata: AttentionMetadata = make_test_metadata(
1062 - test_rsrcs.attn_backend,
1060 + attn_backend,
1061 False,
1062 dec_qkv.q_seq_lens,
1063 decoder_test_params=decphase_dec_test_params,
@@ -1070,10 +1068,10 @@ def test_e2e_enc_dec_attn(
1068 # DECODE: decoder self-attention test
1069
1070 decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
1073 - test_rsrcs,
1071 + dec_test_rsrcs,
1072 decphase_dec_test_params,
1073 decphase_attn_metadata,
1076 - test_pt=test_pt,
1074 + test_pt=dec_test_pt,
1075 vllm_config=vllm_config)
1076
1077 # - Is decode-phase decoder self-attention correct?
@@ -1084,11 +1082,11 @@ def test_e2e_enc_dec_attn(
1082 # DECODE: encoder/decoder cross-attention test
1083
1084 decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
1087 - test_rsrcs,
1085 + enc_dec_test_rsrcs,
1086 decphase_dec_test_params,
1087 None,
1088 decphase_attn_metadata,
1091 - test_pt=test_pt,
1089 + test_pt=enc_dec_test_pt,
1090 vllm_config=vllm_config)
1091
1092 # - Is decode-phase encoder/decoder cross-attention correct?
tests/kernels/utils.py
+7 -5
@@ -13,6 +13,7 @@ from torch._prims_common import TensorLikeType
13
14 from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
15 from vllm.model_executor.layers.activation import SiluAndMul
16 +from vllm.platforms.interface import _Backend
17 from vllm.utils import (STR_BACKEND_ENV_VAR, STR_FLASH_ATTN_VAL,
18 STR_XFORMERS_ATTN_VAL, make_tensor_with_pad)
19
@@ -790,7 +791,7 @@ def make_block_tables_slot_mapping(
791
792
793 def make_test_metadata(
793 - attn_backend: AttentionBackend,
794 + attn_backend: _Backend,
795 is_prompt: bool,
796 seq_lens: Optional[List[int]],
797 decoder_test_params: Optional[PhaseTestParameters],
@@ -815,7 +816,7 @@ def make_test_metadata(
816
817 Arguments:
818
818 - * attn_backend: Backend for sourcing attention kernels
819 + * attn_backend_name: Backend for sourcing attention kernels
820 * is_prompt: prefill if True, o/w decode
821 * seq_lens: list of token counts for each sequence
822 * decoder_test_params: decoder self-attention test params;
@@ -882,6 +883,8 @@ def make_test_metadata(
883 # (kv_mmap)
884 cross_kv_mmap = cross_test_params.kv_mmap
885
886 + attn_backend_obj = make_backend(attn_backend.name)
887 +
888 if is_prompt:
889 # Prefill-phase scenario
890
@@ -902,8 +905,7 @@ def make_test_metadata(
905 context_lens,
906 encoder_seq_lens,
907 device=device)
905 -
906 - return attn_backend.make_metadata(
908 + return attn_backend_obj.make_metadata(
909 num_prefills=num_prefills,
910 slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),
911 multi_modal_placeholder_index_maps=None,
@@ -952,7 +954,7 @@ def make_test_metadata(
954 encoder_seq_lens,
955 device=device)
956
955 - return attn_backend.make_metadata(
957 + return attn_backend_obj.make_metadata(
958 num_prefills=num_prefills,
959 slot_mapping=kv_mmap.slot_mapping,
960 multi_modal_placeholder_index_maps=None,
vllm/attention/backends/abstract.py
+1 -1
@@ -233,6 +233,7 @@ class AttentionImpl(ABC, Generic[T]):
233 kv_cache_dtype: str = "auto",
234 blocksparse_params: Optional[Dict[str, Any]] = None,
235 logits_soft_cap: Optional[float] = None,
236 + attn_type: str = AttentionType.DECODER,
237 ) -> None:
238 raise NotImplementedError
239
@@ -246,7 +247,6 @@ class AttentionImpl(ABC, Generic[T]):
247 attn_metadata: T,
248 k_scale: float = 1.0,
249 v_scale: float = 1.0,
249 - attn_type: str = AttentionType.DECODER,
250 output: Optional[torch.Tensor] = None,
251 ) -> torch.Tensor:
252 raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+7 -7
@@ -300,6 +300,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
300 kv_cache_dtype: str,
301 blocksparse_params: Optional[Dict[str, Any]] = None,
302 logits_soft_cap: Optional[float] = None,
303 + attn_type: str = AttentionType.DECODER,
304 ) -> None:
305 assert blocksparse_params is not None
306 assert alibi_slopes is None, ValueError(
@@ -350,6 +351,12 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
351 active_head_range=self.blocksparse_params.active_head_range,
352 )
353
354 + if attn_type != AttentionType.DECODER:
355 + raise NotImplementedError("Encoder self-attention and "
356 + "encoder/decoder cross-attention "
357 + "are not implemented for "
358 + "BlocksparseFlashAttentionImpl")
359 +
360 def forward(
361 self,
362 query: torch.Tensor,
@@ -359,7 +366,6 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
366 attn_metadata: BlocksparseFlashAttentionMetadata,
367 k_scale: float = 1.0,
368 v_scale: float = 1.0,
362 - attn_type: str = AttentionType.DECODER,
369 output: Optional[torch.Tensor] = None,
370 ) -> torch.Tensor:
371 """Forward pass with FlashAttention and PagedAttention.
@@ -375,12 +381,6 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
381 Returns:
382 shape = [num_tokens, num_heads * head_size]
383 """
378 - if attn_type != AttentionType.DECODER:
379 - raise NotImplementedError("Encoder self-attention and "
380 - "encoder/decoder cross-attention "
381 - "are not implemented for "
382 - "BlocksparseFlashAttentionImpl")
383 -
384 num_tokens, hidden_size = query.shape
385 # Reshape the query, key, and value tensors.
386 query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/flash_attn.py
+3 -1
@@ -600,6 +600,7 @@ class FlashAttentionImpl(AttentionImpl):
600 kv_cache_dtype: str,
601 blocksparse_params: Optional[Dict[str, Any]] = None,
602 logits_soft_cap: Optional[float] = None,
603 + attn_type: str = AttentionType.DECODER,
604 ) -> None:
605 if blocksparse_params is not None:
606 raise ValueError(
@@ -627,6 +628,7 @@ class FlashAttentionImpl(AttentionImpl):
628 raise ValueError(
629 f"Head size {head_size} is not supported by FlashAttention. "
630 f"Supported head sizes are: {support_head_sizes}.")
631 + self.attn_type = attn_type
632
633 def forward(
634 self,
@@ -637,7 +639,6 @@ class FlashAttentionImpl(AttentionImpl):
639 attn_metadata: FlashAttentionMetadata,
640 k_scale: float = 1.0,
641 v_scale: float = 1.0,
640 - attn_type: str = AttentionType.DECODER,
642 output: Optional[torch.Tensor] = None,
643 ) -> torch.Tensor:
644 """Forward pass with FlashAttention.
@@ -659,6 +660,7 @@ class FlashAttentionImpl(AttentionImpl):
660
661 assert output is not None, "Output tensor must be provided."
662
663 + attn_type = self.attn_type
664 if (attn_type == AttentionType.ENCODER
665 and (not attn_metadata.is_all_encoder_attn_metadata_set)):
666 raise AttributeError("Encoder attention requires setting "
vllm/attention/backends/flashinfer.py
+7 -8
@@ -748,6 +748,7 @@ class FlashInferImpl(AttentionImpl):
748 kv_cache_dtype: str,
749 blocksparse_params: Optional[Dict[str, Any]] = None,
750 logits_soft_cap: Optional[float] = None,
751 + attn_type: str = AttentionType.DECODER,
752 ) -> None:
753 self.num_heads = num_heads
754 self.head_size = head_size
@@ -764,6 +765,12 @@ class FlashInferImpl(AttentionImpl):
765 assert self.num_heads % self.num_kv_heads == 0
766 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
767
768 + if attn_type != AttentionType.DECODER:
769 + raise NotImplementedError("Encoder self-attention and "
770 + "encoder/decoder cross-attention "
771 + "are not implemented for "
772 + "FlashInferImpl")
773 +
774 def forward(
775 self,
776 query: torch.Tensor,
@@ -773,18 +780,10 @@ class FlashInferImpl(AttentionImpl):
780 attn_metadata: FlashInferMetadata,
781 k_scale: float = 1.0,
782 v_scale: float = 1.0,
776 - attn_type: str = AttentionType.DECODER,
783 output: Optional[torch.Tensor] = None,
784 ) -> torch.Tensor:
785
786 # TODO: directly write to output tensor
781 -
782 - if attn_type != AttentionType.DECODER:
783 - raise NotImplementedError("Encoder self-attention and "
784 - "encoder/decoder cross-attention "
785 - "are not implemented for "
786 - "FlashInferImpl")
787 -
787 num_heads: int = self.num_heads
788 head_size: int = self.head_size
789 num_kv_heads: int = self.num_kv_heads
vllm/attention/backends/hpu_attn.py
+7 -6
@@ -102,6 +102,7 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
102 kv_cache_dtype: str,
103 blocksparse_params: Optional[Dict[str, Any]] = None,
104 max_seq_len: int = 4096,
105 + attn_type: str = AttentionType.DECODER,
106 ) -> None:
107 super(AttentionImpl, self).__init__()
108 self.kv_cache_dtype = kv_cache_dtype
@@ -143,6 +144,12 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
144 f"Head size {head_size} is not supported by PagedAttention. "
145 f"Supported head sizes are: {suppored_head_sizes}.")
146
147 + if attn_type != AttentionType.DECODER:
148 + raise NotImplementedError("Encoder self-attention and "
149 + "encoder/decoder cross-attention "
150 + "are not implemented for "
151 + "HPUAttentionImpl")
152 +
153 def forward(
154 self,
155 query: torch.Tensor,
@@ -152,7 +159,6 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
159 attn_metadata: HPUAttentionMetadata,
160 k_scale: float = 1.0,
161 v_scale: float = 1.0,
155 - attn_type: str = AttentionType.DECODER,
162 output: Optional[torch.Tensor] = None,
163 ) -> torch.Tensor:
164 """Forward pass with xFormers and PagedAttention.
@@ -166,11 +172,6 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
172 Returns:
173 shape = [num_tokens, num_heads * head_size]
174 """
169 - if attn_type != AttentionType.DECODER:
170 - raise NotImplementedError("Encoder self-attention and "
171 - "encoder/decoder cross-attention "
172 - "are not implemented for "
173 - "HPUAttentionImpl")
175 batch_size, seq_len, hidden_size = query.shape
176 _, seq_len_kv, _ = key.shape
177
vllm/attention/backends/ipex_attn.py
+6 -6
@@ -115,6 +115,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
115 kv_cache_dtype: str,
116 blocksparse_params: Optional[Dict[str, Any]] = None,
117 logits_soft_cap: Optional[float] = None,
118 + attn_type: str = AttentionType.DECODER,
119 ) -> None:
120 if blocksparse_params is not None:
121 raise ValueError(
@@ -146,6 +147,11 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
147 raise NotImplementedError(
148 "IPEX backend does not support FP8 KV cache. "
149 "Please use xFormers backend instead.")
150 + if attn_type != AttentionType.DECODER:
151 + raise NotImplementedError("Encoder self-attention and "
152 + "encoder/decoder cross-attention "
153 + "are not implemented for "
154 + "IpexAttnBackendImpl")
155
156 def split_kv_cache(
157 self,
@@ -172,7 +178,6 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
178 attn_metadata: IpexAttnMetadata, # type: ignore
179 k_scale: float = 1.0,
180 v_scale: float = 1.0,
175 - attn_type: str = AttentionType.DECODER,
181 output: Optional[torch.Tensor] = None,
182 ) -> torch.Tensor:
183 """Forward pass with IPEX varlen_attention and PagedAttention.
@@ -189,11 +194,6 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
194 shape = [num_tokens, num_heads * head_size]
195 """
196 assert k_scale == 1.0 and v_scale == 1.0
192 - if attn_type != AttentionType.DECODER:
193 - raise NotImplementedError("Encoder self-attention and "
194 - "encoder/decoder cross-attention "
195 - "are not implemented for "
196 - "IpexAttnBackendImpl")
197 num_tokens, hidden_size = query.shape
198 # Reshape the query, key, and value tensors.
199 query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/pallas.py
+7 -6
@@ -100,6 +100,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
100 kv_cache_dtype: str,
101 blocksparse_params: Optional[Dict[str, Any]] = None,
102 logits_soft_cap: Optional[float] = None,
103 + attn_type: str = AttentionType.DECODER,
104 ) -> None:
105 self.num_heads = num_heads
106 self.head_size = head_size
@@ -141,6 +142,12 @@ class PallasAttentionBackendImpl(AttentionImpl):
142 # megacore mode will be None.
143 self.megacore_mode = "batch"
144
145 + if attn_type != AttentionType.DECODER:
146 + raise NotImplementedError("Encoder self-attention and "
147 + "encoder/decoder cross-attention "
148 + "are not implemented for "
149 + "PallasAttentionBackendImpl")
150 +
151 def forward(
152 self,
153 query: torch.Tensor,
@@ -150,7 +157,6 @@ class PallasAttentionBackendImpl(AttentionImpl):
157 attn_metadata: PallasMetadata,
158 k_scale: float = 1.0,
159 v_scale: float = 1.0,
153 - attn_type: str = AttentionType.DECODER,
160 output: Optional[torch.Tensor] = None,
161 ) -> torch.Tensor:
162 """Forward pass with Pallas attention.
@@ -168,11 +174,6 @@ class PallasAttentionBackendImpl(AttentionImpl):
174 shape = [batch_size, seq_len, num_heads * head_size]
175 """
176 assert k_scale == 1.0 and v_scale == 1.0
171 - if attn_type != AttentionType.DECODER:
172 - raise NotImplementedError("Encoder self-attention and "
173 - "encoder/decoder cross-attention "
174 - "are not implemented for "
175 - "PallasAttentionBackendImpl")
177 batch_size, seq_len, hidden_size = query.shape
178 query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
179 key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
vllm/attention/backends/rocm_flash_attn.py
+7 -7
@@ -338,6 +338,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
338 kv_cache_dtype: str,
339 blocksparse_params: Optional[Dict[str, Any]] = None,
340 logits_soft_cap: Optional[float] = None,
341 + attn_type: str = AttentionType.DECODER,
342 ) -> None:
343 if blocksparse_params is not None:
344 raise ValueError(
@@ -397,6 +398,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
398 self.attn_func = _sdpa_attention
399 logger.debug("Using naive attention in ROCmBackend")
400
401 + if attn_type != AttentionType.DECODER:
402 + raise NotImplementedError("Encoder self-attention and "
403 + "encoder/decoder cross-attention "
404 + "are not implemented for "
405 + "ROCmFlashAttentionImpl")
406 +
407 def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
408 """torch.repeat_interleave(x, dim=1, repeats=n_rep)"""
409 tokens, n_kv_heads, head_dim = x.shape
@@ -414,7 +421,6 @@ class ROCmFlashAttentionImpl(AttentionImpl):
421 attn_metadata: ROCmFlashAttentionMetadata,
422 k_scale: float = 1.0,
423 v_scale: float = 1.0,
417 - attn_type: str = AttentionType.DECODER,
424 output: Optional[torch.Tensor] = None,
425 ) -> torch.Tensor:
426 """Forward pass with FlashAttention and PagedAttention.
@@ -432,12 +438,6 @@ class ROCmFlashAttentionImpl(AttentionImpl):
438 """
439 # Reminder: Please update docs/source/features/compatibility_matrix.md
440 # If the feature combo become valid
435 - if attn_type != AttentionType.DECODER:
436 - raise NotImplementedError("Encoder self-attention and "
437 - "encoder/decoder cross-attention "
438 - "are not implemented for "
439 - "ROCmFlashAttentionImpl")
440 -
441 num_tokens, hidden_size = query.shape
442 # Reshape the query, key, and value tensors.
443 query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/torch_sdpa.py
+3 -1
@@ -390,6 +390,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
390 kv_cache_dtype: str,
391 blocksparse_params: Optional[Dict[str, Any]] = None,
392 logits_soft_cap: Optional[float] = None,
393 + attn_type: str = AttentionType.DECODER,
394 ) -> None:
395 if blocksparse_params is not None:
396 raise ValueError(
@@ -421,6 +422,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
422 raise NotImplementedError(
423 "Torch SDPA backend does not support FP8 KV cache. "
424 "Please use xFormers backend instead.")
425 + self.attn_type = attn_type
426
427 def forward(
428 self,
@@ -431,7 +433,6 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
433 attn_metadata: TorchSDPAMetadata, # type: ignore
434 k_scale: float = 1.0,
435 v_scale: float = 1.0,
434 - attn_type: str = AttentionType.DECODER,
436 output: Optional[torch.Tensor] = None,
437 ) -> torch.Tensor:
438 """Forward pass with torch SDPA and PagedAttention.
@@ -448,6 +449,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
449 shape = [num_tokens, num_heads * head_size]
450 """
451 assert k_scale == 1.0 and v_scale == 1.0
452 + attn_type = self.attn_type
453 if (attn_type == AttentionType.ENCODER
454 and (not attn_metadata.is_all_encoder_attn_metadata_set)):
455 raise AttributeError("Encoder attention requires setting "
vllm/attention/backends/xformers.py
+4 -2
@@ -379,6 +379,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
379 kv_cache_dtype: str,
380 blocksparse_params: Optional[Dict[str, Any]] = None,
381 logits_soft_cap: Optional[float] = None,
382 + attn_type: str = AttentionType.DECODER,
383 ) -> None:
384 if blocksparse_params is not None:
385 raise ValueError(
@@ -405,6 +406,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
406 f"Head size {head_size} is not supported by PagedAttention. "
407 f"Supported head sizes are: {suppored_head_sizes}.")
408
409 + self.attn_type = attn_type
410 +
411 def forward(
412 self,
413 query: torch.Tensor,
@@ -414,7 +417,6 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
417 attn_metadata: "XFormersMetadata",
418 k_scale: float = 1.0,
419 v_scale: float = 1.0,
417 - attn_type: str = AttentionType.DECODER,
420 output: Optional[torch.Tensor] = None,
421 ) -> torch.Tensor:
422 """Forward pass with xFormers and PagedAttention.
@@ -468,7 +470,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
470 Returns:
471 shape = [num_tokens, num_heads * head_size]
472 """
471 -
473 + attn_type = self.attn_type
474 # Check that appropriate attention metadata attributes are
475 # selected for the desired attention type
476 if (attn_type == AttentionType.ENCODER
vllm/attention/layer.py
+10 -27
@@ -41,6 +41,7 @@ class Attention(nn.Module):
41 logits_soft_cap: Optional[float] = None,
42 per_layer_sliding_window: Optional[int] = None,
43 prefix: str = "",
44 + attn_type: str = AttentionType.DECODER,
45 ) -> None:
46 super().__init__()
47 if per_layer_sliding_window is not None:
@@ -96,7 +97,7 @@ class Attention(nn.Module):
97 impl_cls = attn_backend.get_impl_cls()
98 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
99 alibi_slopes, sliding_window, kv_cache_dtype,
99 - blocksparse_params, logits_soft_cap)
100 + blocksparse_params, logits_soft_cap, attn_type)
101 self.num_heads = num_heads
102 self.head_size = head_size
103 self.num_kv_heads = num_kv_heads
@@ -119,6 +120,7 @@ class Attention(nn.Module):
120 raise ValueError(f"Duplicate layer name: {prefix}")
121 compilation_config.static_forward_context[prefix] = self
122 self.layer_name = prefix
123 + self.attn_type = attn_type
124
125 def forward(
126 self,
@@ -127,18 +129,12 @@ class Attention(nn.Module):
129 value: torch.Tensor,
130 kv_cache: torch.Tensor,
131 attn_metadata: AttentionMetadata,
130 - attn_type: str = AttentionType.DECODER,
132 ) -> torch.Tensor:
133
134 if self.use_direct_call:
134 - return self.impl.forward(query,
135 - key,
136 - value,
137 - kv_cache,
138 - attn_metadata,
139 - self._k_scale,
140 - self._v_scale,
141 - attn_type=attn_type)
135 + return self.impl.forward(query, key, value, kv_cache,
136 + attn_metadata, self._k_scale,
137 + self._v_scale)
138 elif self.use_output:
139 output = torch.empty_like(query)
140 hidden_size = query.size(-1)
@@ -152,13 +148,11 @@ class Attention(nn.Module):
148 if value is not None:
149 value = value.view(-1, self.num_kv_heads, self.head_size)
150 torch.ops.vllm.unified_attention_with_output(
155 - query, key, value, output, kv_cache, attn_type,
156 - self.layer_name)
151 + query, key, value, output, kv_cache, self.layer_name)
152 return output.view(-1, hidden_size)
153 else:
154 return torch.ops.vllm.unified_attention(query, key, value,
160 - kv_cache, attn_type,
161 - self.layer_name)
155 + kv_cache, self.layer_name)
156
157 def extra_repr(self) -> str:
158 s = f"head_size={self.impl.head_size}" # type: ignore
@@ -237,20 +231,13 @@ def unified_attention(
231 key: torch.Tensor,
232 value: torch.Tensor,
233 kv_cache: torch.Tensor,
240 - attn_type: str,
234 layer_name: str,
235 ) -> torch.Tensor:
236 forward_context: ForwardContext = get_forward_context()
237 attn_metadata = forward_context.dynamic_forward_context
238 self = forward_context.static_forward_context[layer_name]
246 - return self.impl.forward(query,
247 - key,
248 - value,
249 - kv_cache,
250 - attn_metadata,
251 - self._k_scale,
252 - self._v_scale,
253 - attn_type=attn_type)
239 + return self.impl.forward(query, key, value, kv_cache, attn_metadata,
240 + self._k_scale, self._v_scale)
241
242
243 def unified_attention_fake(
@@ -258,7 +245,6 @@ def unified_attention_fake(
245 key: torch.Tensor,
246 value: torch.Tensor,
247 kv_cache: torch.Tensor,
261 - attn_type: str,
248 layer_name: str,
249 ) -> torch.Tensor:
250 return torch.empty_like(query).contiguous()
@@ -279,7 +265,6 @@ def unified_attention_with_output(
265 value: torch.Tensor,
266 output: torch.Tensor,
267 kv_cache: torch.Tensor,
282 - attn_type: str,
268 layer_name: str,
269 ) -> None:
270 forward_context: ForwardContext = get_forward_context()
@@ -292,7 +277,6 @@ def unified_attention_with_output(
277 attn_metadata,
278 self._k_scale,
279 self._v_scale,
295 - attn_type=attn_type,
280 output=output)
281
282
@@ -302,7 +286,6 @@ def unified_attention_with_output_fake(
286 value: torch.Tensor,
287 output: torch.Tensor,
288 kv_cache: torch.Tensor,
305 - attn_type: str,
289 layer_name: str,
290 ) -> None:
291 return
vllm/model_executor/models/bart.py
+11 -33
@@ -71,12 +71,8 @@ class BartLearnedPositionalEmbedding(VocabParallelEmbedding):
71 def forward(
72 self,
73 positions: torch.Tensor,
74 - attn_type: AttentionType,
74 ) -> torch.Tensor:
75 """`input_ids' shape is expected to be [bsz x seqlen]."""
77 -
78 - assert attn_type != AttentionType.ENCODER_DECODER
79 -
76 return super().forward(positions + self.offset)
77
78
@@ -180,7 +176,8 @@ class BartEncoderAttention(nn.Module):
176 num_kv_heads=self.num_kv_heads,
177 cache_config=cache_config,
178 quant_config=quant_config,
183 - prefix=f"{prefix}.attn")
179 + prefix=f"{prefix}.attn",
180 + attn_type=AttentionType.ENCODER)
181
182 def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
183 attn_metadata: AttentionMetadata) -> torch.Tensor:
@@ -189,12 +186,7 @@ class BartEncoderAttention(nn.Module):
186 qkv, _ = self.qkv_proj(hidden_states)
187 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
188
192 - attn_output = self.attn(q,
193 - k,
194 - v,
195 - kv_cache,
196 - attn_metadata,
197 - attn_type=AttentionType.ENCODER)
189 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
190
191 output, _ = self.out_proj(attn_output)
192 return output
@@ -264,7 +256,8 @@ class BartDecoderSelfAttention(nn.Module):
256 num_kv_heads=self.num_kv_heads,
257 cache_config=cache_config,
258 quant_config=quant_config,
267 - prefix=f"{prefix}.attn")
259 + prefix=f"{prefix}.attn",
260 + attn_type=AttentionType.DECODER)
261
262 def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
263 attn_metadata: AttentionMetadata) -> torch.Tensor:
@@ -273,12 +266,7 @@ class BartDecoderSelfAttention(nn.Module):
266 qkv, _ = self.qkv_proj(hidden_states)
267 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
268
276 - attn_output = self.attn(q,
277 - k,
278 - v,
279 - kv_cache,
280 - attn_metadata,
281 - attn_type=AttentionType.DECODER)
269 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
270
271 output, _ = self.out_proj(attn_output)
272 return output
@@ -348,7 +336,8 @@ class BartCrossAttention(nn.Module):
336 num_kv_heads=self.num_kv_heads,
337 cache_config=cache_config,
338 quant_config=quant_config,
351 - prefix=f"{prefix}.attn")
339 + prefix=f"{prefix}.attn",
340 + attn_type=AttentionType.ENCODER_DECODER)
341
342 def forward(
343 self,
@@ -372,12 +361,7 @@ class BartCrossAttention(nn.Module):
361 _, k, v = qkv_enc.split([self.q_size, self.kv_size, self.kv_size],
362 dim=-1)
363
375 - attn_output = self.attn(q,
376 - k,
377 - v,
378 - kv_cache,
379 - attn_metadata,
380 - attn_type=AttentionType.ENCODER_DECODER)
364 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
365
366 output, _ = self.out_proj(attn_output)
367 return output
@@ -644,10 +628,7 @@ class BartEncoder(nn.Module):
628 # retrieve input_ids and inputs_embeds
629 inputs_embeds = self.embed_tokens(input_ids)
630
647 - embed_pos = self.embed_positions(
648 - positions,
649 - AttentionType.ENCODER,
650 - )
631 + embed_pos = self.embed_positions(positions)
632 embed_pos = embed_pos.to(inputs_embeds.device)
633
634 hidden_states = inputs_embeds + embed_pos
@@ -734,10 +715,7 @@ class BartDecoder(nn.Module):
715 inputs_embeds = self.embed_tokens(decoder_input_ids)
716
717 # embed positions
737 - embed_pos = self.embed_positions(
738 - decoder_positions,
739 - AttentionType.DECODER,
740 - )
718 + embed_pos = self.embed_positions(decoder_positions)
719 embed_pos = embed_pos.to(inputs_embeds.device)
720
721 hidden_states = inputs_embeds + embed_pos
vllm/model_executor/models/bert.py
+3 -7
@@ -238,7 +238,8 @@ class BertSelfAttention(nn.Module):
238 num_kv_heads=self.num_kv_heads,
239 cache_config=cache_config,
240 quant_config=quant_config,
241 - prefix=f"{prefix}.attn")
241 + prefix=f"{prefix}.attn",
242 + attn_type=AttentionType.ENCODER_ONLY)
243
244 def forward(
245 self,
@@ -248,12 +249,7 @@ class BertSelfAttention(nn.Module):
249 ) -> torch.Tensor:
250 qkv, _ = self.qkv_proj(hidden_states)
251 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
251 - output = self.attn(q,
252 - k,
253 - v,
254 - kv_cache,
255 - attn_metadata,
256 - attn_type=AttentionType.ENCODER_ONLY)
252 + output = self.attn(q, k, v, kv_cache, attn_metadata)
253 return output
254
255
vllm/model_executor/models/mllama.py
+4 -7
@@ -770,6 +770,7 @@ class MllamaTextCrossAttention(nn.Module):
770 self.scaling,
771 self.num_local_key_value_heads,
772 prefix=f"{prefix}.attn",
773 + attn_type=AttentionType.ENCODER_DECODER,
774 )
775
776 def forward(
@@ -805,13 +806,9 @@ class MllamaTextCrossAttention(nn.Module):
806 kv_range_for_decode,
807 attn_metadata)
808 else:
808 - output = self.attn(q.view(-1,
809 - self.num_local_heads * self.head_dim),
810 - k,
811 - v,
812 - kv_cache,
813 - attn_metadata,
814 - attn_type=AttentionType.ENCODER_DECODER)
809 + output = self.attn(
810 + q.view(-1, self.num_local_heads * self.head_dim), k, v,
811 + kv_cache, attn_metadata)
812 out, _ = self.o_proj(output)
813 return out
814
vllm/model_executor/models/qwen2.py
+16 -19
@@ -107,7 +107,8 @@ class Qwen2Attention(nn.Module):
107 cache_config: Optional[CacheConfig] = None,
108 quant_config: Optional[QuantizationConfig] = None,
109 rope_scaling: Optional[Tuple] = None,
110 - prefix: str = "") -> None:
110 + prefix: str = "",
111 + attn_type: str = AttentionType.DECODER) -> None:
112 super().__init__()
113 self.hidden_size = hidden_size
114 tp_size = get_tensor_model_parallel_world_size()
@@ -160,7 +161,8 @@ class Qwen2Attention(nn.Module):
161 num_kv_heads=self.num_kv_heads,
162 cache_config=cache_config,
163 quant_config=quant_config,
163 - prefix=f"{prefix}.attn")
164 + prefix=f"{prefix}.attn",
165 + attn_type=attn_type)
166
167 def forward(
168 self,
@@ -168,17 +170,11 @@ class Qwen2Attention(nn.Module):
170 hidden_states: torch.Tensor,
171 kv_cache: torch.Tensor,
172 attn_metadata: AttentionMetadata,
171 - attn_type: str = AttentionType.DECODER,
173 ) -> torch.Tensor:
174 qkv, _ = self.qkv_proj(hidden_states)
175 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
176 q, k = self.rotary_emb(positions, q, k)
176 - attn_output = self.attn(q,
177 - k,
178 - v,
179 - kv_cache,
180 - attn_metadata,
181 - attn_type=attn_type)
177 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
178 output, _ = self.o_proj(attn_output)
179 return output
180
@@ -197,6 +193,16 @@ class Qwen2DecoderLayer(nn.Module):
193 # Requires transformers > 4.32.0
194 rope_theta = getattr(config, "rope_theta", 1000000)
195 rope_scaling = getattr(config, "rope_scaling", None)
196 +
197 + # By default, Qwen2 uses causal attention as it is a decoder-only model.
198 + # You can override the HF config with `is_causal=False` to enable
199 + # bidirectional attention, which is used in some embedding models
200 + # (e.g. Alibaba-NLP/gte-Qwen2-7B-instruct)
201 + if getattr(config, "is_causal", True):
202 + attn_type = AttentionType.DECODER
203 + else:
204 + attn_type = AttentionType.ENCODER_ONLY
205 +
206 self.self_attn = Qwen2Attention(
207 hidden_size=self.hidden_size,
208 num_heads=config.num_attention_heads,
@@ -207,6 +213,7 @@ class Qwen2DecoderLayer(nn.Module):
213 quant_config=quant_config,
214 rope_scaling=rope_scaling,
215 prefix=f"{prefix}.self_attn",
216 + attn_type=attn_type,
217 )
218 self.mlp = Qwen2MLP(
219 hidden_size=self.hidden_size,
@@ -220,15 +227,6 @@ class Qwen2DecoderLayer(nn.Module):
227 self.post_attention_layernorm = RMSNorm(config.hidden_size,
228 eps=config.rms_norm_eps)
229
223 - # By default, Qwen2 uses causal attention as it is a decoder-only model.
224 - # You can override the HF config with `is_causal=False` to enable
225 - # bidirectional attention, which is used in some embedding models
226 - # (e.g. Alibaba-NLP/gte-Qwen2-7B-instruct)
227 - if getattr(config, "is_causal", True):
228 - self._attn_type = AttentionType.DECODER
229 - else:
230 - self._attn_type = AttentionType.ENCODER_ONLY
231 -
230 def forward(
231 self,
232 positions: torch.Tensor,
@@ -249,7 +247,6 @@ class Qwen2DecoderLayer(nn.Module):
247 hidden_states=hidden_states,
248 kv_cache=kv_cache,
249 attn_metadata=attn_metadata,
252 - attn_type=self._attn_type,
250 )
251
252 # Fully Connected
vllm/v1/attention/backends/flash_attn.py
+7 -7
@@ -89,6 +89,7 @@ class FlashAttentionImpl(AttentionImpl):
89 kv_cache_dtype: str,
90 blocksparse_params: Optional[Dict[str, Any]] = None,
91 logits_soft_cap: Optional[float] = None,
92 + attn_type: AttentionType = AttentionType.DECODER,
93 ) -> None:
94 if blocksparse_params is not None:
95 raise ValueError(
@@ -119,6 +120,12 @@ class FlashAttentionImpl(AttentionImpl):
120 f"Head size {head_size} is not supported by FlashAttention. "
121 f"Supported head sizes are: {support_head_sizes}.")
122
123 + if attn_type != AttentionType.DECODER:
124 + raise NotImplementedError("Encoder self-attention and "
125 + "encoder/decoder cross-attention "
126 + "are not implemented for "
127 + "FlashAttentionImpl")
128 +
129 def forward(
130 self,
131 query: torch.Tensor,
@@ -128,7 +135,6 @@ class FlashAttentionImpl(AttentionImpl):
135 attn_metadata: FlashAttentionMetadata,
136 k_scale: float = 1.0,
137 v_scale: float = 1.0,
131 - attn_type: AttentionType = AttentionType.DECODER,
138 output: Optional[torch.Tensor] = None,
139 ) -> torch.Tensor:
140 """Forward pass with FlashAttention.
@@ -142,12 +148,6 @@ class FlashAttentionImpl(AttentionImpl):
148 Returns:
149 shape = [num_tokens, num_heads * head_size]
150 """
145 - if attn_type != AttentionType.DECODER:
146 - raise NotImplementedError("Encoder self-attention and "
147 - "encoder/decoder cross-attention "
148 - "are not implemented for "
149 - "FlashAttentionImpl")
150 -
151 # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
152 assert k_scale == 1.0 and v_scale == 1.0, (
153 "key/v_scale is not supported in FlashAttention.")