A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Kernel] Move attn_type to Attention.__init__() (#11690)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Jan 7, 2025 at 00:11 UTC
e20c92bb618384ce8d0013e0c9ad273d0c23d65b
18 files changed
+159
-201
tests/kernels/test_encoder_decoder_attn.py
+49
-51
@@ -13,8 +13,7 @@ import pytest
13
import torch
14
15
from tests.kernels.utils import *
16
-from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
17
- AttentionType)
16
+from vllm.attention import Attention, AttentionMetadata, AttentionType
17
from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
18
from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
19
global_force_attn_backend_context_manager)
@@ -64,6 +63,7 @@ class TestPoint(NamedTuple):
63
max_dec_seq_len: int
64
max_enc_seq_len: int
65
num_blocks: int
66
+ attn_type: AttentionType
67
68
69
class TestResources(NamedTuple):
@@ -96,7 +96,6 @@ class TestResources(NamedTuple):
96
'''
97
98
scale: float
99
- attn_backend: AttentionBackend
99
attn: Attention
100
kv_cache: torch.Tensor
101
@@ -129,16 +128,17 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
128
'''
129
130
scale = float(1.0 / (test_pt.head_size**0.5))
132
- attn_backend = make_backend(test_pt.backend_name)
131
attn = Attention(
132
test_pt.num_heads,
133
test_pt.head_size,
134
scale=scale,
135
+ prefix=f"{test_pt.attn_type}",
136
+ attn_type=test_pt.attn_type,
137
)
138
if test_pt.num_blocks is None or test_pt.num_heads is None:
139
# Caller does not require a KV cache
140
return TestResources(
141
- scale, attn_backend, attn,
141
+ scale, attn,
142
torch.tensor([], dtype=torch.float32, device=CUDA_DEVICE))
143
144
# Construct KV cache
@@ -148,7 +148,7 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
148
test_pt.block_size,
149
device=CUDA_DEVICE,
150
backend=test_pt.backend_name)
151
- return TestResources(scale, attn_backend, attn, kv_cache)
151
+ return TestResources(scale, attn, kv_cache)
152
153
154
def _encoder_attn_setup(
@@ -193,6 +193,7 @@ def _encoder_attn_setup(
193
_,
194
max_q_seq_len,
195
_,
196
+ _,
197
) = test_pt
198
199
scale = test_rsrcs.scale
@@ -301,6 +302,7 @@ def _decoder_attn_setup(
302
max_q_seq_len,
303
_,
304
_,
305
+ _,
306
) = test_pt
307
308
scale = test_rsrcs.scale
@@ -488,6 +490,7 @@ def _enc_dec_cross_attn_setup_reuses_query(
490
max_decoder_seq_len,
491
max_encoder_seq_len,
492
_,
493
+ _,
494
) = test_pt
495
496
scale = test_rsrcs.scale
@@ -622,7 +625,6 @@ def _run_encoder_attention_test(
625
& attn_metadata
626
'''
627
assert attn_metadata.num_decode_tokens == 0
625
- attn_type = AttentionType.ENCODER
628
packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
629
assert packed_qkv is not None
630
with set_forward_context(attn_metadata, vllm_config):
@@ -635,14 +637,11 @@ def _run_encoder_attention_test(
637
# is shaped as [num_tokens, hidden_size] and we can skip the reshape.
638
reshaped_query = packed_qkv.query.view(
639
-1, test_pt.num_heads * test_pt.head_size)
638
- return attn.forward(reshaped_query,
639
- packed_qkv.key,
640
- packed_qkv.value,
641
- torch.tensor([],
642
- dtype=torch.float32,
643
- device=packed_qkv.query.device),
644
- attn_metadata,
645
- attn_type=attn_type)
640
+ return attn.forward(
641
+ reshaped_query, packed_qkv.key, packed_qkv.value,
642
+ torch.tensor([],
643
+ dtype=torch.float32,
644
+ device=packed_qkv.query.device), attn_metadata)
645
646
647
def _run_decoder_self_attention_test(
@@ -675,7 +674,6 @@ def _run_decoder_self_attention_test(
674
* Attention.forward() applied to packed_{query,key,value}, kv_cache
675
& attn_metadata
676
'''
678
- attn_type = AttentionType.DECODER
677
attn = test_rsrcs.attn
678
kv_cache = test_rsrcs.kv_cache
679
packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
@@ -690,12 +688,8 @@ def _run_decoder_self_attention_test(
688
# is shaped as [num_tokens, hidden_size] and we can skip the reshape.
689
reshaped_query = packed_qkv.query.view(
690
-1, test_pt.num_heads * test_pt.head_size)
693
- return attn.forward(reshaped_query,
694
- packed_qkv.key,
695
- packed_qkv.value,
696
- kv_cache,
697
- attn_metadata,
698
- attn_type=attn_type)
691
+ return attn.forward(reshaped_query, packed_qkv.key, packed_qkv.value,
692
+ kv_cache, attn_metadata)
693
694
695
def _run_encoder_decoder_cross_attention_test(
@@ -742,7 +736,6 @@ def _run_encoder_decoder_cross_attention_test(
736
'''
737
assert decoder_test_params.packed_qkvo.packed_qkv is not None
738
745
- attn_type = AttentionType.ENCODER_DECODER
739
attn = test_rsrcs.attn
740
kv_cache = test_rsrcs.kv_cache
741
if cross_test_params is None:
@@ -762,12 +755,8 @@ def _run_encoder_decoder_cross_attention_test(
755
# is shaped as [num_tokens, hidden_size] and we can skip the reshape.
756
reshaped_query = decoder_test_params.packed_qkvo.packed_qkv.query.view(
757
-1, test_pt.num_heads * test_pt.head_size)
765
- return attn.forward(reshaped_query,
766
- key,
767
- value,
768
- kv_cache,
769
- attn_metadata,
770
- attn_type=attn_type)
758
+ return attn.forward(reshaped_query, key, value, kv_cache,
759
+ attn_metadata)
760
761
762
@pytest.fixture(autouse=True)
@@ -839,7 +828,7 @@ def test_encoder_only(
828
# is not part of this test
829
test_pt = TestPoint(num_heads, head_size, attn_backend.name,
830
batch_size, block_size, max_dec_seq_len,
842
- max_enc_seq_len, 4096)
831
+ max_enc_seq_len, 4096, AttentionType.ENCODER)
832
833
# Attention scale factor, attention backend instance, attention wrapper
834
# instance, KV cache init
@@ -855,7 +844,7 @@ def test_encoder_only(
844
# Shared prefill metadata structure
845
846
prephase_attn_metadata: AttentionMetadata = make_test_metadata(
858
- test_rsrcs.attn_backend,
847
+ attn_backend,
848
True,
849
None,
850
decoder_test_params=None,
@@ -961,20 +950,29 @@ def test_e2e_enc_dec_attn(
950
# Note: KV cache size of 4096 is arbitrary & chosen intentionally
951
# to be more than necessary, since exceeding the kv cache size
952
# is not part of this test
964
- test_pt = TestPoint(num_heads, head_size, attn_backend.name,
965
- batch_size, block_size, max_dec_seq_len,
966
- max_enc_seq_len, 4096)
953
+ enc_test_pt = TestPoint(num_heads, head_size, attn_backend.name,
954
+ batch_size, block_size, max_dec_seq_len,
955
+ max_enc_seq_len, 4096, AttentionType.ENCODER)
956
+ enc_dec_test_pt = TestPoint(num_heads, head_size, attn_backend.name,
957
+ batch_size, block_size, max_dec_seq_len,
958
+ max_enc_seq_len, 4096,
959
+ AttentionType.ENCODER_DECODER)
960
+ dec_test_pt = TestPoint(num_heads, head_size, attn_backend.name,
961
+ batch_size, block_size, max_dec_seq_len,
962
+ max_enc_seq_len, 4096, AttentionType.DECODER)
963
964
# Attention scale factor, attention backend instance, attention wrapper
965
# instance, KV cache init
966
vllm_config = VllmConfig()
967
with set_current_vllm_config(vllm_config):
972
- test_rsrcs = _make_test_resources(test_pt)
968
+ enc_test_rsrcs = _make_test_resources(enc_test_pt)
969
+ enc_dec_test_rsrcs = _make_test_resources(enc_dec_test_pt)
970
+ dec_test_rsrcs = _make_test_resources(dec_test_pt)
971
972
# Construct encoder attention test params (only used
973
# during prefill)
974
977
- enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
975
+ enc_test_params = _encoder_attn_setup(enc_test_pt, enc_test_rsrcs)
976
977
# Construct Decoder self-attention prefill-phase & decode-phase
978
# test params, including query/key/value tensors, decoder self-attention
@@ -987,7 +985,7 @@ def test_e2e_enc_dec_attn(
985
prephase_dec_test_params,
986
decphase_dec_test_params,
987
cross_block_base_addr,
990
- ) = _decoder_attn_setup(test_pt, test_rsrcs)
988
+ ) = _decoder_attn_setup(dec_test_pt, dec_test_rsrcs)
989
990
# Construct encoder/decoder cross-attention prefill-phase
991
# & decode-phase test params, including key/value tensors,
@@ -1000,14 +998,14 @@ def test_e2e_enc_dec_attn(
998
dec_qkv,
999
enc_test_params,
1000
prephase_dec_test_params,
1003
- test_pt,
1004
- test_rsrcs,
1001
+ enc_dec_test_pt,
1002
+ enc_dec_test_rsrcs,
1003
block_base_addr=cross_block_base_addr)
1004
1005
# Shared prefill metadata structure
1006
assert prephase_dec_test_params.packed_qkvo.packed_qkv is not None
1007
prephase_attn_metadata: AttentionMetadata = make_test_metadata(
1010
- test_rsrcs.attn_backend,
1008
+ attn_backend,
1009
True,
1010
prephase_dec_test_params.packed_qkvo.packed_qkv.q_seq_lens,
1011
decoder_test_params=prephase_dec_test_params,
@@ -1017,10 +1015,10 @@ def test_e2e_enc_dec_attn(
1015
1016
# PREFILL: encoder attention
1017
1020
- enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
1018
+ enc_pckd_act_out = _run_encoder_attention_test(enc_test_rsrcs.attn,
1019
enc_test_params,
1020
prephase_attn_metadata,
1023
- test_pt=test_pt,
1021
+ test_pt=enc_test_pt,
1022
vllm_config=vllm_config)
1023
1024
# - Is encoder attention result correct?
@@ -1030,10 +1028,10 @@ def test_e2e_enc_dec_attn(
1028
# PREFILL: decoder self-attention test
1029
1030
prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
1033
- test_rsrcs,
1031
+ dec_test_rsrcs,
1032
prephase_dec_test_params,
1033
prephase_attn_metadata,
1036
- test_pt=test_pt,
1034
+ test_pt=dec_test_pt,
1035
vllm_config=vllm_config)
1036
1037
# - Is prefill decoder self-attention correct?
@@ -1044,11 +1042,11 @@ def test_e2e_enc_dec_attn(
1042
# PREFILL: encoder/decoder cross-attention test
1043
1044
prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
1047
- test_rsrcs,
1045
+ enc_dec_test_rsrcs,
1046
prephase_dec_test_params,
1047
prephase_cross_test_params,
1048
prephase_attn_metadata,
1051
- test_pt=test_pt,
1049
+ test_pt=enc_dec_test_pt,
1050
vllm_config=vllm_config)
1051
1052
# - Is prefill encoder/decoder cross-attention correct?
@@ -1059,7 +1057,7 @@ def test_e2e_enc_dec_attn(
1057
# DECODE: build decode-phase attention metadata
1058
1059
decphase_attn_metadata: AttentionMetadata = make_test_metadata(
1062
- test_rsrcs.attn_backend,
1060
+ attn_backend,
1061
False,
1062
dec_qkv.q_seq_lens,
1063
decoder_test_params=decphase_dec_test_params,
@@ -1070,10 +1068,10 @@ def test_e2e_enc_dec_attn(
1068
# DECODE: decoder self-attention test
1069
1070
decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
1073
- test_rsrcs,
1071
+ dec_test_rsrcs,
1072
decphase_dec_test_params,
1073
decphase_attn_metadata,
1076
- test_pt=test_pt,
1074
+ test_pt=dec_test_pt,
1075
vllm_config=vllm_config)
1076
1077
# - Is decode-phase decoder self-attention correct?
@@ -1084,11 +1082,11 @@ def test_e2e_enc_dec_attn(
1082
# DECODE: encoder/decoder cross-attention test
1083
1084
decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
1087
- test_rsrcs,
1085
+ enc_dec_test_rsrcs,
1086
decphase_dec_test_params,
1087
None,
1088
decphase_attn_metadata,
1091
- test_pt=test_pt,
1089
+ test_pt=enc_dec_test_pt,
1090
vllm_config=vllm_config)
1091
1092
# - Is decode-phase encoder/decoder cross-attention correct?
tests/kernels/utils.py
+7
-5
@@ -13,6 +13,7 @@ from torch._prims_common import TensorLikeType
13
14
from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
15
from vllm.model_executor.layers.activation import SiluAndMul
16
+from vllm.platforms.interface import _Backend
17
from vllm.utils import (STR_BACKEND_ENV_VAR, STR_FLASH_ATTN_VAL,
18
STR_XFORMERS_ATTN_VAL, make_tensor_with_pad)
19
@@ -790,7 +791,7 @@ def make_block_tables_slot_mapping(
791
792
793
def make_test_metadata(
793
- attn_backend: AttentionBackend,
794
+ attn_backend: _Backend,
795
is_prompt: bool,
796
seq_lens: Optional[List[int]],
797
decoder_test_params: Optional[PhaseTestParameters],
@@ -815,7 +816,7 @@ def make_test_metadata(
816
817
Arguments:
818
818
- * attn_backend: Backend for sourcing attention kernels
819
+ * attn_backend_name: Backend for sourcing attention kernels
820
* is_prompt: prefill if True, o/w decode
821
* seq_lens: list of token counts for each sequence
822
* decoder_test_params: decoder self-attention test params;
@@ -882,6 +883,8 @@ def make_test_metadata(
883
# (kv_mmap)
884
cross_kv_mmap = cross_test_params.kv_mmap
885
886
+ attn_backend_obj = make_backend(attn_backend.name)
887
+
888
if is_prompt:
889
# Prefill-phase scenario
890
@@ -902,8 +905,7 @@ def make_test_metadata(
905
context_lens,
906
encoder_seq_lens,
907
device=device)
905
-
906
- return attn_backend.make_metadata(
908
+ return attn_backend_obj.make_metadata(
909
num_prefills=num_prefills,
910
slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),
911
multi_modal_placeholder_index_maps=None,
@@ -952,7 +954,7 @@ def make_test_metadata(
954
encoder_seq_lens,
955
device=device)
956
955
- return attn_backend.make_metadata(
957
+ return attn_backend_obj.make_metadata(
958
num_prefills=num_prefills,
959
slot_mapping=kv_mmap.slot_mapping,
960
multi_modal_placeholder_index_maps=None,
vllm/attention/backends/abstract.py
+1
-1
@@ -233,6 +233,7 @@ class AttentionImpl(ABC, Generic[T]):
233
kv_cache_dtype: str = "auto",
234
blocksparse_params: Optional[Dict[str, Any]] = None,
235
logits_soft_cap: Optional[float] = None,
236
+ attn_type: str = AttentionType.DECODER,
237
) -> None:
238
raise NotImplementedError
239
@@ -246,7 +247,6 @@ class AttentionImpl(ABC, Generic[T]):
247
attn_metadata: T,
248
k_scale: float = 1.0,
249
v_scale: float = 1.0,
249
- attn_type: str = AttentionType.DECODER,
250
output: Optional[torch.Tensor] = None,
251
) -> torch.Tensor:
252
raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+7
-7
@@ -300,6 +300,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
300
kv_cache_dtype: str,
301
blocksparse_params: Optional[Dict[str, Any]] = None,
302
logits_soft_cap: Optional[float] = None,
303
+ attn_type: str = AttentionType.DECODER,
304
) -> None:
305
assert blocksparse_params is not None
306
assert alibi_slopes is None, ValueError(
@@ -350,6 +351,12 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
351
active_head_range=self.blocksparse_params.active_head_range,
352
)
353
354
+ if attn_type != AttentionType.DECODER:
355
+ raise NotImplementedError("Encoder self-attention and "
356
+ "encoder/decoder cross-attention "
357
+ "are not implemented for "
358
+ "BlocksparseFlashAttentionImpl")
359
+
360
def forward(
361
self,
362
query: torch.Tensor,
@@ -359,7 +366,6 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
366
attn_metadata: BlocksparseFlashAttentionMetadata,
367
k_scale: float = 1.0,
368
v_scale: float = 1.0,
362
- attn_type: str = AttentionType.DECODER,
369
output: Optional[torch.Tensor] = None,
370
) -> torch.Tensor:
371
"""Forward pass with FlashAttention and PagedAttention.
@@ -375,12 +381,6 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
381
Returns:
382
shape = [num_tokens, num_heads * head_size]
383
"""
378
- if attn_type != AttentionType.DECODER:
379
- raise NotImplementedError("Encoder self-attention and "
380
- "encoder/decoder cross-attention "
381
- "are not implemented for "
382
- "BlocksparseFlashAttentionImpl")
383
-
384
num_tokens, hidden_size = query.shape
385
# Reshape the query, key, and value tensors.
386
query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/flash_attn.py
+3
-1
@@ -600,6 +600,7 @@ class FlashAttentionImpl(AttentionImpl):
600
kv_cache_dtype: str,
601
blocksparse_params: Optional[Dict[str, Any]] = None,
602
logits_soft_cap: Optional[float] = None,
603
+ attn_type: str = AttentionType.DECODER,
604
) -> None:
605
if blocksparse_params is not None:
606
raise ValueError(
@@ -627,6 +628,7 @@ class FlashAttentionImpl(AttentionImpl):
628
raise ValueError(
629
f"Head size {head_size} is not supported by FlashAttention. "
630
f"Supported head sizes are: {support_head_sizes}.")
631
+ self.attn_type = attn_type
632
633
def forward(
634
self,
@@ -637,7 +639,6 @@ class FlashAttentionImpl(AttentionImpl):
639
attn_metadata: FlashAttentionMetadata,
640
k_scale: float = 1.0,
641
v_scale: float = 1.0,
640
- attn_type: str = AttentionType.DECODER,
642
output: Optional[torch.Tensor] = None,
643
) -> torch.Tensor:
644
"""Forward pass with FlashAttention.
@@ -659,6 +660,7 @@ class FlashAttentionImpl(AttentionImpl):
660
661
assert output is not None, "Output tensor must be provided."
662
663
+ attn_type = self.attn_type
664
if (attn_type == AttentionType.ENCODER
665
and (not attn_metadata.is_all_encoder_attn_metadata_set)):
666
raise AttributeError("Encoder attention requires setting "
vllm/attention/backends/flashinfer.py
+7
-8
@@ -748,6 +748,7 @@ class FlashInferImpl(AttentionImpl):
748
kv_cache_dtype: str,
749
blocksparse_params: Optional[Dict[str, Any]] = None,
750
logits_soft_cap: Optional[float] = None,
751
+ attn_type: str = AttentionType.DECODER,
752
) -> None:
753
self.num_heads = num_heads
754
self.head_size = head_size
@@ -764,6 +765,12 @@ class FlashInferImpl(AttentionImpl):
765
assert self.num_heads % self.num_kv_heads == 0
766
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
767
768
+ if attn_type != AttentionType.DECODER:
769
+ raise NotImplementedError("Encoder self-attention and "
770
+ "encoder/decoder cross-attention "
771
+ "are not implemented for "
772
+ "FlashInferImpl")
773
+
774
def forward(
775
self,
776
query: torch.Tensor,
@@ -773,18 +780,10 @@ class FlashInferImpl(AttentionImpl):
780
attn_metadata: FlashInferMetadata,
781
k_scale: float = 1.0,
782
v_scale: float = 1.0,
776
- attn_type: str = AttentionType.DECODER,
783
output: Optional[torch.Tensor] = None,
784
) -> torch.Tensor:
785
786
# TODO: directly write to output tensor
781
-
782
- if attn_type != AttentionType.DECODER:
783
- raise NotImplementedError("Encoder self-attention and "
784
- "encoder/decoder cross-attention "
785
- "are not implemented for "
786
- "FlashInferImpl")
787
-
787
num_heads: int = self.num_heads
788
head_size: int = self.head_size
789
num_kv_heads: int = self.num_kv_heads
vllm/attention/backends/hpu_attn.py
+7
-6
@@ -102,6 +102,7 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
102
kv_cache_dtype: str,
103
blocksparse_params: Optional[Dict[str, Any]] = None,
104
max_seq_len: int = 4096,
105
+ attn_type: str = AttentionType.DECODER,
106
) -> None:
107
super(AttentionImpl, self).__init__()
108
self.kv_cache_dtype = kv_cache_dtype
@@ -143,6 +144,12 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
144
f"Head size {head_size} is not supported by PagedAttention. "
145
f"Supported head sizes are: {suppored_head_sizes}.")
146
147
+ if attn_type != AttentionType.DECODER:
148
+ raise NotImplementedError("Encoder self-attention and "
149
+ "encoder/decoder cross-attention "
150
+ "are not implemented for "
151
+ "HPUAttentionImpl")
152
+
153
def forward(
154
self,
155
query: torch.Tensor,
@@ -152,7 +159,6 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
159
attn_metadata: HPUAttentionMetadata,
160
k_scale: float = 1.0,
161
v_scale: float = 1.0,
155
- attn_type: str = AttentionType.DECODER,
162
output: Optional[torch.Tensor] = None,
163
) -> torch.Tensor:
164
"""Forward pass with xFormers and PagedAttention.
@@ -166,11 +172,6 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
172
Returns:
173
shape = [num_tokens, num_heads * head_size]
174
"""
169
- if attn_type != AttentionType.DECODER:
170
- raise NotImplementedError("Encoder self-attention and "
171
- "encoder/decoder cross-attention "
172
- "are not implemented for "
173
- "HPUAttentionImpl")
175
batch_size, seq_len, hidden_size = query.shape
176
_, seq_len_kv, _ = key.shape
177
vllm/attention/backends/ipex_attn.py
+6
-6
@@ -115,6 +115,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
115
kv_cache_dtype: str,
116
blocksparse_params: Optional[Dict[str, Any]] = None,
117
logits_soft_cap: Optional[float] = None,
118
+ attn_type: str = AttentionType.DECODER,
119
) -> None:
120
if blocksparse_params is not None:
121
raise ValueError(
@@ -146,6 +147,11 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
147
raise NotImplementedError(
148
"IPEX backend does not support FP8 KV cache. "
149
"Please use xFormers backend instead.")
150
+ if attn_type != AttentionType.DECODER:
151
+ raise NotImplementedError("Encoder self-attention and "
152
+ "encoder/decoder cross-attention "
153
+ "are not implemented for "
154
+ "IpexAttnBackendImpl")
155
156
def split_kv_cache(
157
self,
@@ -172,7 +178,6 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
178
attn_metadata: IpexAttnMetadata, # type: ignore
179
k_scale: float = 1.0,
180
v_scale: float = 1.0,
175
- attn_type: str = AttentionType.DECODER,
181
output: Optional[torch.Tensor] = None,
182
) -> torch.Tensor:
183
"""Forward pass with IPEX varlen_attention and PagedAttention.
@@ -189,11 +194,6 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
194
shape = [num_tokens, num_heads * head_size]
195
"""
196
assert k_scale == 1.0 and v_scale == 1.0
192
- if attn_type != AttentionType.DECODER:
193
- raise NotImplementedError("Encoder self-attention and "
194
- "encoder/decoder cross-attention "
195
- "are not implemented for "
196
- "IpexAttnBackendImpl")
197
num_tokens, hidden_size = query.shape
198
# Reshape the query, key, and value tensors.
199
query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/pallas.py
+7
-6
@@ -100,6 +100,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
100
kv_cache_dtype: str,
101
blocksparse_params: Optional[Dict[str, Any]] = None,
102
logits_soft_cap: Optional[float] = None,
103
+ attn_type: str = AttentionType.DECODER,
104
) -> None:
105
self.num_heads = num_heads
106
self.head_size = head_size
@@ -141,6 +142,12 @@ class PallasAttentionBackendImpl(AttentionImpl):
142
# megacore mode will be None.
143
self.megacore_mode = "batch"
144
145
+ if attn_type != AttentionType.DECODER:
146
+ raise NotImplementedError("Encoder self-attention and "
147
+ "encoder/decoder cross-attention "
148
+ "are not implemented for "
149
+ "PallasAttentionBackendImpl")
150
+
151
def forward(
152
self,
153
query: torch.Tensor,
@@ -150,7 +157,6 @@ class PallasAttentionBackendImpl(AttentionImpl):
157
attn_metadata: PallasMetadata,
158
k_scale: float = 1.0,
159
v_scale: float = 1.0,
153
- attn_type: str = AttentionType.DECODER,
160
output: Optional[torch.Tensor] = None,
161
) -> torch.Tensor:
162
"""Forward pass with Pallas attention.
@@ -168,11 +174,6 @@ class PallasAttentionBackendImpl(AttentionImpl):
174
shape = [batch_size, seq_len, num_heads * head_size]
175
"""
176
assert k_scale == 1.0 and v_scale == 1.0
171
- if attn_type != AttentionType.DECODER:
172
- raise NotImplementedError("Encoder self-attention and "
173
- "encoder/decoder cross-attention "
174
- "are not implemented for "
175
- "PallasAttentionBackendImpl")
177
batch_size, seq_len, hidden_size = query.shape
178
query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
179
key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
vllm/attention/backends/rocm_flash_attn.py
+7
-7
@@ -338,6 +338,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
338
kv_cache_dtype: str,
339
blocksparse_params: Optional[Dict[str, Any]] = None,
340
logits_soft_cap: Optional[float] = None,
341
+ attn_type: str = AttentionType.DECODER,
342
) -> None:
343
if blocksparse_params is not None:
344
raise ValueError(
@@ -397,6 +398,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
398
self.attn_func = _sdpa_attention
399
logger.debug("Using naive attention in ROCmBackend")
400
401
+ if attn_type != AttentionType.DECODER:
402
+ raise NotImplementedError("Encoder self-attention and "
403
+ "encoder/decoder cross-attention "
404
+ "are not implemented for "
405
+ "ROCmFlashAttentionImpl")
406
+
407
def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
408
"""torch.repeat_interleave(x, dim=1, repeats=n_rep)"""
409
tokens, n_kv_heads, head_dim = x.shape
@@ -414,7 +421,6 @@ class ROCmFlashAttentionImpl(AttentionImpl):
421
attn_metadata: ROCmFlashAttentionMetadata,
422
k_scale: float = 1.0,
423
v_scale: float = 1.0,
417
- attn_type: str = AttentionType.DECODER,
424
output: Optional[torch.Tensor] = None,
425
) -> torch.Tensor:
426
"""Forward pass with FlashAttention and PagedAttention.
@@ -432,12 +438,6 @@ class ROCmFlashAttentionImpl(AttentionImpl):
438
"""
439
# Reminder: Please update docs/source/features/compatibility_matrix.md
440
# If the feature combo become valid
435
- if attn_type != AttentionType.DECODER:
436
- raise NotImplementedError("Encoder self-attention and "
437
- "encoder/decoder cross-attention "
438
- "are not implemented for "
439
- "ROCmFlashAttentionImpl")
440
-
441
num_tokens, hidden_size = query.shape
442
# Reshape the query, key, and value tensors.
443
query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/torch_sdpa.py
+3
-1
@@ -390,6 +390,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
390
kv_cache_dtype: str,
391
blocksparse_params: Optional[Dict[str, Any]] = None,
392
logits_soft_cap: Optional[float] = None,
393
+ attn_type: str = AttentionType.DECODER,
394
) -> None:
395
if blocksparse_params is not None:
396
raise ValueError(
@@ -421,6 +422,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
422
raise NotImplementedError(
423
"Torch SDPA backend does not support FP8 KV cache. "
424
"Please use xFormers backend instead.")
425
+ self.attn_type = attn_type
426
427
def forward(
428
self,
@@ -431,7 +433,6 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
433
attn_metadata: TorchSDPAMetadata, # type: ignore
434
k_scale: float = 1.0,
435
v_scale: float = 1.0,
434
- attn_type: str = AttentionType.DECODER,
436
output: Optional[torch.Tensor] = None,
437
) -> torch.Tensor:
438
"""Forward pass with torch SDPA and PagedAttention.
@@ -448,6 +449,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
449
shape = [num_tokens, num_heads * head_size]
450
"""
451
assert k_scale == 1.0 and v_scale == 1.0
452
+ attn_type = self.attn_type
453
if (attn_type == AttentionType.ENCODER
454
and (not attn_metadata.is_all_encoder_attn_metadata_set)):
455
raise AttributeError("Encoder attention requires setting "
vllm/attention/backends/xformers.py
+4
-2
@@ -379,6 +379,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
379
kv_cache_dtype: str,
380
blocksparse_params: Optional[Dict[str, Any]] = None,
381
logits_soft_cap: Optional[float] = None,
382
+ attn_type: str = AttentionType.DECODER,
383
) -> None:
384
if blocksparse_params is not None:
385
raise ValueError(
@@ -405,6 +406,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
406
f"Head size {head_size} is not supported by PagedAttention. "
407
f"Supported head sizes are: {suppored_head_sizes}.")
408
409
+ self.attn_type = attn_type
410
+
411
def forward(
412
self,
413
query: torch.Tensor,
@@ -414,7 +417,6 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
417
attn_metadata: "XFormersMetadata",
418
k_scale: float = 1.0,
419
v_scale: float = 1.0,
417
- attn_type: str = AttentionType.DECODER,
420
output: Optional[torch.Tensor] = None,
421
) -> torch.Tensor:
422
"""Forward pass with xFormers and PagedAttention.
@@ -468,7 +470,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
470
Returns:
471
shape = [num_tokens, num_heads * head_size]
472
"""
471
-
473
+ attn_type = self.attn_type
474
# Check that appropriate attention metadata attributes are
475
# selected for the desired attention type
476
if (attn_type == AttentionType.ENCODER
vllm/attention/layer.py
+10
-27
@@ -41,6 +41,7 @@ class Attention(nn.Module):
41
logits_soft_cap: Optional[float] = None,
42
per_layer_sliding_window: Optional[int] = None,
43
prefix: str = "",
44
+ attn_type: str = AttentionType.DECODER,
45
) -> None:
46
super().__init__()
47
if per_layer_sliding_window is not None:
@@ -96,7 +97,7 @@ class Attention(nn.Module):
97
impl_cls = attn_backend.get_impl_cls()
98
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
99
alibi_slopes, sliding_window, kv_cache_dtype,
99
- blocksparse_params, logits_soft_cap)
100
+ blocksparse_params, logits_soft_cap, attn_type)
101
self.num_heads = num_heads
102
self.head_size = head_size
103
self.num_kv_heads = num_kv_heads
@@ -119,6 +120,7 @@ class Attention(nn.Module):
120
raise ValueError(f"Duplicate layer name: {prefix}")
121
compilation_config.static_forward_context[prefix] = self
122
self.layer_name = prefix
123
+ self.attn_type = attn_type
124
125
def forward(
126
self,
@@ -127,18 +129,12 @@ class Attention(nn.Module):
129
value: torch.Tensor,
130
kv_cache: torch.Tensor,
131
attn_metadata: AttentionMetadata,
130
- attn_type: str = AttentionType.DECODER,
132
) -> torch.Tensor:
133
134
if self.use_direct_call:
134
- return self.impl.forward(query,
135
- key,
136
- value,
137
- kv_cache,
138
- attn_metadata,
139
- self._k_scale,
140
- self._v_scale,
141
- attn_type=attn_type)
135
+ return self.impl.forward(query, key, value, kv_cache,
136
+ attn_metadata, self._k_scale,
137
+ self._v_scale)
138
elif self.use_output:
139
output = torch.empty_like(query)
140
hidden_size = query.size(-1)
@@ -152,13 +148,11 @@ class Attention(nn.Module):
148
if value is not None:
149
value = value.view(-1, self.num_kv_heads, self.head_size)
150
torch.ops.vllm.unified_attention_with_output(
155
- query, key, value, output, kv_cache, attn_type,
156
- self.layer_name)
151
+ query, key, value, output, kv_cache, self.layer_name)
152
return output.view(-1, hidden_size)
153
else:
154
return torch.ops.vllm.unified_attention(query, key, value,
160
- kv_cache, attn_type,
161
- self.layer_name)
155
+ kv_cache, self.layer_name)
156
157
def extra_repr(self) -> str:
158
s = f"head_size={self.impl.head_size}" # type: ignore
@@ -237,20 +231,13 @@ def unified_attention(
231
key: torch.Tensor,
232
value: torch.Tensor,
233
kv_cache: torch.Tensor,
240
- attn_type: str,
234
layer_name: str,
235
) -> torch.Tensor:
236
forward_context: ForwardContext = get_forward_context()
237
attn_metadata = forward_context.dynamic_forward_context
238
self = forward_context.static_forward_context[layer_name]
246
- return self.impl.forward(query,
247
- key,
248
- value,
249
- kv_cache,
250
- attn_metadata,
251
- self._k_scale,
252
- self._v_scale,
253
- attn_type=attn_type)
239
+ return self.impl.forward(query, key, value, kv_cache, attn_metadata,
240
+ self._k_scale, self._v_scale)
241
242
243
def unified_attention_fake(
@@ -258,7 +245,6 @@ def unified_attention_fake(
245
key: torch.Tensor,
246
value: torch.Tensor,
247
kv_cache: torch.Tensor,
261
- attn_type: str,
248
layer_name: str,
249
) -> torch.Tensor:
250
return torch.empty_like(query).contiguous()
@@ -279,7 +265,6 @@ def unified_attention_with_output(
265
value: torch.Tensor,
266
output: torch.Tensor,
267
kv_cache: torch.Tensor,
282
- attn_type: str,
268
layer_name: str,
269
) -> None:
270
forward_context: ForwardContext = get_forward_context()
@@ -292,7 +277,6 @@ def unified_attention_with_output(
277
attn_metadata,
278
self._k_scale,
279
self._v_scale,
295
- attn_type=attn_type,
280
output=output)
281
282
@@ -302,7 +286,6 @@ def unified_attention_with_output_fake(
286
value: torch.Tensor,
287
output: torch.Tensor,
288
kv_cache: torch.Tensor,
305
- attn_type: str,
289
layer_name: str,
290
) -> None:
291
return
vllm/model_executor/models/bart.py
+11
-33
@@ -71,12 +71,8 @@ class BartLearnedPositionalEmbedding(VocabParallelEmbedding):
71
def forward(
72
self,
73
positions: torch.Tensor,
74
- attn_type: AttentionType,
74
) -> torch.Tensor:
75
"""`input_ids' shape is expected to be [bsz x seqlen]."""
77
-
78
- assert attn_type != AttentionType.ENCODER_DECODER
79
-
76
return super().forward(positions + self.offset)
77
78
@@ -180,7 +176,8 @@ class BartEncoderAttention(nn.Module):
176
num_kv_heads=self.num_kv_heads,
177
cache_config=cache_config,
178
quant_config=quant_config,
183
- prefix=f"{prefix}.attn")
179
+ prefix=f"{prefix}.attn",
180
+ attn_type=AttentionType.ENCODER)
181
182
def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
183
attn_metadata: AttentionMetadata) -> torch.Tensor:
@@ -189,12 +186,7 @@ class BartEncoderAttention(nn.Module):
186
qkv, _ = self.qkv_proj(hidden_states)
187
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
188
192
- attn_output = self.attn(q,
193
- k,
194
- v,
195
- kv_cache,
196
- attn_metadata,
197
- attn_type=AttentionType.ENCODER)
189
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
190
191
output, _ = self.out_proj(attn_output)
192
return output
@@ -264,7 +256,8 @@ class BartDecoderSelfAttention(nn.Module):
256
num_kv_heads=self.num_kv_heads,
257
cache_config=cache_config,
258
quant_config=quant_config,
267
- prefix=f"{prefix}.attn")
259
+ prefix=f"{prefix}.attn",
260
+ attn_type=AttentionType.DECODER)
261
262
def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
263
attn_metadata: AttentionMetadata) -> torch.Tensor:
@@ -273,12 +266,7 @@ class BartDecoderSelfAttention(nn.Module):
266
qkv, _ = self.qkv_proj(hidden_states)
267
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
268
276
- attn_output = self.attn(q,
277
- k,
278
- v,
279
- kv_cache,
280
- attn_metadata,
281
- attn_type=AttentionType.DECODER)
269
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
270
271
output, _ = self.out_proj(attn_output)
272
return output
@@ -348,7 +336,8 @@ class BartCrossAttention(nn.Module):
336
num_kv_heads=self.num_kv_heads,
337
cache_config=cache_config,
338
quant_config=quant_config,
351
- prefix=f"{prefix}.attn")
339
+ prefix=f"{prefix}.attn",
340
+ attn_type=AttentionType.ENCODER_DECODER)
341
342
def forward(
343
self,
@@ -372,12 +361,7 @@ class BartCrossAttention(nn.Module):
361
_, k, v = qkv_enc.split([self.q_size, self.kv_size, self.kv_size],
362
dim=-1)
363
375
- attn_output = self.attn(q,
376
- k,
377
- v,
378
- kv_cache,
379
- attn_metadata,
380
- attn_type=AttentionType.ENCODER_DECODER)
364
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
365
366
output, _ = self.out_proj(attn_output)
367
return output
@@ -644,10 +628,7 @@ class BartEncoder(nn.Module):
628
# retrieve input_ids and inputs_embeds
629
inputs_embeds = self.embed_tokens(input_ids)
630
647
- embed_pos = self.embed_positions(
648
- positions,
649
- AttentionType.ENCODER,
650
- )
631
+ embed_pos = self.embed_positions(positions)
632
embed_pos = embed_pos.to(inputs_embeds.device)
633
634
hidden_states = inputs_embeds + embed_pos
@@ -734,10 +715,7 @@ class BartDecoder(nn.Module):
715
inputs_embeds = self.embed_tokens(decoder_input_ids)
716
717
# embed positions
737
- embed_pos = self.embed_positions(
738
- decoder_positions,
739
- AttentionType.DECODER,
740
- )
718
+ embed_pos = self.embed_positions(decoder_positions)
719
embed_pos = embed_pos.to(inputs_embeds.device)
720
721
hidden_states = inputs_embeds + embed_pos
vllm/model_executor/models/bert.py
+3
-7
@@ -238,7 +238,8 @@ class BertSelfAttention(nn.Module):
238
num_kv_heads=self.num_kv_heads,
239
cache_config=cache_config,
240
quant_config=quant_config,
241
- prefix=f"{prefix}.attn")
241
+ prefix=f"{prefix}.attn",
242
+ attn_type=AttentionType.ENCODER_ONLY)
243
244
def forward(
245
self,
@@ -248,12 +249,7 @@ class BertSelfAttention(nn.Module):
249
) -> torch.Tensor:
250
qkv, _ = self.qkv_proj(hidden_states)
251
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
251
- output = self.attn(q,
252
- k,
253
- v,
254
- kv_cache,
255
- attn_metadata,
256
- attn_type=AttentionType.ENCODER_ONLY)
252
+ output = self.attn(q, k, v, kv_cache, attn_metadata)
253
return output
254
255
vllm/model_executor/models/mllama.py
+4
-7
@@ -770,6 +770,7 @@ class MllamaTextCrossAttention(nn.Module):
770
self.scaling,
771
self.num_local_key_value_heads,
772
prefix=f"{prefix}.attn",
773
+ attn_type=AttentionType.ENCODER_DECODER,
774
)
775
776
def forward(
@@ -805,13 +806,9 @@ class MllamaTextCrossAttention(nn.Module):
806
kv_range_for_decode,
807
attn_metadata)
808
else:
808
- output = self.attn(q.view(-1,
809
- self.num_local_heads * self.head_dim),
810
- k,
811
- v,
812
- kv_cache,
813
- attn_metadata,
814
- attn_type=AttentionType.ENCODER_DECODER)
809
+ output = self.attn(
810
+ q.view(-1, self.num_local_heads * self.head_dim), k, v,
811
+ kv_cache, attn_metadata)
812
out, _ = self.o_proj(output)
813
return out
814
vllm/model_executor/models/qwen2.py
+16
-19
@@ -107,7 +107,8 @@ class Qwen2Attention(nn.Module):
107
cache_config: Optional[CacheConfig] = None,
108
quant_config: Optional[QuantizationConfig] = None,
109
rope_scaling: Optional[Tuple] = None,
110
- prefix: str = "") -> None:
110
+ prefix: str = "",
111
+ attn_type: str = AttentionType.DECODER) -> None:
112
super().__init__()
113
self.hidden_size = hidden_size
114
tp_size = get_tensor_model_parallel_world_size()
@@ -160,7 +161,8 @@ class Qwen2Attention(nn.Module):
161
num_kv_heads=self.num_kv_heads,
162
cache_config=cache_config,
163
quant_config=quant_config,
163
- prefix=f"{prefix}.attn")
164
+ prefix=f"{prefix}.attn",
165
+ attn_type=attn_type)
166
167
def forward(
168
self,
@@ -168,17 +170,11 @@ class Qwen2Attention(nn.Module):
170
hidden_states: torch.Tensor,
171
kv_cache: torch.Tensor,
172
attn_metadata: AttentionMetadata,
171
- attn_type: str = AttentionType.DECODER,
173
) -> torch.Tensor:
174
qkv, _ = self.qkv_proj(hidden_states)
175
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
176
q, k = self.rotary_emb(positions, q, k)
176
- attn_output = self.attn(q,
177
- k,
178
- v,
179
- kv_cache,
180
- attn_metadata,
181
- attn_type=attn_type)
177
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
178
output, _ = self.o_proj(attn_output)
179
return output
180
@@ -197,6 +193,16 @@ class Qwen2DecoderLayer(nn.Module):
193
# Requires transformers > 4.32.0
194
rope_theta = getattr(config, "rope_theta", 1000000)
195
rope_scaling = getattr(config, "rope_scaling", None)
196
+
197
+ # By default, Qwen2 uses causal attention as it is a decoder-only model.
198
+ # You can override the HF config with `is_causal=False` to enable
199
+ # bidirectional attention, which is used in some embedding models
200
+ # (e.g. Alibaba-NLP/gte-Qwen2-7B-instruct)
201
+ if getattr(config, "is_causal", True):
202
+ attn_type = AttentionType.DECODER
203
+ else:
204
+ attn_type = AttentionType.ENCODER_ONLY
205
+
206
self.self_attn = Qwen2Attention(
207
hidden_size=self.hidden_size,
208
num_heads=config.num_attention_heads,
@@ -207,6 +213,7 @@ class Qwen2DecoderLayer(nn.Module):
213
quant_config=quant_config,
214
rope_scaling=rope_scaling,
215
prefix=f"{prefix}.self_attn",
216
+ attn_type=attn_type,
217
)
218
self.mlp = Qwen2MLP(
219
hidden_size=self.hidden_size,
@@ -220,15 +227,6 @@ class Qwen2DecoderLayer(nn.Module):
227
self.post_attention_layernorm = RMSNorm(config.hidden_size,
228
eps=config.rms_norm_eps)
229
223
- # By default, Qwen2 uses causal attention as it is a decoder-only model.
224
- # You can override the HF config with `is_causal=False` to enable
225
- # bidirectional attention, which is used in some embedding models
226
- # (e.g. Alibaba-NLP/gte-Qwen2-7B-instruct)
227
- if getattr(config, "is_causal", True):
228
- self._attn_type = AttentionType.DECODER
229
- else:
230
- self._attn_type = AttentionType.ENCODER_ONLY
231
-
230
def forward(
231
self,
232
positions: torch.Tensor,
@@ -249,7 +247,6 @@ class Qwen2DecoderLayer(nn.Module):
247
hidden_states=hidden_states,
248
kv_cache=kv_cache,
249
attn_metadata=attn_metadata,
252
- attn_type=self._attn_type,
250
)
251
252
# Fully Connected
vllm/v1/attention/backends/flash_attn.py
+7
-7
@@ -89,6 +89,7 @@ class FlashAttentionImpl(AttentionImpl):
89
kv_cache_dtype: str,
90
blocksparse_params: Optional[Dict[str, Any]] = None,
91
logits_soft_cap: Optional[float] = None,
92
+ attn_type: AttentionType = AttentionType.DECODER,
93
) -> None:
94
if blocksparse_params is not None:
95
raise ValueError(
@@ -119,6 +120,12 @@ class FlashAttentionImpl(AttentionImpl):
120
f"Head size {head_size} is not supported by FlashAttention. "
121
f"Supported head sizes are: {support_head_sizes}.")
122
123
+ if attn_type != AttentionType.DECODER:
124
+ raise NotImplementedError("Encoder self-attention and "
125
+ "encoder/decoder cross-attention "
126
+ "are not implemented for "
127
+ "FlashAttentionImpl")
128
+
129
def forward(
130
self,
131
query: torch.Tensor,
@@ -128,7 +135,6 @@ class FlashAttentionImpl(AttentionImpl):
135
attn_metadata: FlashAttentionMetadata,
136
k_scale: float = 1.0,
137
v_scale: float = 1.0,
131
- attn_type: AttentionType = AttentionType.DECODER,
138
output: Optional[torch.Tensor] = None,
139
) -> torch.Tensor:
140
"""Forward pass with FlashAttention.
@@ -142,12 +148,6 @@ class FlashAttentionImpl(AttentionImpl):
148
Returns:
149
shape = [num_tokens, num_heads * head_size]
150
"""
145
- if attn_type != AttentionType.DECODER:
146
- raise NotImplementedError("Encoder self-attention and "
147
- "encoder/decoder cross-attention "
148
- "are not implemented for "
149
- "FlashAttentionImpl")
150
-
151
# NOTE(woosuk): FlashAttention does not support FP8 KV cache.
152
assert k_scale == 1.0 and v_scale == 1.0, (
153
"key/v_scale is not supported in FlashAttention.")