A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Encoder Decoder] Add flash_attn kernel support for encoder-decoder models (#9559)
sroy745 committed
Nov 1, 2024 at 23:22 UTC
a78dd3303efac284afc6785eddba5f175285863b
11 files changed
+716
-317
tests/encoder_decoder/test_e2e_correctness.py
+51
-37
@@ -7,12 +7,18 @@ from typing import List, Optional, Tuple
7
import pytest
8
from transformers import AutoModelForSeq2SeqLM
9
10
+from vllm.attention.selector import (_Backend,
11
+ global_force_attn_backend_context_manager)
12
from vllm.platforms import current_platform
13
from vllm.sequence import SampleLogprobs
14
15
from ..conftest import DecoderPromptType
16
from ..models.utils import check_logprobs_close
17
18
+LIST_ENC_DEC_SUPPORTED_BACKENDS = [
19
+ _Backend.XFORMERS, _Backend.FLASH_ATTN, None
20
+]
21
+
22
23
def vllm_to_hf_output(
24
vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
@@ -29,7 +35,8 @@ def vllm_to_hf_output(
35
36
37
@pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
32
-@pytest.mark.parametrize("dtype", ["bfloat16"])
38
+@pytest.mark.parametrize("dtype", ["float"])
39
+@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
40
@pytest.mark.parametrize("max_tokens", [128])
41
@pytest.mark.parametrize("num_logprobs", [5])
42
@pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
@@ -48,6 +55,7 @@ def test_encoder_decoder_e2e(
55
num_logprobs: int,
56
decoder_prompt_type: DecoderPromptType,
57
enforce_eager: bool,
58
+ attn_backend: _Backend,
59
) -> None:
60
'''
61
End-to-End (E2E) test for the encoder-decoder framework.
@@ -56,43 +64,49 @@ def test_encoder_decoder_e2e(
64
implementations to ensure that both implementations produce consistent
65
and correct results.
66
'''
59
- test_case_prompts = example_encoder_decoder_prompts[decoder_prompt_type]
67
+ with global_force_attn_backend_context_manager(attn_backend):
68
+ if attn_backend == _Backend.FLASH_ATTN:
69
+ # Flash Attention works only with bfloat16 data-type
70
+ dtype = 'bfloat16'
71
+ test_case_prompts = example_encoder_decoder_prompts[
72
+ decoder_prompt_type]
73
61
- # Configuration settings for HF baseline
62
- hf_kwargs = {
63
- "top_k": None,
64
- "num_beams": 1,
65
- "repetition_penalty": 1.0,
66
- "top_p": 1.0,
67
- "length_penalty": 1.0,
68
- "early_stopping": False,
69
- "no_repeat_ngram_size": None,
70
- "min_length": 0
71
- }
74
+ # Configuration settings for HF baseline
75
+ hf_kwargs = {
76
+ "top_k": None,
77
+ "num_beams": 1,
78
+ "repetition_penalty": 1.0,
79
+ "top_p": 1.0,
80
+ "length_penalty": 1.0,
81
+ "early_stopping": False,
82
+ "no_repeat_ngram_size": None,
83
+ "min_length": 0
84
+ }
85
73
- with hf_runner(model, dtype=dtype,
74
- auto_cls=AutoModelForSeq2SeqLM) as hf_model:
75
- hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
76
- test_case_prompts,
77
- max_tokens,
78
- num_logprobs,
79
- **hf_kwargs,
80
- ))
81
- with vllm_runner(model, dtype=dtype,
82
- enforce_eager=enforce_eager) as vllm_model:
83
- vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
84
- test_case_prompts, max_tokens, num_logprobs)
86
+ with hf_runner(model, dtype=dtype,
87
+ auto_cls=AutoModelForSeq2SeqLM) as hf_model:
88
+ hf_outputs = (
89
+ hf_model.generate_encoder_decoder_greedy_logprobs_limit(
90
+ test_case_prompts,
91
+ max_tokens,
92
+ num_logprobs,
93
+ **hf_kwargs,
94
+ ))
95
+ with vllm_runner(model, dtype=dtype,
96
+ enforce_eager=enforce_eager) as vllm_model:
97
+ vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
98
+ test_case_prompts, max_tokens, num_logprobs)
99
86
- hf_skip_tokens = (1
87
- if decoder_prompt_type == DecoderPromptType.NONE else 0)
100
+ hf_skip_tokens = (1 if decoder_prompt_type == DecoderPromptType.NONE
101
+ else 0)
102
89
- check_logprobs_close(
90
- outputs_0_lst=hf_outputs,
91
- outputs_1_lst=[
92
- vllm_to_hf_output(vllm_output, decoder_prompt_type)
93
- for vllm_output in vllm_outputs
94
- ],
95
- name_0="hf",
96
- name_1="vllm",
97
- num_outputs_0_skip_tokens=hf_skip_tokens,
98
- )
103
+ check_logprobs_close(
104
+ outputs_0_lst=hf_outputs,
105
+ outputs_1_lst=[
106
+ vllm_to_hf_output(vllm_output, decoder_prompt_type)
107
+ for vllm_output in vllm_outputs
108
+ ],
109
+ name_0="hf",
110
+ name_1="vllm",
111
+ num_outputs_0_skip_tokens=hf_skip_tokens,
112
+ )
tests/kernels/test_encoder_decoder_attn.py
+115
-41
@@ -16,13 +16,13 @@ from tests.kernels.utils import *
16
from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
17
AttentionType)
18
from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19
-from vllm.attention.selector import (_Backend,
19
+from vllm.attention.selector import (_Backend, get_attn_backend,
20
global_force_attn_backend_context_manager)
21
+from vllm.forward_context import set_forward_context
22
from vllm.platforms import current_platform
23
24
# List of support backends for encoder/decoder models
24
-LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS]
25
-
25
+LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
26
HEAD_SIZES = [64, 256]
27
28
NUM_HEADS = [1, 16]
@@ -145,7 +145,8 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
145
test_pt.num_heads,
146
test_pt.head_size,
147
test_pt.block_size,
148
- device=CUDA_DEVICE)
148
+ device=CUDA_DEVICE,
149
+ backend=test_pt.backend_name)
150
return TestResources(scale, attn_backend, attn, kv_cache)
151
152
@@ -592,6 +593,7 @@ def _run_encoder_attention_test(
593
attn: Attention,
594
encoder_test_params: PhaseTestParameters,
595
attn_metadata: AttentionMetadata,
596
+ test_pt: TestPoint,
597
) -> torch.Tensor:
598
'''
599
Run encoder attention.
@@ -610,6 +612,8 @@ def _run_encoder_attention_test(
612
(number_of_tokens x num_heads x head_size)
613
query/key/value fields
614
* attn_metadata: attention metadata for encoder/decoder-self attention
615
+ * test_pt: The TestPoint object containing test details like number of
616
+ model heads, head size, name of the backend being used etc.
617
618
Returns:
619
* Attention.forward() applied to packed {query,key,value} and
@@ -619,20 +623,31 @@ def _run_encoder_attention_test(
623
attn_type = AttentionType.ENCODER
624
packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
625
assert packed_qkv is not None
622
- return attn.forward(packed_qkv.query,
623
- packed_qkv.key,
624
- packed_qkv.value,
625
- torch.tensor([],
626
- dtype=torch.float32,
627
- device=packed_qkv.query.device),
628
- attn_metadata,
629
- attn_type=attn_type)
626
+ with set_forward_context(attn_metadata):
627
+ # In the test setup the shape of the query is
628
+ # [batch_size, seq_len, num_heads, head_size]. However
629
+ # the attention backend expect the shape to be
630
+ # [num_tokens, hidden_size]. Hence reshape the query before
631
+ # invoking the forward method.
632
+ # TODO - Update the way we construct the query so that it
633
+ # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
634
+ reshaped_query = packed_qkv.query.view(
635
+ -1, test_pt.num_heads * test_pt.head_size)
636
+ return attn.forward(reshaped_query,
637
+ packed_qkv.key,
638
+ packed_qkv.value,
639
+ torch.tensor([],
640
+ dtype=torch.float32,
641
+ device=packed_qkv.query.device),
642
+ attn_metadata,
643
+ attn_type=attn_type)
644
645
646
def _run_decoder_self_attention_test(
647
test_rsrcs: TestResources,
648
decoder_test_params: PhaseTestParameters,
649
attn_metadata: AttentionMetadata,
650
+ test_pt: TestPoint,
651
) -> torch.Tensor:
652
'''
653
Run decoder self-attention test.
@@ -650,6 +665,8 @@ def _run_decoder_self_attention_test(
665
query/key/value fields
666
* attn_metadata: attention metadata for decoder-self attention
667
(contains KV cache memory-mapping)
668
+ * test_pt: The TestPoint object containing test details like number of
669
+ model heads, head size, name of the backend being used etc.
670
671
Returns:
672
* Attention.forward() applied to packed_{query,key,value}, kv_cache
@@ -660,12 +677,22 @@ def _run_decoder_self_attention_test(
677
kv_cache = test_rsrcs.kv_cache
678
packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
679
assert packed_qkv is not None
663
- return attn.forward(packed_qkv.query,
664
- packed_qkv.key,
665
- packed_qkv.value,
666
- kv_cache,
667
- attn_metadata,
668
- attn_type=attn_type)
680
+ with set_forward_context(attn_metadata):
681
+ # In the test setup the shape of the query is
682
+ # [batch_size, seq_len, num_heads, head_size]. However
683
+ # the attention backend expect the shape to be
684
+ # [num_tokens, hidden_size]. Hence reshape the query before
685
+ # invoking the forward method.
686
+ # TODO - Update the way we construct the query so that it
687
+ # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
688
+ reshaped_query = packed_qkv.query.view(
689
+ -1, test_pt.num_heads * test_pt.head_size)
690
+ return attn.forward(reshaped_query,
691
+ packed_qkv.key,
692
+ packed_qkv.value,
693
+ kv_cache,
694
+ attn_metadata,
695
+ attn_type=attn_type)
696
697
698
def _run_encoder_decoder_cross_attention_test(
@@ -673,6 +700,7 @@ def _run_encoder_decoder_cross_attention_test(
700
decoder_test_params: PhaseTestParameters,
701
cross_test_params: Optional[PhaseTestParameters],
702
attn_metadata: AttentionMetadata,
703
+ test_pt: TestPoint,
704
) -> torch.Tensor:
705
'''
706
Run encoder/decoder cross-attention test.
@@ -701,6 +729,8 @@ def _run_encoder_decoder_cross_attention_test(
729
(number_of_tokens x num_heads x head_size)
730
key/value fields
731
* attn_metadata: attention metadata for encoder/decoder-self attention
732
+ * test_pt: The TestPoint object containing test details like number of
733
+ model heads, head size, name of the backend being used etc.
734
735
Returns:
736
* Attention.forward() applied to packed_{query,key,value}, kv_cache
@@ -718,12 +748,37 @@ def _run_encoder_decoder_cross_attention_test(
748
cross_pckd_qkv = cross_test_params.packed_qkvo.packed_qkv
749
key = (None if cross_pckd_qkv is None else cross_pckd_qkv.key)
750
value = (None if cross_pckd_qkv is None else cross_pckd_qkv.value)
721
- return attn.forward(decoder_test_params.packed_qkvo.packed_qkv.query,
722
- key,
723
- value,
724
- kv_cache,
725
- attn_metadata,
726
- attn_type=attn_type)
751
+ with set_forward_context(attn_metadata):
752
+ # In the test setup the shape of the query is
753
+ # [batch_size, seq_len, num_heads, head_size]. However
754
+ # the attention backend expect the shape to be
755
+ # [num_tokens, hidden_size]. Hence reshape the query before
756
+ # invoking the forward method.
757
+ # TODO - Update the way we construct the query so that it
758
+ # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
759
+ reshaped_query = decoder_test_params.packed_qkvo.packed_qkv.query.view(
760
+ -1, test_pt.num_heads * test_pt.head_size)
761
+ return attn.forward(reshaped_query,
762
+ key,
763
+ value,
764
+ kv_cache,
765
+ attn_metadata,
766
+ attn_type=attn_type)
767
+
768
+
769
+@pytest.fixture(autouse=True)
770
+def set_reset_environment(attn_backend):
771
+ # Set the default torch datatype to bfloat16 to enable
772
+ # testing of the Flash Attention backend. Also clear the
773
+ # cached value of the backend.
774
+ default_dtype = torch.get_default_dtype()
775
+ if attn_backend.name == 'FLASH_ATTN':
776
+ torch.set_default_dtype(torch.bfloat16)
777
+ get_attn_backend.cache_clear()
778
+ yield
779
+ # Reset the torch datatype to what it was before the test
780
+ # so as not to impact the remaining tests.
781
+ torch.set_default_dtype(default_dtype)
782
783
784
@pytest.mark.skipif(current_platform.is_rocm(),
@@ -773,10 +828,8 @@ def test_encoder_only(
828
* max_dec_seq_len: max length of decoder input sequences
829
* max_enc_seq_len: max length of encoder input sequences
830
'''
776
-
831
# Force Attention wrapper backend
832
with global_force_attn_backend_context_manager(attn_backend):
779
-
833
# Note: KV cache size of 4096 is arbitrary & chosen intentionally
834
# to be more than necessary, since exceeding the kv cache size
835
# is not part of this test
@@ -807,10 +860,14 @@ def test_encoder_only(
860
# PREFILL: encoder attention
861
862
enc_pckd_act_out: torch.Tensor = (_run_encoder_attention_test(
810
- test_rsrcs.attn, enc_test_params, prephase_attn_metadata))
863
+ test_rsrcs.attn,
864
+ enc_test_params,
865
+ prephase_attn_metadata,
866
+ test_pt=test_pt))
867
868
# - Is encoder attention result correct?
813
- assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
869
+ assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
870
+ attn_backend.name)
871
872
873
@pytest.mark.skipif(current_platform.is_rocm(),
@@ -892,10 +949,8 @@ def test_e2e_enc_dec_attn(
949
* max_dec_seq_len: max length of decoder input sequences
950
* max_enc_seq_len: max length of encoder input sequences
951
'''
895
-
952
# Force Attention wrapper backend
953
with global_force_attn_backend_context_manager(attn_backend):
898
-
954
# Note: KV cache size of 4096 is arbitrary & chosen intentionally
955
# to be more than necessary, since exceeding the kv cache size
956
# is not part of this test
@@ -955,29 +1010,39 @@ def test_e2e_enc_dec_attn(
1010
1011
enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
1012
enc_test_params,
958
- prephase_attn_metadata)
1013
+ prephase_attn_metadata,
1014
+ test_pt=test_pt)
1015
1016
# - Is encoder attention result correct?
961
- assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
1017
+ assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
1018
+ attn_backend.name)
1019
1020
# PREFILL: decoder self-attention test
1021
1022
prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
966
- test_rsrcs, prephase_dec_test_params, prephase_attn_metadata)
1023
+ test_rsrcs,
1024
+ prephase_dec_test_params,
1025
+ prephase_attn_metadata,
1026
+ test_pt=test_pt)
1027
1028
# - Is prefill decoder self-attention correct?
1029
assert_actual_matches_ideal(prephase_dec_test_params,
970
- prephase_dec_pckd_act_out)
1030
+ prephase_dec_pckd_act_out,
1031
+ attn_backend.name)
1032
1033
# PREFILL: encoder/decoder cross-attention test
1034
1035
prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
975
- test_rsrcs, prephase_dec_test_params, prephase_cross_test_params,
976
- prephase_attn_metadata)
1036
+ test_rsrcs,
1037
+ prephase_dec_test_params,
1038
+ prephase_cross_test_params,
1039
+ prephase_attn_metadata,
1040
+ test_pt=test_pt)
1041
1042
# - Is prefill encoder/decoder cross-attention correct?
1043
assert_actual_matches_ideal(prephase_cross_test_params,
980
- prephase_cross_pckd_act_out)
1044
+ prephase_cross_pckd_act_out,
1045
+ attn_backend.name)
1046
1047
# DECODE: build decode-phase attention metadata
1048
@@ -993,17 +1058,26 @@ def test_e2e_enc_dec_attn(
1058
# DECODE: decoder self-attention test
1059
1060
decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
996
- test_rsrcs, decphase_dec_test_params, decphase_attn_metadata)
1061
+ test_rsrcs,
1062
+ decphase_dec_test_params,
1063
+ decphase_attn_metadata,
1064
+ test_pt=test_pt)
1065
1066
# - Is decode-phase decoder self-attention correct?
1067
assert_actual_matches_ideal(decphase_dec_test_params,
1000
- decphase_dec_pckd_act_out)
1068
+ decphase_dec_pckd_act_out,
1069
+ attn_backend.name)
1070
1071
# DECODE: encoder/decoder cross-attention test
1072
1073
decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
1005
- test_rsrcs, decphase_dec_test_params, None, decphase_attn_metadata)
1074
+ test_rsrcs,
1075
+ decphase_dec_test_params,
1076
+ None,
1077
+ decphase_attn_metadata,
1078
+ test_pt=test_pt)
1079
1080
# - Is decode-phase encoder/decoder cross-attention correct?
1081
assert_actual_matches_ideal(decphase_cross_test_params,
1009
- decphase_cross_pckd_act_out)
1082
+ decphase_cross_pckd_act_out,
1083
+ attn_backend.name)
tests/kernels/utils.py
+74
-16
@@ -13,8 +13,8 @@ from torch._prims_common import TensorLikeType
13
14
from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
15
from vllm.model_executor.layers.activation import SiluAndMul
16
-from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
17
- make_tensor_with_pad)
16
+from vllm.utils import (STR_BACKEND_ENV_VAR, STR_FLASH_ATTN_VAL,
17
+ STR_XFORMERS_ATTN_VAL, make_tensor_with_pad)
18
19
# For now, disable "test_aot_dispatch_dynamic" since there are some
20
# bugs related to this test in PyTorch 2.4.
@@ -525,17 +525,22 @@ def make_backend(backend_name: str) -> AttentionBackend:
525
if backend_name == STR_XFORMERS_ATTN_VAL:
526
# NOTE: xFormers backend cannot be imported for CPU and AMD GPUs.
527
from vllm.attention.backends.xformers import XFormersBackend
528
-
528
return XFormersBackend()
529
+ elif backend_name == STR_FLASH_ATTN_VAL:
530
+ from vllm.attention.backends.flash_attn import FlashAttentionBackend
531
+ return FlashAttentionBackend()
532
+
533
raise AssertionError(
534
f"Unrecognized backend_name {backend_name} for unit test")
535
536
537
def _make_metadata_tensors(
535
- seq_lens: Optional[List[int]], context_lens: Optional[List[int]],
536
- encoder_seq_lens: Optional[List[int]], device: Union[torch.device, str]
537
-) -> Tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[List[int]],
538
- torch.Tensor, Optional[int]]:
538
+ seq_lens: Optional[List[int]],
539
+ context_lens: Optional[List[int]],
540
+ encoder_seq_lens: Optional[List[int]],
541
+ device: Union[torch.device, str],
542
+) -> Tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[torch.Tensor],
543
+ torch.Tensor, torch.Tensor, Optional[int]]:
544
'''
545
Build scalar & tensor values required to build attention metadata structure.
546
@@ -553,6 +558,8 @@ def _make_metadata_tensors(
558
* max_context_len: max(context_lens)
559
* max_seq_len: max(seq_lens)
560
* seq_start_loc: start idx of each sequence
561
+ * encoder_seq_lens_tensor: encoder seq_lens list, as tensor
562
+ * encoder_seq_start_loc: start idx of each encoder sequence
563
* max_encoder_seq_len: encoder seq_lens list, as tensor
564
'''
565
seq_lens_tensor = maybe_make_int_tensor(seq_lens, device)
@@ -566,8 +573,26 @@ def _make_metadata_tensors(
573
574
seq_start_loc = None
575
576
+ if seq_lens_tensor is not None:
577
+ seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
578
+ dtype=torch.int32,
579
+ device=seq_lens_tensor.device)
580
+ torch.cumsum(seq_lens_tensor,
581
+ dim=0,
582
+ dtype=seq_start_loc.dtype,
583
+ out=seq_start_loc[1:])
584
+
585
+ encoder_seq_start_loc = torch.zeros(encoder_seq_lens_tensor.shape[0] + 1,
586
+ dtype=torch.int32,
587
+ device=encoder_seq_lens_tensor.device)
588
+ torch.cumsum(encoder_seq_lens_tensor,
589
+ dim=0,
590
+ dtype=encoder_seq_start_loc.dtype,
591
+ out=encoder_seq_start_loc[1:])
592
+
593
return (seq_lens_tensor, context_lens_tensor, max_context_len, max_seq_len,
570
- seq_start_loc, encoder_seq_lens_tensor, max_encoder_seq_len)
594
+ seq_start_loc, encoder_seq_lens_tensor, encoder_seq_start_loc,
595
+ max_encoder_seq_len)
596
597
598
def make_kv_cache(num_blocks: int,
@@ -575,6 +600,7 @@ def make_kv_cache(num_blocks: int,
600
head_size: int,
601
block_size: int,
602
device: Union[torch.device, str],
603
+ backend: str,
604
default_val: float = 0.0) -> torch.Tensor:
605
'''
606
Create a fake KV cache.
@@ -591,10 +617,20 @@ def make_kv_cache(num_blocks: int,
617
Returns:
618
619
* kv_cache: 2 x num_blocks x (block_size * num_heads * head_size)
620
+ * for backend 'XFORMERS'
621
+ * kv_cache: 2 x num_blocks x block_size x num_heads x head_size
622
+ * for backend 'FLASH_ATTN'
623
'''
595
-
596
- kv_cache = torch.rand(
597
- (2, num_blocks, block_size * num_heads * head_size)).to(device)
624
+ if backend == 'XFORMERS':
625
+ kv_cache = torch.rand(
626
+ (2, num_blocks, block_size * num_heads * head_size)).to(device)
627
+ elif backend == 'FLASH_ATTN':
628
+ kv_cache = torch.rand(
629
+ (2, num_blocks, block_size, num_heads, head_size)).to(device)
630
+ else:
631
+ raise ValueError(
632
+ f"Unknown backend value: '{backend}'. Expected 'XFORMERS' or "
633
+ f"'FLASH_ATTN'.")
634
if default_val is not None:
635
kv_cache[:, :, :] = default_val
636
return kv_cache
@@ -858,8 +894,9 @@ def make_test_metadata(
894
context_lens_tensor,
895
_,
896
_,
861
- _,
897
+ seq_start_loc,
898
encoder_seq_lens_tensor,
899
+ encoder_seq_start_loc,
900
max_encoder_seq_len,
901
) = _make_metadata_tensors(seq_lens,
902
context_lens,
@@ -874,6 +911,7 @@ def make_test_metadata(
911
num_decode_tokens=num_decode_tokens,
912
seq_lens=seq_lens,
913
seq_lens_tensor=seq_lens_tensor,
914
+ seq_start_loc=seq_start_loc,
915
max_prefill_seq_len=None if seq_lens is None else max(seq_lens),
916
max_decode_seq_len=0,
917
context_lens_tensor=context_lens_tensor,
@@ -882,6 +920,7 @@ def make_test_metadata(
920
num_encoder_tokens=num_encoder_tokens,
921
encoder_seq_lens=encoder_seq_lens,
922
encoder_seq_lens_tensor=encoder_seq_lens_tensor,
923
+ encoder_seq_start_loc=encoder_seq_start_loc,
924
max_encoder_seq_len=max_encoder_seq_len,
925
cross_slot_mapping=(None if cross_kv_mmap is None else
926
cross_kv_mmap.slot_mapping),
@@ -904,8 +943,9 @@ def make_test_metadata(
943
context_lens_tensor,
944
_,
945
_,
907
- _,
946
+ seq_start_loc,
947
encoder_seq_lens_tensor,
948
+ encoder_seq_start_loc,
949
max_encoder_seq_len,
950
) = _make_metadata_tensors(seq_lens,
951
context_lens,
@@ -920,14 +960,17 @@ def make_test_metadata(
960
num_decode_tokens=num_decode_tokens,
961
seq_lens=seq_lens,
962
seq_lens_tensor=seq_lens_tensor,
963
+ seq_start_loc=seq_start_loc,
964
max_prefill_seq_len=0,
965
max_decode_seq_len=max(seq_lens),
966
+ max_decode_query_len=1,
967
context_lens_tensor=context_lens_tensor,
968
block_tables=kv_mmap.block_tables,
969
use_cuda_graph=False,
970
num_encoder_tokens=num_encoder_tokens,
971
encoder_seq_lens=encoder_seq_lens,
972
encoder_seq_lens_tensor=encoder_seq_lens_tensor,
973
+ encoder_seq_start_loc=encoder_seq_start_loc,
974
max_encoder_seq_len=max_encoder_seq_len,
975
cross_slot_mapping=(None if cross_kv_mmap is None else
976
cross_kv_mmap.slot_mapping),
@@ -936,7 +979,8 @@ def make_test_metadata(
979
980
981
def assert_actual_matches_ideal(test_params: PhaseTestParameters,
939
- output_under_test: torch.Tensor) -> None:
982
+ output_under_test: torch.Tensor,
983
+ backend: str) -> None:
984
'''
985
Assert that observed output matches the ideal output
986
contained in the test parameters data structure.
@@ -947,8 +991,22 @@ def assert_actual_matches_ideal(test_params: PhaseTestParameters,
991
* output_under_test: actually observed output value
992
'''
993
ideal_output = test_params.packed_qkvo.ideal_output
950
- torch.testing.assert_close(ideal_output,
951
- output_under_test.view_as(ideal_output))
994
+ if backend == 'XFORMERS':
995
+ torch.testing.assert_close(ideal_output,
996
+ output_under_test.view_as(ideal_output))
997
+
998
+ elif backend == 'FLASH_ATTN':
999
+ # For FlashAttention override the accuracy thresholds to non default
1000
+ # values since we notice a higher difference between the ideal and
1001
+ # actual output.
1002
+ torch.testing.assert_close(ideal_output,
1003
+ output_under_test.view_as(ideal_output),
1004
+ atol=0.01,
1005
+ rtol=0.016)
1006
+ else:
1007
+ raise ValueError(
1008
+ f"Unknown backend value: '{backend}'. Expected 'XFORMERS' or "
1009
+ f"'FLASH_ATTN'.")
1010
1011
1012
# Copied/modified from torch._refs.__init__.py
tests/models/encoder_decoder/vision_language/test_florence2.py
+1
-1
@@ -85,7 +85,7 @@ def run_test(
85
86
87
@pytest.mark.parametrize("model", MODELS)
88
-@pytest.mark.parametrize("dtype", ["float"])
88
+@pytest.mark.parametrize("dtype", ["float", "bfloat16"])
89
@pytest.mark.parametrize("max_tokens", [64])
90
@pytest.mark.parametrize("num_logprobs", [5])
91
def test_models(hf_runner, vllm_runner, model, dtype, max_tokens,
vllm/attention/backends/flash_attn.py
+278
-86
@@ -10,10 +10,11 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
AttentionMetadata,
11
AttentionMetadataBuilder,
12
AttentionType)
13
-from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
14
- compute_slot_mapping,
15
- compute_slot_mapping_start_idx,
16
- is_block_tables_empty)
13
+from vllm.attention.backends.utils import (
14
+ PAD_SLOT_ID, CommonAttentionState, compute_slot_mapping,
15
+ compute_slot_mapping_start_idx, get_num_prefill_decode_query_kv_tokens,
16
+ get_seq_len_block_table_args, is_all_cross_attn_metadata_set,
17
+ is_all_encoder_attn_metadata_set, is_block_tables_empty)
18
from vllm.forward_context import get_forward_context
19
from vllm.multimodal import MultiModalPlaceholderMap
20
from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
@@ -73,7 +74,6 @@ class FlashAttentionBackend(AttentionBackend):
74
src_key_cache = src_kv_cache[0]
75
dst_key_cache = dst_kv_cache[0]
76
ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
76
-
77
src_value_cache = src_kv_cache[1]
78
dst_value_cache = dst_kv_cache[1]
79
ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
@@ -85,6 +85,7 @@ class FlashAttentionBackend(AttentionBackend):
85
) -> None:
86
key_caches = [kv_cache[0] for kv_cache in kv_caches]
87
value_caches = [kv_cache[1] for kv_cache in kv_caches]
88
+
89
ops.copy_blocks(key_caches, value_caches, src_to_dists)
90
91
@@ -111,26 +112,12 @@ class FlashAttentionMetadata(AttentionMetadata):
112
# |-------------------- seq_len ---------------------|
113
# |-- query_len ---|
114
114
- # Maximum query length in the batch.
115
- max_query_len: Optional[int]
116
-
117
- # Max number of query tokens among request in the batch.
118
- max_decode_query_len: Optional[int]
119
-
115
# Maximum sequence length among prefill batch. 0 if there are decoding
116
# requests only.
117
max_prefill_seq_len: int
118
# Maximum sequence length among decode batch. 0 if there are prefill
119
# requests only.
120
max_decode_seq_len: int
126
- # (batch_size + 1,). The cumulative subquery lengths of the sequences in
127
- # the batch, used to index into subquery. E.g., if the subquery length
128
- # is [4, 6], it is [0, 4, 10].
129
- query_start_loc: Optional[torch.Tensor]
130
- # (batch_size + 1,). The cumulative sequence lengths of the sequences in
131
- # the batch, used to index into sequence. E.g., if the sequence length is
132
- # [4, 6], it is [0, 4, 10].
133
- seq_start_loc: Optional[torch.Tensor]
121
# (batch_size,) A tensor of context lengths (tokens that are computed
122
# so far).
123
context_lens_tensor: Optional[torch.Tensor]
@@ -146,11 +133,62 @@ class FlashAttentionMetadata(AttentionMetadata):
133
# Whether or not if cuda graph is enabled.
134
# Cuda-graph is currently enabled for decoding only.
135
# TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
136
+
137
use_cuda_graph: bool
138
139
+ # Maximum query length in the batch.
140
+ max_query_len: Optional[int] = None
141
+
142
+ # Max number of query tokens among request in the batch.
143
+ max_decode_query_len: Optional[int] = None
144
+
145
+ # (batch_size + 1,). The cumulative subquery lengths of the sequences in
146
+ # the batch, used to index into subquery. E.g., if the subquery length
147
+ # is [4, 6], it is [0, 4, 10].
148
+ query_start_loc: Optional[torch.Tensor] = None
149
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
150
+ # the batch, used to index into sequence. E.g., if the sequence length is
151
+ # [4, 6], it is [0, 4, 10].
152
+ seq_start_loc: Optional[torch.Tensor] = None
153
+
154
_cached_prefill_metadata: Optional["FlashAttentionMetadata"] = None
155
_cached_decode_metadata: Optional["FlashAttentionMetadata"] = None
156
157
+ # Begin encoder attn & enc/dec cross-attn fields...
158
+
159
+ # Encoder sequence lengths representation
160
+ encoder_seq_lens: Optional[List[int]] = None
161
+ encoder_seq_lens_tensor: Optional[torch.Tensor] = None
162
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
163
+ # the batch, used to index into sequence. E.g., if the sequence length is
164
+ # [4, 6], it is [0, 4, 10].
165
+ encoder_seq_start_loc: Optional[torch.Tensor] = None
166
+ # Maximum sequence length among encoder sequences
167
+ max_encoder_seq_len: Optional[int] = None
168
+ # Number of tokens input to encoder
169
+ num_encoder_tokens: Optional[int] = None
170
+
171
+ # Cross-attention memory-mapping data structures: slot mapping
172
+ # and block tables
173
+ cross_slot_mapping: Optional[torch.Tensor] = None
174
+ cross_block_tables: Optional[torch.Tensor] = None
175
+
176
+ @property
177
+ def is_all_encoder_attn_metadata_set(self):
178
+ '''
179
+ All attention metadata required for encoder attention is set.
180
+ '''
181
+ return is_all_encoder_attn_metadata_set(self)
182
+
183
+ @property
184
+ def is_all_cross_attn_metadata_set(self):
185
+ '''
186
+ All attention metadata required for enc/dec cross-attention is set.
187
+
188
+ Superset of encoder attention required metadata.
189
+ '''
190
+ return is_all_cross_attn_metadata_set(self)
191
+
192
@property
193
def prefill_metadata(self) -> Optional["FlashAttentionMetadata"]:
194
if self.num_prefills == 0:
@@ -159,32 +197,52 @@ class FlashAttentionMetadata(AttentionMetadata):
197
if self._cached_prefill_metadata is not None:
198
return self._cached_prefill_metadata
199
162
- assert self.seq_lens is not None
163
- assert self.seq_lens_tensor is not None
164
- assert self.query_start_loc is not None
165
- assert self.context_lens_tensor is not None
166
- assert self.block_tables is not None
167
- assert self.seq_start_loc is not None
200
+ assert ((self.seq_lens is not None)
201
+ or (self.encoder_seq_lens is not None))
202
+ assert ((self.seq_lens_tensor is not None)
203
+ or (self.encoder_seq_lens_tensor is not None))
204
+
205
+ # Compute some attn_metadata fields which default to None
206
+ query_start_loc = (None if self.query_start_loc is None else
207
+ self.query_start_loc[:self.num_prefills + 1])
208
+ slot_mapping = (None if self.slot_mapping is None else
209
+ self.slot_mapping[:self.num_prefill_tokens])
210
+ seq_lens = (None if self.seq_lens is None else
211
+ self.seq_lens[:self.num_prefills])
212
+ seq_lens_tensor = (None if self.seq_lens_tensor is None else
213
+ self.seq_lens_tensor[:self.num_prefills])
214
+ seq_start_loc = (None if self.seq_start_loc is None else
215
+ self.seq_start_loc[:self.num_prefills + 1])
216
+ context_lens_tensor = (None if self.context_lens_tensor is None else
217
+ self.context_lens_tensor[:self.num_prefills])
218
+ block_tables = (None if self.block_tables is None else
219
+ self.block_tables[:self.num_prefills])
220
221
self._cached_prefill_metadata = FlashAttentionMetadata(
222
num_prefills=self.num_prefills,
223
num_prefill_tokens=self.num_prefill_tokens,
224
num_decode_tokens=0,
173
- slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
225
+ slot_mapping=slot_mapping,
226
multi_modal_placeholder_index_maps=self.
227
multi_modal_placeholder_index_maps,
176
- seq_lens=self.seq_lens[:self.num_prefills],
177
- seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
228
+ seq_lens=seq_lens,
229
+ seq_lens_tensor=seq_lens_tensor,
230
max_query_len=self.max_query_len,
231
max_prefill_seq_len=self.max_prefill_seq_len,
232
max_decode_query_len=0,
233
max_decode_seq_len=0,
182
- query_start_loc=self.query_start_loc[:self.num_prefills + 1],
183
- seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
184
- context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
185
- block_tables=self.block_tables[:self.num_prefills],
234
+ query_start_loc=query_start_loc,
235
+ seq_start_loc=seq_start_loc,
236
+ context_lens_tensor=context_lens_tensor,
237
+ block_tables=block_tables,
238
use_cuda_graph=False,
187
- )
239
+ # Begin encoder & cross attn fields below...
240
+ encoder_seq_lens=self.encoder_seq_lens,
241
+ encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
242
+ encoder_seq_start_loc=self.encoder_seq_start_loc,
243
+ max_encoder_seq_len=self.max_encoder_seq_len,
244
+ cross_slot_mapping=self.cross_slot_mapping,
245
+ cross_block_tables=self.cross_block_tables)
246
return self._cached_prefill_metadata
247
248
@property
@@ -194,17 +252,25 @@ class FlashAttentionMetadata(AttentionMetadata):
252
253
if self._cached_decode_metadata is not None:
254
return self._cached_decode_metadata
197
- assert self.block_tables is not None
198
- assert self.seq_lens_tensor is not None
255
+ assert ((self.seq_lens_tensor is not None)
256
+ or (self.encoder_seq_lens_tensor is not None))
257
+
258
+ # Compute some attn_metadata fields which default to None
259
+ slot_mapping = (None if self.slot_mapping is None else
260
+ self.slot_mapping[self.num_prefill_tokens:])
261
+ seq_lens_tensor = (None if self.seq_lens_tensor is None else
262
+ self.seq_lens_tensor[self.num_prefills:])
263
+ block_tables = (None if self.block_tables is None else
264
+ self.block_tables[self.num_prefills:])
265
266
self._cached_decode_metadata = FlashAttentionMetadata(
267
num_prefills=0,
268
num_prefill_tokens=0,
269
num_decode_tokens=self.num_decode_tokens,
204
- slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
270
+ slot_mapping=slot_mapping,
271
multi_modal_placeholder_index_maps=None,
272
seq_lens=None,
207
- seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
273
+ seq_lens_tensor=seq_lens_tensor,
274
max_decode_query_len=self.max_decode_query_len,
275
max_query_len=self.max_query_len,
276
max_prefill_seq_len=0,
@@ -214,9 +280,15 @@ class FlashAttentionMetadata(AttentionMetadata):
280
seq_start_loc=self.seq_start_loc[self.num_prefills:]
281
if self.seq_start_loc is not None else None,
282
context_lens_tensor=None,
217
- block_tables=self.block_tables[self.num_prefills:],
283
+ block_tables=block_tables,
284
use_cuda_graph=self.use_cuda_graph,
219
- )
285
+ # Begin encoder & cross attn fields below...
286
+ encoder_seq_lens=self.encoder_seq_lens,
287
+ encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
288
+ encoder_seq_start_loc=self.encoder_seq_start_loc,
289
+ max_encoder_seq_len=self.max_encoder_seq_len,
290
+ cross_slot_mapping=self.cross_slot_mapping,
291
+ cross_block_tables=self.cross_block_tables)
292
return self._cached_decode_metadata
293
294
def advance_step(self,
@@ -586,16 +658,20 @@ class FlashAttentionImpl(AttentionImpl):
658
Returns:
659
shape = [num_tokens, num_heads * head_size]
660
"""
589
- if attn_type != AttentionType.DECODER:
590
- raise NotImplementedError("Encoder self-attention and "
591
- "encoder/decoder cross-attention "
592
- "are not implemented for "
593
- "FlashAttentionImpl")
594
-
661
# NOTE(woosuk): FlashAttention does not support FP8 KV cache.
662
assert k_scale == 1.0 and v_scale == 1.0, (
663
"key/v_scale is not supported in FlashAttention.")
664
665
+ if (attn_type == AttentionType.ENCODER
666
+ and (not attn_metadata.is_all_encoder_attn_metadata_set)):
667
+ raise AttributeError("Encoder attention requires setting "
668
+ "encoder metadata attributes.")
669
+ elif (attn_type == AttentionType.ENCODER_DECODER
670
+ and (not attn_metadata.is_all_cross_attn_metadata_set)):
671
+ raise AttributeError("Encoder/decoder cross-attention "
672
+ "requires setting cross-attention "
673
+ "metadata attributes.")
674
+
675
output = torch.ops.vllm.unified_flash_attention(
676
query,
677
key,
@@ -608,6 +684,7 @@ class FlashAttentionImpl(AttentionImpl):
684
k_scale,
685
v_scale,
686
self.scale,
687
+ attn_type.value,
688
self.sliding_window,
689
self.alibi_slopes,
690
self.logits_soft_cap,
@@ -616,6 +693,89 @@ class FlashAttentionImpl(AttentionImpl):
693
return output
694
695
696
+def _get_query_key_seq_metadata(
697
+ attn_metadata,
698
+ is_prompt: bool,
699
+ attn_type: AttentionType,
700
+) -> tuple:
701
+ """
702
+ Returns sequence metadata for key and query based on the specified
703
+ attention type and whether input is a prompt.
704
+
705
+ This function computes the starting locations and maximum sequence lengths
706
+ for key and query sequences for different attention types.
707
+
708
+ Args:
709
+ attn_metadata: The attention metadata object
710
+ is_prompt (bool): A flag indicating if the input is a prompt
711
+ attn_type (AttentionType): The type of attention being used.
712
+
713
+ Returns:
714
+ tuple: A tuple containing four integers:
715
+ - Starting location for the query sequence.
716
+ - Maximum sequence length for the query sequence.
717
+ - Starting location for the key sequence.
718
+ - Maximum sequence length for the key sequence.
719
+
720
+ Raises:
721
+ AttributeError: If an invalid attention type is provided.
722
+ """
723
+ if attn_type == AttentionType.DECODER:
724
+ # Decoder self-attention
725
+ # Choose max_seq_len based on whether we are in prompt_run
726
+ if is_prompt:
727
+ max_seq_len = attn_metadata.max_prefill_seq_len
728
+ else:
729
+ max_seq_len = attn_metadata.max_decode_seq_len
730
+ return (attn_metadata.seq_start_loc, max_seq_len,
731
+ attn_metadata.seq_start_loc, max_seq_len)
732
+
733
+ elif attn_type == AttentionType.ENCODER_DECODER:
734
+ # This is cross attention between the where the key
735
+ # is the precomputed encoder attention and query
736
+ # is the input sequence.
737
+ # Choose query max length based on whether it is prompt
738
+ # or not.
739
+ if is_prompt:
740
+ max_seq_len = attn_metadata.max_prefill_seq_len
741
+ else:
742
+ max_seq_len = attn_metadata.max_decode_seq_len
743
+ return (attn_metadata.seq_start_loc, max_seq_len,
744
+ attn_metadata.encoder_seq_start_loc,
745
+ attn_metadata.max_encoder_seq_len)
746
+ elif attn_type == AttentionType.ENCODER:
747
+ # For encoder attention both the query and the key are same i.e the
748
+ # encoder sequence.
749
+ return (attn_metadata.encoder_seq_start_loc,
750
+ attn_metadata.max_encoder_seq_len,
751
+ attn_metadata.encoder_seq_start_loc,
752
+ attn_metadata.max_encoder_seq_len)
753
+ elif attn_type == AttentionType.ENCODER_ONLY:
754
+ assert is_prompt, "Should not have decode for encoder only model."
755
+ return (attn_metadata.seq_start_loc, attn_metadata.max_prefill_seq_len,
756
+ attn_metadata.seq_start_loc, attn_metadata.max_prefill_seq_len)
757
+ else:
758
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
759
+
760
+
761
+def _get_causal_option(attn_type: AttentionType) -> bool:
762
+ """
763
+ Determine whether the given attention type is suitable for causal
764
+ attention mechanisms.
765
+
766
+ Args:
767
+ attn_type (AttentionType): The type of attention being evaluated
768
+
769
+ Returns:
770
+ bool: Returns `True` if the attention type is suitable for causal
771
+ attention (i.e., not encoder, encoder-only, or encoder-decoder),
772
+ otherwise returns `False`.
773
+ """
774
+ return not (attn_type == AttentionType.ENCODER
775
+ or attn_type == AttentionType.ENCODER_ONLY
776
+ or attn_type == AttentionType.ENCODER_DECODER)
777
+
778
+
779
def unified_flash_attention(
780
query: torch.Tensor,
781
key: torch.Tensor,
@@ -628,60 +788,76 @@ def unified_flash_attention(
788
k_scale: float,
789
v_scale: float,
790
softmax_scale: float,
791
+ attn_type_int_val: int,
792
window_size: Optional[List[int]] = None,
793
alibi_slopes: Optional[torch.Tensor] = None,
794
logits_soft_cap: Optional[float] = None,
795
) -> torch.Tensor:
796
797
+ # Convert integer attn_type to enum
798
+ try:
799
+ attn_type = AttentionType(attn_type_int_val)
800
+ except ValueError as err:
801
+ raise AttributeError(
802
+ f"Invalid attention type {str(attn_type_int_val)}") from err
803
+
804
current_metadata = get_forward_context()
805
assert current_metadata is not None
806
assert isinstance(current_metadata, FlashAttentionMetadata)
807
attn_metadata: FlashAttentionMetadata = current_metadata
808
809
num_tokens, hidden_size = query.shape
810
+
811
# Reshape the query, key, and value tensors.
812
query = query.view(-1, num_heads, head_size)
644
- key = key.view(-1, num_kv_heads, head_size)
645
- value = value.view(-1, num_kv_heads, head_size)
813
+ if (key is not None) and (value is not None):
814
+ key = key.view(-1, num_kv_heads, head_size)
815
+ value = value.view(-1, num_kv_heads, head_size)
816
817
if kv_cache.numel() > 0:
818
key_cache = kv_cache[0]
819
value_cache = kv_cache[1]
820
+ # We skip updating the KV cache under two conditions:
821
+ # a. When the Attention Type is ENCODER. In this phase, we compute
822
+ # only the encoder attention without updating the cache.
823
+ # b. When both Key and Value are None. This occurs during
824
+ # cross-attention computation in the decoding phase, where the KV
825
+ # cache is already populated with the cross-attention tensor.
826
+ # Thus, we skip cache updates during this time.
827
+ if (attn_type != AttentionType.ENCODER) and (key is not None) and (
828
+ value is not None):
829
+ if attn_type == AttentionType.ENCODER_DECODER:
830
+ # Update cross-attention KV cache (prefill-only)
831
+ updated_slot_mapping = attn_metadata.cross_slot_mapping
832
+ else:
833
+ # Update self-attention KV cache (prefill/decode)
834
+ updated_slot_mapping = attn_metadata.slot_mapping
835
+
836
+ # Reshape the input keys and values and store them in the cache.
837
+ # If kv_cache is not provided, the new key and value tensors are
838
+ # not cached. This happens during the initial memory profiling run.
839
+ torch.ops._C_cache_ops.reshape_and_cache_flash(
840
+ key,
841
+ value,
842
+ kv_cache[0],
843
+ kv_cache[1],
844
+ updated_slot_mapping.flatten(), # type: ignore[union-attr]
845
+ kv_cache_dtype,
846
+ k_scale,
847
+ v_scale,
848
+ )
849
651
- # Reshape the input keys and values and store them in the cache.
652
- # If kv_cache is not provided, the new key and value tensors are
653
- # not cached. This happens during the initial memory profiling run.
654
- torch.ops._C_cache_ops.reshape_and_cache_flash(
655
- key,
656
- value,
657
- kv_cache[0],
658
- kv_cache[1],
659
- attn_metadata.slot_mapping.flatten(),
660
- kv_cache_dtype,
661
- k_scale,
662
- v_scale,
663
- )
664
-
665
- num_prefill_tokens = attn_metadata.num_prefill_tokens
666
- num_decode_tokens = attn_metadata.num_decode_tokens
667
- assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
668
- f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
669
- assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
670
- f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
671
-
672
- # Query for decode. KV is not needed because it is already cached.
673
- decode_query = query[num_prefill_tokens:]
850
+ (num_prefill_query_tokens, num_prefill_kv_tokens,
851
+ num_decode_query_tokens) = \
852
+ get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
853
+ decode_query = query[num_prefill_query_tokens:]
854
# QKV for prefill.
675
- query = query[:num_prefill_tokens]
676
- key = key[:num_prefill_tokens]
677
- value = value[:num_prefill_tokens]
678
-
679
- assert query.shape[0] == num_prefill_tokens
680
- assert decode_query.shape[0] == num_decode_tokens
855
+ query = query[:num_prefill_query_tokens]
856
+ assert query.shape[0] == num_prefill_query_tokens
857
+ assert decode_query.shape[0] == num_decode_query_tokens
858
859
prefill_output: Optional[torch.Tensor] = None
860
decode_output: Optional[torch.Tensor] = None
684
-
861
if prefill_meta := attn_metadata.prefill_metadata:
862
# Prompt run.
863
if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
@@ -689,22 +865,30 @@ def unified_flash_attention(
865
# normal attention
866
# When block_tables are not filled, it means q and k are the
867
# prompt, and they have the same length.
868
+ q_seq_start_loc, q_seq_len, k_seq_start_loc, k_seq_len = \
869
+ _get_query_key_seq_metadata(prefill_meta, True, attn_type)
870
+
871
+ key = key[:num_prefill_kv_tokens]
872
+ value = value[:num_prefill_kv_tokens]
873
+
874
prefill_output = flash_attn_varlen_func(
875
q=query,
876
k=key,
877
v=value,
696
- cu_seqlens_q=prefill_meta.seq_start_loc,
697
- cu_seqlens_k=prefill_meta.seq_start_loc,
698
- max_seqlen_q=prefill_meta.max_prefill_seq_len,
699
- max_seqlen_k=prefill_meta.max_prefill_seq_len,
878
+ cu_seqlens_q=q_seq_start_loc,
879
+ cu_seqlens_k=k_seq_start_loc,
880
+ max_seqlen_q=q_seq_len,
881
+ max_seqlen_k=k_seq_len,
882
softmax_scale=softmax_scale,
701
- causal=True,
883
+ causal=_get_causal_option(attn_type),
884
window_size=window_size,
885
alibi_slopes=alibi_slopes,
886
softcap=logits_soft_cap,
887
)
888
else:
889
# prefix-enabled attention
890
+ assert attn_type == AttentionType.DECODER, (
891
+ "Only decoder-only models support prefix caching")
892
assert prefill_meta.seq_lens is not None
893
max_seq_len = max(prefill_meta.seq_lens)
894
prefill_output = flash_attn_varlen_func( # noqa
@@ -729,6 +913,8 @@ def unified_flash_attention(
913
# because different queries might have different lengths.
914
assert decode_meta.max_decode_query_len is not None
915
if decode_meta.max_decode_query_len > 1:
916
+ assert attn_type == AttentionType.DECODER, (
917
+ "Only decoder-only models support max_decode_query_len > 1")
918
decode_output = flash_attn_varlen_func(
919
q=decode_query,
920
k=key_cache,
@@ -746,12 +932,17 @@ def unified_flash_attention(
932
)
933
else:
934
# Use flash_attn_with_kvcache for normal decoding.
935
+ (
936
+ seq_lens_arg,
937
+ _,
938
+ block_tables_arg,
939
+ ) = get_seq_len_block_table_args(decode_meta, False, attn_type)
940
decode_output = flash_attn_with_kvcache(
941
q=decode_query.unsqueeze(1),
942
k_cache=key_cache,
943
v_cache=value_cache,
753
- block_table=decode_meta.block_tables,
754
- cache_seqlens=decode_meta.seq_lens_tensor,
944
+ block_table=block_tables_arg,
945
+ cache_seqlens=seq_lens_arg,
946
softmax_scale=softmax_scale,
947
causal=True,
948
window_size=window_size,
@@ -761,10 +952,10 @@ def unified_flash_attention(
952
953
if prefill_output is None:
954
assert decode_output is not None
764
- return decode_output.view(num_decode_tokens, hidden_size)
955
+ return decode_output.view(num_decode_query_tokens, hidden_size)
956
if decode_output is None:
957
assert prefill_output is not None
767
- return prefill_output.view(num_prefill_tokens, hidden_size)
958
+ return prefill_output.view(num_prefill_query_tokens, hidden_size)
959
960
# Chunked prefill does not work with speculative decoding.
961
# Therefore, the query length for decode should be 1 in chunked prefill.
@@ -786,6 +977,7 @@ def unified_flash_attention_fake(
977
k_scale: float,
978
v_scale: float,
979
softmax_scale: float,
980
+ attn_type_int_val: int,
981
window_size: Optional[List[int]] = None,
982
alibi_slopes: Optional[torch.Tensor] = None,
983
logits_soft_cap: Optional[float] = None,
vllm/attention/backends/utils.py
+143
-16
@@ -1,13 +1,14 @@
1
"""Attention backend utils"""
2
from collections import defaultdict
3
from contextlib import contextmanager
4
-from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union
4
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type, TypeVar, Union
5
6
import numpy as np
7
import torch
8
9
from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder,
10
AttentionState)
11
+from vllm.attention.backends.abstract import AttentionType
12
from vllm.multimodal import MultiModalPlaceholderMap
13
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
14
@@ -336,11 +337,13 @@ class CommonAttentionState(AttentionState):
337
use_cuda_graph=True,
338
)
339
if is_encoder_decoder_model:
339
- # The encoder decoder model works only with XFormers backend.
340
- # Assert the same.
341
- assert self.runner.attn_backend.get_name() == "XFORMERS", \
342
- f"Expected attn_backend name to be 'XFORMERS', but "\
343
- f" got '{self.runner.attn_backend.get_name()}'"
340
+ # The encoder decoder model works only with XFormers and
341
+ # Flash Attention backend. Assert the same.
342
+ assert self.runner.attn_backend.get_name() in\
343
+ ["XFORMERS", "FLASH_ATTN"], \
344
+ f"Expected attn_backend name to be either 'XFORMERS' or " \
345
+ f"'FLASH_ATTN', but "\
346
+ f"got '{self.runner.attn_backend.get_name()}'"
347
self._update_captured_metadata_for_enc_dec_model(
348
batch_size=batch_size, attn_metadata=attn_metadata)
349
@@ -356,11 +359,13 @@ class CommonAttentionState(AttentionState):
359
"block_tables": attn_metadata.decode_metadata.block_tables,
360
}
361
if is_encoder_decoder_model:
359
- # The encoder decoder model works only with XFormers backend.
360
- # Assert the same.
361
- assert self.runner.attn_backend.get_name() == "XFORMERS", \
362
- f"Expected attn_backend name to be 'XFORMERS', but "\
363
- f" got '{self.runner.attn_backend.get_name()}'"
362
+ # The encoder decoder model works only with XFormers and
363
+ # Flash Attention backend. Assert the same.
364
+ assert self.runner.attn_backend.get_name() in\
365
+ ["XFORMERS", "FLASH_ATTN"], \
366
+ f"Expected attn_backend name to be either 'XFORMERS' or "\
367
+ f"'FLASH_ATTN', but "\
368
+ f"got '{self.runner.attn_backend.get_name()}'"
369
self._add_additonal_input_buffers_for_enc_dec_model(
370
attn_metadata=attn_metadata, input_buffers=input_buffers)
371
return input_buffers
@@ -375,11 +380,13 @@ class CommonAttentionState(AttentionState):
380
input_buffers["block_tables"].copy_(
381
attn_metadata.decode_metadata.block_tables, non_blocking=True)
382
if is_encoder_decoder_model:
378
- # The encoder decoder model works only with XFormers backend.
379
- # Assert the same.
380
- assert self.runner.attn_backend.get_name() == "XFORMERS", \
381
- f"Expected attn_backend name to be 'XFORMERS', but "\
382
- f" got '{self.runner.attn_backend.get_name()}'"
383
+ # The encoder decoder model works only with XFormers and
384
+ # Flash Attention backend. Assert the same.
385
+ assert self.runner.attn_backend.get_name() in\
386
+ ["XFORMERS", "FLASH_ATTN"], \
387
+ f"Expected attn_backend name to be either 'XFORMERS' or "\
388
+ f"'FLASH_ATTN', but "\
389
+ f"got '{self.runner.attn_backend.get_name()}'"
390
self._prepare_input_buffers_for_enc_dec_model(
391
attn_metadata, input_buffers)
392
@@ -411,6 +418,7 @@ class CommonAttentionState(AttentionState):
418
attn_metadata.encoder_seq_lens_tensor = torch.full(
419
(batch_size, ), 1, dtype=torch.int).cuda()
420
attn_metadata.max_encoder_seq_len = self.runner.max_seq_len_to_capture
421
+ attn_metadata.num_encoder_tokens = 0
422
423
def _add_additonal_input_buffers_for_enc_dec_model(
424
self, attn_metadata, input_buffers: Dict[str, Any]):
@@ -453,3 +461,122 @@ class CommonAttentionState(AttentionState):
461
input_buffers["cross_block_tables"].copy_(
462
attn_metadata.decode_metadata.cross_block_tables,
463
non_blocking=True)
464
+
465
+
466
+def is_all_encoder_attn_metadata_set(attn_metadata):
467
+ '''
468
+ All attention metadata required for encoder attention is set.
469
+ '''
470
+ return ((attn_metadata.encoder_seq_lens is not None)
471
+ and (attn_metadata.encoder_seq_lens_tensor is not None)
472
+ and (attn_metadata.max_encoder_seq_len is not None))
473
+
474
+
475
+def is_all_cross_attn_metadata_set(attn_metadata):
476
+ '''
477
+ All attention metadata required for enc/dec cross-attention is set.
478
+
479
+ Superset of encoder attention required metadata.
480
+ '''
481
+ return (attn_metadata.is_all_encoder_attn_metadata_set
482
+ and (attn_metadata.cross_slot_mapping is not None)
483
+ and (attn_metadata.cross_block_tables is not None))
484
+
485
+
486
+def get_seq_len_block_table_args(
487
+ attn_metadata,
488
+ is_prompt: bool,
489
+ attn_type: AttentionType,
490
+) -> tuple:
491
+ '''
492
+ The particular choice of sequence-length- and block-table-related
493
+ attributes which should be extracted from attn_metadata is dependent
494
+ on the type of attention operation.
495
+
496
+ Decoder attn -> select entirely decoder self-attention-related fields
497
+ Encoder/decoder cross-attn -> select encoder sequence lengths &
498
+ cross-attn block-tables fields
499
+ Encoder attn -> select encoder sequence lengths fields & no block tables
500
+
501
+ Arguments:
502
+
503
+ * attn_metadata: Attention metadata structure associated with attention op
504
+ * is_prompt: True if prefill, False otherwise
505
+ * attn_type: encoder attention, decoder self-attention,
506
+ encoder/decoder cross-attention
507
+
508
+ Returns:
509
+
510
+ * Appropriate sequence-lengths tensor
511
+ * Appropriate max sequence-length scalar
512
+ * Appropriate block tables (or None)
513
+ '''
514
+
515
+ if attn_type == AttentionType.DECODER:
516
+ # Decoder self-attention
517
+ # Choose max_seq_len based on whether we are in prompt_run
518
+ if is_prompt:
519
+ max_seq_len = attn_metadata.max_prefill_seq_len
520
+ else:
521
+ max_seq_len = attn_metadata.max_decode_seq_len
522
+ return (attn_metadata.seq_lens_tensor, max_seq_len,
523
+ attn_metadata.block_tables)
524
+ elif attn_type == AttentionType.ENCODER_DECODER:
525
+ # Enc/dec cross-attention KVs match encoder sequence length;
526
+ # cross-attention utilizes special "cross" block tables
527
+ return (attn_metadata.encoder_seq_lens_tensor,
528
+ attn_metadata.max_encoder_seq_len,
529
+ attn_metadata.cross_block_tables)
530
+ elif attn_type == AttentionType.ENCODER:
531
+ # No block tables associated with encoder attention
532
+ return (attn_metadata.encoder_seq_lens_tensor,
533
+ attn_metadata.max_encoder_seq_len, None)
534
+ else:
535
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
536
+
537
+
538
+def get_num_prefill_decode_query_kv_tokens(
539
+ attn_metadata,
540
+ attn_type: AttentionType,
541
+) -> Tuple[int, int, int]:
542
+ """
543
+ Calculate the number of prefill and decode tokens for query, key/value
544
+ based on the attention metadata and the specified attention type.
545
+
546
+ Args:
547
+ attn_metadata (FlashAttentionMetadata): Attention Metadata object.
548
+ attn_type (AttentionType): The type of attention being used.
549
+ Returns:
550
+ Tuple[int, int, int]: A tuple containing three integers:
551
+ - The number of prefill query tokens.
552
+ - The number of prefill key/value tokens.
553
+ - The number of decode query tokens.
554
+
555
+ Raises:
556
+ AssertionError: If the number of encoder tokens in `attn_metadata`
557
+ is `None` when required for the calculations.
558
+ """
559
+ num_prefill_query_tokens = 0
560
+ num_decode_query_tokens = 0
561
+ num_prefill_kv_tokens = 0
562
+ if attn_type == AttentionType.ENCODER:
563
+ # Encoder attention is only invoked during prefill phase.
564
+ # The same input servers a both query and key.
565
+ assert attn_metadata.num_encoder_tokens is not None
566
+ num_prefill_query_tokens = attn_metadata.num_encoder_tokens
567
+ num_prefill_kv_tokens = attn_metadata.num_encoder_tokens
568
+ num_decode_query_tokens = 0
569
+ elif attn_type == AttentionType.ENCODER_DECODER:
570
+ assert attn_metadata.num_encoder_tokens is not None
571
+ num_prefill_query_tokens = attn_metadata.num_prefill_tokens
572
+ # The key is the encoder/cross-attention.
573
+ num_prefill_kv_tokens = attn_metadata.num_encoder_tokens
574
+ num_decode_query_tokens = attn_metadata.num_decode_tokens
575
+ else: # attn_type == AttentionType.DECODER or
576
+ # attn_type == AttentionType.ENCODER_ONLY
577
+ num_prefill_query_tokens = attn_metadata.num_prefill_tokens
578
+ num_prefill_kv_tokens = attn_metadata.num_prefill_tokens
579
+ num_decode_query_tokens = attn_metadata.num_decode_tokens
580
+
581
+ return (num_prefill_query_tokens, num_prefill_kv_tokens,
582
+ num_decode_query_tokens)
vllm/attention/backends/xformers.py
+26
-105
@@ -11,8 +11,10 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
11
12
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13
AttentionMetadata, AttentionType)
14
-from vllm.attention.backends.utils import (CommonAttentionState,
15
- CommonMetadataBuilder)
14
+from vllm.attention.backends.utils import (
15
+ CommonAttentionState, CommonMetadataBuilder,
16
+ get_num_prefill_decode_query_kv_tokens, get_seq_len_block_table_args,
17
+ is_all_cross_attn_metadata_set, is_all_encoder_attn_metadata_set)
18
from vllm.attention.ops.paged_attn import (PagedAttention,
19
PagedAttentionMetadata)
20
from vllm.logger import init_logger
@@ -135,6 +137,11 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
137
# Encoder sequence lengths representation
138
encoder_seq_lens: Optional[List[int]] = None
139
encoder_seq_lens_tensor: Optional[torch.Tensor] = None
140
+ # FIXME: It is for flash attn.
141
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
142
+ # the batch, used to index into sequence. E.g., if the sequence length is
143
+ # [4, 6], it is [0, 4, 10].
144
+ encoder_seq_start_loc: Optional[torch.Tensor] = None
145
146
# Maximum sequence length among encoder sequences
147
max_encoder_seq_len: Optional[int] = None
@@ -162,9 +169,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
169
'''
170
All attention metadata required for encoder attention is set.
171
'''
165
- return ((self.encoder_seq_lens is not None)
166
- and (self.encoder_seq_lens_tensor is not None)
167
- and (self.max_encoder_seq_len is not None))
172
+ return is_all_encoder_attn_metadata_set(self)
173
174
@property
175
def is_all_cross_attn_metadata_set(self):
@@ -173,9 +178,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
178
179
Superset of encoder attention required metadata.
180
'''
176
- return (self.is_all_encoder_attn_metadata_set
177
- and (self.cross_slot_mapping is not None)
178
- and (self.cross_block_tables is not None))
181
+ return is_all_cross_attn_metadata_set(self)
182
183
@property
184
def prefill_metadata(self) -> Optional["XFormersMetadata"]:
@@ -329,64 +332,6 @@ def _set_attn_bias(
332
raise AttributeError(f"Invalid attention type {str(attn_type)}")
333
334
332
-def _get_seq_len_block_table_args(
333
- attn_metadata: XFormersMetadata,
334
- is_prompt: bool,
335
- attn_type: AttentionType,
336
-) -> tuple:
337
- '''
338
- The particular choice of sequence-length- and block-table-related
339
- attributes which should be extracted from attn_metadata is dependent
340
- on the type of attention operation.
341
-
342
- Decoder attn -> select entirely decoder self-attention-related fields
343
- Encoder/decoder cross-attn -> select encoder sequence lengths &
344
- cross-attn block-tables fields
345
- Encoder attn -> select encoder sequence lengths fields & no block tables
346
-
347
- Arguments:
348
-
349
- * attn_metadata: Attention metadata structure associated with attention op
350
- * is_prompt: True if prefill, False otherwise
351
- * attn_type: encoder attention, decoder self-attention,
352
- encoder/decoder cross-attention
353
-
354
- Returns:
355
-
356
- * Appropriate sequence-lengths tensor
357
- * Appropriate max sequence-length scalar
358
- * Appropriate block tables (or None)
359
- '''
360
-
361
- if attn_type == AttentionType.DECODER:
362
- # Decoder self-attention
363
- # Choose max_seq_len based on whether we are in prompt_run
364
- if is_prompt:
365
- max_seq_len = attn_metadata.max_prefill_seq_len
366
- else:
367
- max_seq_len = attn_metadata.max_decode_seq_len
368
- return (attn_metadata.seq_lens_tensor, max_seq_len,
369
- attn_metadata.block_tables)
370
- elif attn_type == AttentionType.ENCODER_DECODER:
371
- # Enc/dec cross-attention KVs match encoder sequence length;
372
- # cross-attention utilizes special "cross" block tables
373
- return (attn_metadata.encoder_seq_lens_tensor,
374
- attn_metadata.max_encoder_seq_len,
375
- attn_metadata.cross_block_tables)
376
- elif attn_type == AttentionType.ENCODER:
377
- # No block tables associated with encoder attention
378
- return (attn_metadata.encoder_seq_lens_tensor,
379
- attn_metadata.max_encoder_seq_len, None)
380
- elif attn_type == AttentionType.ENCODER_ONLY:
381
- assert is_prompt, "Should not have decode for encoder only model."
382
-
383
- # No block tables associated with encoder attention
384
- return (attn_metadata.seq_lens_tensor,
385
- attn_metadata.max_prefill_seq_len, None)
386
- else:
387
- raise AttributeError(f"Invalid attention type {str(attn_type)}")
388
-
389
-
335
class XFormersMetadataBuilder(CommonMetadataBuilder[XFormersMetadata]):
336
337
_metadata_cls = XFormersMetadata
@@ -574,45 +519,21 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
519
updated_slot_mapping,
520
self.kv_cache_dtype,
521
k_scale, v_scale)
577
-
578
- if attn_type == AttentionType.ENCODER:
579
- # Encoder attention - chunked prefill is not applicable;
580
- # derive token-count from query shape & and treat them
581
- # as 100% prefill tokens
582
- assert attn_metadata.num_encoder_tokens is not None
583
- num_prefill_tokens = attn_metadata.num_encoder_tokens
584
- num_encoder_tokens = attn_metadata.num_encoder_tokens
585
- num_decode_tokens = 0
586
- elif attn_type == AttentionType.DECODER:
587
- # Decoder self-attention supports chunked prefill.
588
- num_prefill_tokens = attn_metadata.num_prefill_tokens
589
- num_encoder_tokens = attn_metadata.num_prefill_tokens
590
- num_decode_tokens = attn_metadata.num_decode_tokens
591
- # Only enforce this shape-constraint for decoder
592
- # self-attention
593
- assert key.shape[0] == num_prefill_tokens + num_decode_tokens
594
- assert value.shape[0] == num_prefill_tokens + num_decode_tokens
595
- else: # attn_type == AttentionType.ENCODER_DECODER
596
- # Encoder/decoder cross-attention requires no chunked
597
- # prefill (100% prefill or 100% decode tokens, no mix)
598
- num_prefill_tokens = attn_metadata.num_prefill_tokens
599
- if attn_metadata.num_encoder_tokens is not None:
600
- num_encoder_tokens = attn_metadata.num_encoder_tokens
601
- else:
602
- num_encoder_tokens = attn_metadata.num_prefill_tokens
603
- num_decode_tokens = attn_metadata.num_decode_tokens
522
+ (num_prefill_query_tokens, num_prefill_kv_tokens,
523
+ num_decode_query_tokens) = \
524
+ get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
525
526
output = torch.empty_like(query)
527
# Query for decode. KV is not needed because it is already cached.
607
- decode_query = query[num_prefill_tokens:]
528
+ decode_query = query[num_prefill_query_tokens:]
529
# QKV for prefill.
609
- query = query[:num_prefill_tokens]
530
+ query = query[:num_prefill_query_tokens]
531
if key is not None and value is not None:
611
- key = key[:num_encoder_tokens]
612
- value = value[:num_encoder_tokens]
532
+ key = key[:num_prefill_kv_tokens]
533
+ value = value[:num_prefill_kv_tokens]
534
614
- assert query.shape[0] == num_prefill_tokens
615
- assert decode_query.shape[0] == num_decode_tokens
535
+ assert query.shape[0] == num_prefill_query_tokens
536
+ assert decode_query.shape[0] == num_decode_query_tokens
537
538
if prefill_meta := attn_metadata.prefill_metadata:
539
# Prompt run.
@@ -622,8 +543,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
543
# prefix.
544
out = self._run_memory_efficient_xformers_forward(
545
query, key, value, prefill_meta, attn_type=attn_type)
625
- assert out.shape == output[:num_prefill_tokens].shape
626
- output[:num_prefill_tokens] = out
546
+ assert out.shape == output[:num_prefill_query_tokens].shape
547
+ output[:num_prefill_query_tokens] = out
548
else:
549
assert attn_type != AttentionType.ENCODER_ONLY, (
550
"Encoder-only models should not have prefix attention.")
@@ -652,8 +573,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
573
k_scale,
574
v_scale,
575
)
655
- assert output[:num_prefill_tokens].shape == out.shape
656
- output[:num_prefill_tokens] = out
576
+ assert output[:num_prefill_query_tokens].shape == out.shape
577
+ output[:num_prefill_query_tokens] = out
578
579
if decode_meta := attn_metadata.decode_metadata:
580
assert attn_type != AttentionType.ENCODER_ONLY, (
@@ -663,9 +584,9 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
584
seq_lens_arg,
585
max_seq_len_arg,
586
block_tables_arg,
666
- ) = _get_seq_len_block_table_args(decode_meta, False, attn_type)
587
+ ) = get_seq_len_block_table_args(decode_meta, False, attn_type)
588
668
- output[num_prefill_tokens:] = PagedAttention.forward_decode(
589
+ output[num_prefill_query_tokens:] = PagedAttention.forward_decode(
590
decode_query,
591
key_cache,
592
value_cache,
vllm/attention/selector.py
+1
-1
@@ -98,7 +98,6 @@ def get_attn_backend(
98
is_blocksparse: bool = False,
99
) -> Type[AttentionBackend]:
100
"""Selects which attention backend to use and lazily imports it."""
101
-
101
if is_blocksparse:
102
logger.info("Using BlocksparseFlashAttention backend.")
103
from vllm.attention.backends.blocksparse_attn import (
@@ -108,6 +107,7 @@ def get_attn_backend(
107
backend = which_attn_to_use(head_size, dtype, kv_cache_dtype, block_size,
108
is_attention_free)
109
if backend == _Backend.FLASH_ATTN:
110
+ logger.info("Using Flash Attention backend.")
111
from vllm.attention.backends.flash_attn import ( # noqa: F401
112
FlashAttentionBackend)
113
return FlashAttentionBackend
vllm/model_executor/models/bart.py
-2
@@ -624,8 +624,6 @@ class BartEncoder(nn.Module):
624
Decoder output torch.Tensor
625
"""
626
# retrieve input_ids and inputs_embeds
627
-
628
- input_ids = input_ids.view(-1, input_ids.shape[-1])
627
inputs_embeds = self.embed_tokens(input_ids)
628
629
embed_pos = self.embed_positions(
vllm/utils.py
+2
-2
@@ -80,8 +80,8 @@ STR_NOT_IMPL_ENC_DEC_SPEC_DEC = ("Speculative decoding is not "
80
"currently supported with encoder/"
81
"decoder models.")
82
83
-STR_NOT_IMPL_ENC_DEC_BACKEND = ("XFormers is the only backend "
84
- "currently supported with encoder/"
83
+STR_NOT_IMPL_ENC_DEC_BACKEND = ("XFormers and Flash-Attention are the only "
84
+ "backends currently supported with encoder/"
85
"decoder models.")
86
87
STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER = ("Prompt adapters are not "
vllm/worker/enc_dec_model_runner.py
+25
-10
@@ -19,6 +19,7 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry
19
from vllm.logger import init_logger
20
from vllm.model_executor import SamplingMetadata
21
from vllm.model_executor.layers.sampler import SamplerOutput
22
+from vllm.model_executor.model_loader.utils import get_architecture_class_name
23
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
24
MultiModalRegistry)
25
from vllm.sampling_params import SamplingParams
@@ -36,6 +37,11 @@ from vllm.worker.utils import assert_enc_dec_mr_supported_scenario
37
38
logger = init_logger(__name__)
39
40
+# The Mllama model has PagedAttention specific logic because of which it
41
+# can only be run with the XFORMERS backend
42
+# TODO Make Mllama model work with Flash Attention backend.
43
+_XFORMERS_ONLY_ENCODER_DECODER_ARCHS = ["MllamaForConditionalGeneration"]
44
+
45
46
@dataclasses.dataclass(frozen=True)
47
class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata):
@@ -101,9 +107,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
107
models) but these arguments are present here for compatibility with
108
the base-class constructor.
109
'''
104
-
105
- self._maybe_force_supported_attention_backend()
106
-
110
+ self._maybe_force_supported_attention_backend(model_config)
111
super().__init__(
112
model_config,
113
parallel_config,
@@ -119,7 +123,12 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
123
# Crash for unsupported encoder/scenarios
124
assert_enc_dec_mr_supported_scenario(self)
125
122
- def _maybe_force_supported_attention_backend(self):
126
+ def _is_xformers_only_encoder_decoder_model(self,
127
+ model: ModelConfig) -> bool:
128
+ return get_architecture_class_name(
129
+ model) in _XFORMERS_ONLY_ENCODER_DECODER_ARCHS
130
+
131
+ def _maybe_force_supported_attention_backend(self, model: ModelConfig):
132
'''
133
Force vLLM to use the XFormers attention backend,
134
which is currently the only supported option.
@@ -135,22 +144,26 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
144
is_forced_by_global = maybe_global_forced_backend is not None
145
is_forced_by_env_var = maybe_env_var_forced_backend is not None
146
138
- if not (is_forced_by_global or is_forced_by_env_var):
147
+ if not (is_forced_by_global or is_forced_by_env_var) \
148
+ and self._is_xformers_only_encoder_decoder_model(model):
149
# The user has not already specified an attention backend
150
# override
141
- logger.info("EncoderDecoderModelRunner requires "
142
- "XFormers backend; overriding backend "
143
- "auto-selection and forcing XFormers.")
151
+ logger.info(
152
+ "Encoder-Decoder Model Architecture %s requires XFormers "
153
+ "backend; overriding backend auto-selection and "
154
+ "forcing XFormers.", get_architecture_class_name(model))
155
global_force_attn_backend(_Backend.XFORMERS)
156
elif is_forced_by_global:
157
# Backend override enforced by global variable takes
158
# precedence over vLLM backend environment variable.
148
- if maybe_global_forced_backend != _Backend.XFORMERS:
159
+ if maybe_global_forced_backend not in\
160
+ [_Backend.XFORMERS, _Backend.FLASH_ATTN]:
161
raise_backend_err()
162
elif is_forced_by_env_var:
163
# Backend override enforced by vLLM backend
164
# environment variable
153
- if maybe_env_var_forced_backend != _Backend.XFORMERS:
165
+ if maybe_env_var_forced_backend not in\
166
+ [_Backend.XFORMERS, _Backend.FLASH_ATTN]:
167
raise_backend_err()
168
169
def _list_to_int32_tensor(
@@ -532,6 +545,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
545
attn_metadata.encoder_seq_lens,
546
attn_metadata.encoder_seq_lens_tensor,
547
attn_metadata.max_encoder_seq_len,
548
+ attn_metadata.encoder_seq_start_loc,
549
attn_metadata.cross_slot_mapping,
550
attn_metadata.cross_block_tables,
551
) = (
@@ -539,6 +553,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
553
encoder_seq_lens,
554
encoder_seq_lens_tensor,
555
max_encoder_seq_len,
556
+ encoder_seq_start_loc,
557
cross_slot_mapping_tensor,
558
cross_block_tables,
559
)