@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a78dd3303e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Encoder Decoder] Add flash_attn kernel support for encoder-decoder models (#9559)

sroy745 committed Nov 1, 2024 at 23:22 UTC a78dd3303efac284afc6785eddba5f175285863b
11 files changed +716 -317
tests/encoder_decoder/test_e2e_correctness.py
+51 -37
@@ -7,12 +7,18 @@ from typing import List, Optional, Tuple
7 import pytest
8 from transformers import AutoModelForSeq2SeqLM
9
10 +from vllm.attention.selector import (_Backend,
11 + global_force_attn_backend_context_manager)
12 from vllm.platforms import current_platform
13 from vllm.sequence import SampleLogprobs
14
15 from ..conftest import DecoderPromptType
16 from ..models.utils import check_logprobs_close
17
18 +LIST_ENC_DEC_SUPPORTED_BACKENDS = [
19 + _Backend.XFORMERS, _Backend.FLASH_ATTN, None
20 +]
21 +
22
23 def vllm_to_hf_output(
24 vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
@@ -29,7 +35,8 @@ def vllm_to_hf_output(
35
36
37 @pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
32 -@pytest.mark.parametrize("dtype", ["bfloat16"])
38 +@pytest.mark.parametrize("dtype", ["float"])
39 +@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
40 @pytest.mark.parametrize("max_tokens", [128])
41 @pytest.mark.parametrize("num_logprobs", [5])
42 @pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
@@ -48,6 +55,7 @@ def test_encoder_decoder_e2e(
55 num_logprobs: int,
56 decoder_prompt_type: DecoderPromptType,
57 enforce_eager: bool,
58 + attn_backend: _Backend,
59 ) -> None:
60 '''
61 End-to-End (E2E) test for the encoder-decoder framework.
@@ -56,43 +64,49 @@ def test_encoder_decoder_e2e(
64 implementations to ensure that both implementations produce consistent
65 and correct results.
66 '''
59 - test_case_prompts = example_encoder_decoder_prompts[decoder_prompt_type]
67 + with global_force_attn_backend_context_manager(attn_backend):
68 + if attn_backend == _Backend.FLASH_ATTN:
69 + # Flash Attention works only with bfloat16 data-type
70 + dtype = 'bfloat16'
71 + test_case_prompts = example_encoder_decoder_prompts[
72 + decoder_prompt_type]
73
61 - # Configuration settings for HF baseline
62 - hf_kwargs = {
63 - "top_k": None,
64 - "num_beams": 1,
65 - "repetition_penalty": 1.0,
66 - "top_p": 1.0,
67 - "length_penalty": 1.0,
68 - "early_stopping": False,
69 - "no_repeat_ngram_size": None,
70 - "min_length": 0
71 - }
74 + # Configuration settings for HF baseline
75 + hf_kwargs = {
76 + "top_k": None,
77 + "num_beams": 1,
78 + "repetition_penalty": 1.0,
79 + "top_p": 1.0,
80 + "length_penalty": 1.0,
81 + "early_stopping": False,
82 + "no_repeat_ngram_size": None,
83 + "min_length": 0
84 + }
85
73 - with hf_runner(model, dtype=dtype,
74 - auto_cls=AutoModelForSeq2SeqLM) as hf_model:
75 - hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
76 - test_case_prompts,
77 - max_tokens,
78 - num_logprobs,
79 - **hf_kwargs,
80 - ))
81 - with vllm_runner(model, dtype=dtype,
82 - enforce_eager=enforce_eager) as vllm_model:
83 - vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
84 - test_case_prompts, max_tokens, num_logprobs)
86 + with hf_runner(model, dtype=dtype,
87 + auto_cls=AutoModelForSeq2SeqLM) as hf_model:
88 + hf_outputs = (
89 + hf_model.generate_encoder_decoder_greedy_logprobs_limit(
90 + test_case_prompts,
91 + max_tokens,
92 + num_logprobs,
93 + **hf_kwargs,
94 + ))
95 + with vllm_runner(model, dtype=dtype,
96 + enforce_eager=enforce_eager) as vllm_model:
97 + vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
98 + test_case_prompts, max_tokens, num_logprobs)
99
86 - hf_skip_tokens = (1
87 - if decoder_prompt_type == DecoderPromptType.NONE else 0)
100 + hf_skip_tokens = (1 if decoder_prompt_type == DecoderPromptType.NONE
101 + else 0)
102
89 - check_logprobs_close(
90 - outputs_0_lst=hf_outputs,
91 - outputs_1_lst=[
92 - vllm_to_hf_output(vllm_output, decoder_prompt_type)
93 - for vllm_output in vllm_outputs
94 - ],
95 - name_0="hf",
96 - name_1="vllm",
97 - num_outputs_0_skip_tokens=hf_skip_tokens,
98 - )
103 + check_logprobs_close(
104 + outputs_0_lst=hf_outputs,
105 + outputs_1_lst=[
106 + vllm_to_hf_output(vllm_output, decoder_prompt_type)
107 + for vllm_output in vllm_outputs
108 + ],
109 + name_0="hf",
110 + name_1="vllm",
111 + num_outputs_0_skip_tokens=hf_skip_tokens,
112 + )
tests/kernels/test_encoder_decoder_attn.py
+115 -41
@@ -16,13 +16,13 @@ from tests.kernels.utils import *
16 from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
17 AttentionType)
18 from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19 -from vllm.attention.selector import (_Backend,
19 +from vllm.attention.selector import (_Backend, get_attn_backend,
20 global_force_attn_backend_context_manager)
21 +from vllm.forward_context import set_forward_context
22 from vllm.platforms import current_platform
23
24 # List of support backends for encoder/decoder models
24 -LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS]
25 -
25 +LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
26 HEAD_SIZES = [64, 256]
27
28 NUM_HEADS = [1, 16]
@@ -145,7 +145,8 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
145 test_pt.num_heads,
146 test_pt.head_size,
147 test_pt.block_size,
148 - device=CUDA_DEVICE)
148 + device=CUDA_DEVICE,
149 + backend=test_pt.backend_name)
150 return TestResources(scale, attn_backend, attn, kv_cache)
151
152
@@ -592,6 +593,7 @@ def _run_encoder_attention_test(
593 attn: Attention,
594 encoder_test_params: PhaseTestParameters,
595 attn_metadata: AttentionMetadata,
596 + test_pt: TestPoint,
597 ) -> torch.Tensor:
598 '''
599 Run encoder attention.
@@ -610,6 +612,8 @@ def _run_encoder_attention_test(
612 (number_of_tokens x num_heads x head_size)
613 query/key/value fields
614 * attn_metadata: attention metadata for encoder/decoder-self attention
615 + * test_pt: The TestPoint object containing test details like number of
616 + model heads, head size, name of the backend being used etc.
617
618 Returns:
619 * Attention.forward() applied to packed {query,key,value} and
@@ -619,20 +623,31 @@ def _run_encoder_attention_test(
623 attn_type = AttentionType.ENCODER
624 packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
625 assert packed_qkv is not None
622 - return attn.forward(packed_qkv.query,
623 - packed_qkv.key,
624 - packed_qkv.value,
625 - torch.tensor([],
626 - dtype=torch.float32,
627 - device=packed_qkv.query.device),
628 - attn_metadata,
629 - attn_type=attn_type)
626 + with set_forward_context(attn_metadata):
627 + # In the test setup the shape of the query is
628 + # [batch_size, seq_len, num_heads, head_size]. However
629 + # the attention backend expect the shape to be
630 + # [num_tokens, hidden_size]. Hence reshape the query before
631 + # invoking the forward method.
632 + # TODO - Update the way we construct the query so that it
633 + # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
634 + reshaped_query = packed_qkv.query.view(
635 + -1, test_pt.num_heads * test_pt.head_size)
636 + return attn.forward(reshaped_query,
637 + packed_qkv.key,
638 + packed_qkv.value,
639 + torch.tensor([],
640 + dtype=torch.float32,
641 + device=packed_qkv.query.device),
642 + attn_metadata,
643 + attn_type=attn_type)
644
645
646 def _run_decoder_self_attention_test(
647 test_rsrcs: TestResources,
648 decoder_test_params: PhaseTestParameters,
649 attn_metadata: AttentionMetadata,
650 + test_pt: TestPoint,
651 ) -> torch.Tensor:
652 '''
653 Run decoder self-attention test.
@@ -650,6 +665,8 @@ def _run_decoder_self_attention_test(
665 query/key/value fields
666 * attn_metadata: attention metadata for decoder-self attention
667 (contains KV cache memory-mapping)
668 + * test_pt: The TestPoint object containing test details like number of
669 + model heads, head size, name of the backend being used etc.
670
671 Returns:
672 * Attention.forward() applied to packed_{query,key,value}, kv_cache
@@ -660,12 +677,22 @@ def _run_decoder_self_attention_test(
677 kv_cache = test_rsrcs.kv_cache
678 packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
679 assert packed_qkv is not None
663 - return attn.forward(packed_qkv.query,
664 - packed_qkv.key,
665 - packed_qkv.value,
666 - kv_cache,
667 - attn_metadata,
668 - attn_type=attn_type)
680 + with set_forward_context(attn_metadata):
681 + # In the test setup the shape of the query is
682 + # [batch_size, seq_len, num_heads, head_size]. However
683 + # the attention backend expect the shape to be
684 + # [num_tokens, hidden_size]. Hence reshape the query before
685 + # invoking the forward method.
686 + # TODO - Update the way we construct the query so that it
687 + # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
688 + reshaped_query = packed_qkv.query.view(
689 + -1, test_pt.num_heads * test_pt.head_size)
690 + return attn.forward(reshaped_query,
691 + packed_qkv.key,
692 + packed_qkv.value,
693 + kv_cache,
694 + attn_metadata,
695 + attn_type=attn_type)
696
697
698 def _run_encoder_decoder_cross_attention_test(
@@ -673,6 +700,7 @@ def _run_encoder_decoder_cross_attention_test(
700 decoder_test_params: PhaseTestParameters,
701 cross_test_params: Optional[PhaseTestParameters],
702 attn_metadata: AttentionMetadata,
703 + test_pt: TestPoint,
704 ) -> torch.Tensor:
705 '''
706 Run encoder/decoder cross-attention test.
@@ -701,6 +729,8 @@ def _run_encoder_decoder_cross_attention_test(
729 (number_of_tokens x num_heads x head_size)
730 key/value fields
731 * attn_metadata: attention metadata for encoder/decoder-self attention
732 + * test_pt: The TestPoint object containing test details like number of
733 + model heads, head size, name of the backend being used etc.
734
735 Returns:
736 * Attention.forward() applied to packed_{query,key,value}, kv_cache
@@ -718,12 +748,37 @@ def _run_encoder_decoder_cross_attention_test(
748 cross_pckd_qkv = cross_test_params.packed_qkvo.packed_qkv
749 key = (None if cross_pckd_qkv is None else cross_pckd_qkv.key)
750 value = (None if cross_pckd_qkv is None else cross_pckd_qkv.value)
721 - return attn.forward(decoder_test_params.packed_qkvo.packed_qkv.query,
722 - key,
723 - value,
724 - kv_cache,
725 - attn_metadata,
726 - attn_type=attn_type)
751 + with set_forward_context(attn_metadata):
752 + # In the test setup the shape of the query is
753 + # [batch_size, seq_len, num_heads, head_size]. However
754 + # the attention backend expect the shape to be
755 + # [num_tokens, hidden_size]. Hence reshape the query before
756 + # invoking the forward method.
757 + # TODO - Update the way we construct the query so that it
758 + # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
759 + reshaped_query = decoder_test_params.packed_qkvo.packed_qkv.query.view(
760 + -1, test_pt.num_heads * test_pt.head_size)
761 + return attn.forward(reshaped_query,
762 + key,
763 + value,
764 + kv_cache,
765 + attn_metadata,
766 + attn_type=attn_type)
767 +
768 +
769 +@pytest.fixture(autouse=True)
770 +def set_reset_environment(attn_backend):
771 + # Set the default torch datatype to bfloat16 to enable
772 + # testing of the Flash Attention backend. Also clear the
773 + # cached value of the backend.
774 + default_dtype = torch.get_default_dtype()
775 + if attn_backend.name == 'FLASH_ATTN':
776 + torch.set_default_dtype(torch.bfloat16)
777 + get_attn_backend.cache_clear()
778 + yield
779 + # Reset the torch datatype to what it was before the test
780 + # so as not to impact the remaining tests.
781 + torch.set_default_dtype(default_dtype)
782
783
784 @pytest.mark.skipif(current_platform.is_rocm(),
@@ -773,10 +828,8 @@ def test_encoder_only(
828 * max_dec_seq_len: max length of decoder input sequences
829 * max_enc_seq_len: max length of encoder input sequences
830 '''
776 -
831 # Force Attention wrapper backend
832 with global_force_attn_backend_context_manager(attn_backend):
779 -
833 # Note: KV cache size of 4096 is arbitrary & chosen intentionally
834 # to be more than necessary, since exceeding the kv cache size
835 # is not part of this test
@@ -807,10 +860,14 @@ def test_encoder_only(
860 # PREFILL: encoder attention
861
862 enc_pckd_act_out: torch.Tensor = (_run_encoder_attention_test(
810 - test_rsrcs.attn, enc_test_params, prephase_attn_metadata))
863 + test_rsrcs.attn,
864 + enc_test_params,
865 + prephase_attn_metadata,
866 + test_pt=test_pt))
867
868 # - Is encoder attention result correct?
813 - assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
869 + assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
870 + attn_backend.name)
871
872
873 @pytest.mark.skipif(current_platform.is_rocm(),
@@ -892,10 +949,8 @@ def test_e2e_enc_dec_attn(
949 * max_dec_seq_len: max length of decoder input sequences
950 * max_enc_seq_len: max length of encoder input sequences
951 '''
895 -
952 # Force Attention wrapper backend
953 with global_force_attn_backend_context_manager(attn_backend):
898 -
954 # Note: KV cache size of 4096 is arbitrary & chosen intentionally
955 # to be more than necessary, since exceeding the kv cache size
956 # is not part of this test
@@ -955,29 +1010,39 @@ def test_e2e_enc_dec_attn(
1010
1011 enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
1012 enc_test_params,
958 - prephase_attn_metadata)
1013 + prephase_attn_metadata,
1014 + test_pt=test_pt)
1015
1016 # - Is encoder attention result correct?
961 - assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
1017 + assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
1018 + attn_backend.name)
1019
1020 # PREFILL: decoder self-attention test
1021
1022 prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
966 - test_rsrcs, prephase_dec_test_params, prephase_attn_metadata)
1023 + test_rsrcs,
1024 + prephase_dec_test_params,
1025 + prephase_attn_metadata,
1026 + test_pt=test_pt)
1027
1028 # - Is prefill decoder self-attention correct?
1029 assert_actual_matches_ideal(prephase_dec_test_params,
970 - prephase_dec_pckd_act_out)
1030 + prephase_dec_pckd_act_out,
1031 + attn_backend.name)
1032
1033 # PREFILL: encoder/decoder cross-attention test
1034
1035 prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
975 - test_rsrcs, prephase_dec_test_params, prephase_cross_test_params,
976 - prephase_attn_metadata)
1036 + test_rsrcs,
1037 + prephase_dec_test_params,
1038 + prephase_cross_test_params,
1039 + prephase_attn_metadata,
1040 + test_pt=test_pt)
1041
1042 # - Is prefill encoder/decoder cross-attention correct?
1043 assert_actual_matches_ideal(prephase_cross_test_params,
980 - prephase_cross_pckd_act_out)
1044 + prephase_cross_pckd_act_out,
1045 + attn_backend.name)
1046
1047 # DECODE: build decode-phase attention metadata
1048
@@ -993,17 +1058,26 @@ def test_e2e_enc_dec_attn(
1058 # DECODE: decoder self-attention test
1059
1060 decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
996 - test_rsrcs, decphase_dec_test_params, decphase_attn_metadata)
1061 + test_rsrcs,
1062 + decphase_dec_test_params,
1063 + decphase_attn_metadata,
1064 + test_pt=test_pt)
1065
1066 # - Is decode-phase decoder self-attention correct?
1067 assert_actual_matches_ideal(decphase_dec_test_params,
1000 - decphase_dec_pckd_act_out)
1068 + decphase_dec_pckd_act_out,
1069 + attn_backend.name)
1070
1071 # DECODE: encoder/decoder cross-attention test
1072
1073 decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
1005 - test_rsrcs, decphase_dec_test_params, None, decphase_attn_metadata)
1074 + test_rsrcs,
1075 + decphase_dec_test_params,
1076 + None,
1077 + decphase_attn_metadata,
1078 + test_pt=test_pt)
1079
1080 # - Is decode-phase encoder/decoder cross-attention correct?
1081 assert_actual_matches_ideal(decphase_cross_test_params,
1009 - decphase_cross_pckd_act_out)
1082 + decphase_cross_pckd_act_out,
1083 + attn_backend.name)
tests/kernels/utils.py
+74 -16
@@ -13,8 +13,8 @@ from torch._prims_common import TensorLikeType
13
14 from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
15 from vllm.model_executor.layers.activation import SiluAndMul
16 -from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
17 - make_tensor_with_pad)
16 +from vllm.utils import (STR_BACKEND_ENV_VAR, STR_FLASH_ATTN_VAL,
17 + STR_XFORMERS_ATTN_VAL, make_tensor_with_pad)
18
19 # For now, disable "test_aot_dispatch_dynamic" since there are some
20 # bugs related to this test in PyTorch 2.4.
@@ -525,17 +525,22 @@ def make_backend(backend_name: str) -> AttentionBackend:
525 if backend_name == STR_XFORMERS_ATTN_VAL:
526 # NOTE: xFormers backend cannot be imported for CPU and AMD GPUs.
527 from vllm.attention.backends.xformers import XFormersBackend
528 -
528 return XFormersBackend()
529 + elif backend_name == STR_FLASH_ATTN_VAL:
530 + from vllm.attention.backends.flash_attn import FlashAttentionBackend
531 + return FlashAttentionBackend()
532 +
533 raise AssertionError(
534 f"Unrecognized backend_name {backend_name} for unit test")
535
536
537 def _make_metadata_tensors(
535 - seq_lens: Optional[List[int]], context_lens: Optional[List[int]],
536 - encoder_seq_lens: Optional[List[int]], device: Union[torch.device, str]
537 -) -> Tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[List[int]],
538 - torch.Tensor, Optional[int]]:
538 + seq_lens: Optional[List[int]],
539 + context_lens: Optional[List[int]],
540 + encoder_seq_lens: Optional[List[int]],
541 + device: Union[torch.device, str],
542 +) -> Tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[torch.Tensor],
543 + torch.Tensor, torch.Tensor, Optional[int]]:
544 '''
545 Build scalar & tensor values required to build attention metadata structure.
546
@@ -553,6 +558,8 @@ def _make_metadata_tensors(
558 * max_context_len: max(context_lens)
559 * max_seq_len: max(seq_lens)
560 * seq_start_loc: start idx of each sequence
561 + * encoder_seq_lens_tensor: encoder seq_lens list, as tensor
562 + * encoder_seq_start_loc: start idx of each encoder sequence
563 * max_encoder_seq_len: encoder seq_lens list, as tensor
564 '''
565 seq_lens_tensor = maybe_make_int_tensor(seq_lens, device)
@@ -566,8 +573,26 @@ def _make_metadata_tensors(
573
574 seq_start_loc = None
575
576 + if seq_lens_tensor is not None:
577 + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
578 + dtype=torch.int32,
579 + device=seq_lens_tensor.device)
580 + torch.cumsum(seq_lens_tensor,
581 + dim=0,
582 + dtype=seq_start_loc.dtype,
583 + out=seq_start_loc[1:])
584 +
585 + encoder_seq_start_loc = torch.zeros(encoder_seq_lens_tensor.shape[0] + 1,
586 + dtype=torch.int32,
587 + device=encoder_seq_lens_tensor.device)
588 + torch.cumsum(encoder_seq_lens_tensor,
589 + dim=0,
590 + dtype=encoder_seq_start_loc.dtype,
591 + out=encoder_seq_start_loc[1:])
592 +
593 return (seq_lens_tensor, context_lens_tensor, max_context_len, max_seq_len,
570 - seq_start_loc, encoder_seq_lens_tensor, max_encoder_seq_len)
594 + seq_start_loc, encoder_seq_lens_tensor, encoder_seq_start_loc,
595 + max_encoder_seq_len)
596
597
598 def make_kv_cache(num_blocks: int,
@@ -575,6 +600,7 @@ def make_kv_cache(num_blocks: int,
600 head_size: int,
601 block_size: int,
602 device: Union[torch.device, str],
603 + backend: str,
604 default_val: float = 0.0) -> torch.Tensor:
605 '''
606 Create a fake KV cache.
@@ -591,10 +617,20 @@ def make_kv_cache(num_blocks: int,
617 Returns:
618
619 * kv_cache: 2 x num_blocks x (block_size * num_heads * head_size)
620 + * for backend 'XFORMERS'
621 + * kv_cache: 2 x num_blocks x block_size x num_heads x head_size
622 + * for backend 'FLASH_ATTN'
623 '''
595 -
596 - kv_cache = torch.rand(
597 - (2, num_blocks, block_size * num_heads * head_size)).to(device)
624 + if backend == 'XFORMERS':
625 + kv_cache = torch.rand(
626 + (2, num_blocks, block_size * num_heads * head_size)).to(device)
627 + elif backend == 'FLASH_ATTN':
628 + kv_cache = torch.rand(
629 + (2, num_blocks, block_size, num_heads, head_size)).to(device)
630 + else:
631 + raise ValueError(
632 + f"Unknown backend value: '{backend}'. Expected 'XFORMERS' or "
633 + f"'FLASH_ATTN'.")
634 if default_val is not None:
635 kv_cache[:, :, :] = default_val
636 return kv_cache
@@ -858,8 +894,9 @@ def make_test_metadata(
894 context_lens_tensor,
895 _,
896 _,
861 - _,
897 + seq_start_loc,
898 encoder_seq_lens_tensor,
899 + encoder_seq_start_loc,
900 max_encoder_seq_len,
901 ) = _make_metadata_tensors(seq_lens,
902 context_lens,
@@ -874,6 +911,7 @@ def make_test_metadata(
911 num_decode_tokens=num_decode_tokens,
912 seq_lens=seq_lens,
913 seq_lens_tensor=seq_lens_tensor,
914 + seq_start_loc=seq_start_loc,
915 max_prefill_seq_len=None if seq_lens is None else max(seq_lens),
916 max_decode_seq_len=0,
917 context_lens_tensor=context_lens_tensor,
@@ -882,6 +920,7 @@ def make_test_metadata(
920 num_encoder_tokens=num_encoder_tokens,
921 encoder_seq_lens=encoder_seq_lens,
922 encoder_seq_lens_tensor=encoder_seq_lens_tensor,
923 + encoder_seq_start_loc=encoder_seq_start_loc,
924 max_encoder_seq_len=max_encoder_seq_len,
925 cross_slot_mapping=(None if cross_kv_mmap is None else
926 cross_kv_mmap.slot_mapping),
@@ -904,8 +943,9 @@ def make_test_metadata(
943 context_lens_tensor,
944 _,
945 _,
907 - _,
946 + seq_start_loc,
947 encoder_seq_lens_tensor,
948 + encoder_seq_start_loc,
949 max_encoder_seq_len,
950 ) = _make_metadata_tensors(seq_lens,
951 context_lens,
@@ -920,14 +960,17 @@ def make_test_metadata(
960 num_decode_tokens=num_decode_tokens,
961 seq_lens=seq_lens,
962 seq_lens_tensor=seq_lens_tensor,
963 + seq_start_loc=seq_start_loc,
964 max_prefill_seq_len=0,
965 max_decode_seq_len=max(seq_lens),
966 + max_decode_query_len=1,
967 context_lens_tensor=context_lens_tensor,
968 block_tables=kv_mmap.block_tables,
969 use_cuda_graph=False,
970 num_encoder_tokens=num_encoder_tokens,
971 encoder_seq_lens=encoder_seq_lens,
972 encoder_seq_lens_tensor=encoder_seq_lens_tensor,
973 + encoder_seq_start_loc=encoder_seq_start_loc,
974 max_encoder_seq_len=max_encoder_seq_len,
975 cross_slot_mapping=(None if cross_kv_mmap is None else
976 cross_kv_mmap.slot_mapping),
@@ -936,7 +979,8 @@ def make_test_metadata(
979
980
981 def assert_actual_matches_ideal(test_params: PhaseTestParameters,
939 - output_under_test: torch.Tensor) -> None:
982 + output_under_test: torch.Tensor,
983 + backend: str) -> None:
984 '''
985 Assert that observed output matches the ideal output
986 contained in the test parameters data structure.
@@ -947,8 +991,22 @@ def assert_actual_matches_ideal(test_params: PhaseTestParameters,
991 * output_under_test: actually observed output value
992 '''
993 ideal_output = test_params.packed_qkvo.ideal_output
950 - torch.testing.assert_close(ideal_output,
951 - output_under_test.view_as(ideal_output))
994 + if backend == 'XFORMERS':
995 + torch.testing.assert_close(ideal_output,
996 + output_under_test.view_as(ideal_output))
997 +
998 + elif backend == 'FLASH_ATTN':
999 + # For FlashAttention override the accuracy thresholds to non default
1000 + # values since we notice a higher difference between the ideal and
1001 + # actual output.
1002 + torch.testing.assert_close(ideal_output,
1003 + output_under_test.view_as(ideal_output),
1004 + atol=0.01,
1005 + rtol=0.016)
1006 + else:
1007 + raise ValueError(
1008 + f"Unknown backend value: '{backend}'. Expected 'XFORMERS' or "
1009 + f"'FLASH_ATTN'.")
1010
1011
1012 # Copied/modified from torch._refs.__init__.py
tests/models/encoder_decoder/vision_language/test_florence2.py
+1 -1
@@ -85,7 +85,7 @@ def run_test(
85
86
87 @pytest.mark.parametrize("model", MODELS)
88 -@pytest.mark.parametrize("dtype", ["float"])
88 +@pytest.mark.parametrize("dtype", ["float", "bfloat16"])
89 @pytest.mark.parametrize("max_tokens", [64])
90 @pytest.mark.parametrize("num_logprobs", [5])
91 def test_models(hf_runner, vllm_runner, model, dtype, max_tokens,
vllm/attention/backends/flash_attn.py
+278 -86
@@ -10,10 +10,11 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 AttentionMetadata,
11 AttentionMetadataBuilder,
12 AttentionType)
13 -from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
14 - compute_slot_mapping,
15 - compute_slot_mapping_start_idx,
16 - is_block_tables_empty)
13 +from vllm.attention.backends.utils import (
14 + PAD_SLOT_ID, CommonAttentionState, compute_slot_mapping,
15 + compute_slot_mapping_start_idx, get_num_prefill_decode_query_kv_tokens,
16 + get_seq_len_block_table_args, is_all_cross_attn_metadata_set,
17 + is_all_encoder_attn_metadata_set, is_block_tables_empty)
18 from vllm.forward_context import get_forward_context
19 from vllm.multimodal import MultiModalPlaceholderMap
20 from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
@@ -73,7 +74,6 @@ class FlashAttentionBackend(AttentionBackend):
74 src_key_cache = src_kv_cache[0]
75 dst_key_cache = dst_kv_cache[0]
76 ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
76 -
77 src_value_cache = src_kv_cache[1]
78 dst_value_cache = dst_kv_cache[1]
79 ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
@@ -85,6 +85,7 @@ class FlashAttentionBackend(AttentionBackend):
85 ) -> None:
86 key_caches = [kv_cache[0] for kv_cache in kv_caches]
87 value_caches = [kv_cache[1] for kv_cache in kv_caches]
88 +
89 ops.copy_blocks(key_caches, value_caches, src_to_dists)
90
91
@@ -111,26 +112,12 @@ class FlashAttentionMetadata(AttentionMetadata):
112 # |-------------------- seq_len ---------------------|
113 # |-- query_len ---|
114
114 - # Maximum query length in the batch.
115 - max_query_len: Optional[int]
116 -
117 - # Max number of query tokens among request in the batch.
118 - max_decode_query_len: Optional[int]
119 -
115 # Maximum sequence length among prefill batch. 0 if there are decoding
116 # requests only.
117 max_prefill_seq_len: int
118 # Maximum sequence length among decode batch. 0 if there are prefill
119 # requests only.
120 max_decode_seq_len: int
126 - # (batch_size + 1,). The cumulative subquery lengths of the sequences in
127 - # the batch, used to index into subquery. E.g., if the subquery length
128 - # is [4, 6], it is [0, 4, 10].
129 - query_start_loc: Optional[torch.Tensor]
130 - # (batch_size + 1,). The cumulative sequence lengths of the sequences in
131 - # the batch, used to index into sequence. E.g., if the sequence length is
132 - # [4, 6], it is [0, 4, 10].
133 - seq_start_loc: Optional[torch.Tensor]
121 # (batch_size,) A tensor of context lengths (tokens that are computed
122 # so far).
123 context_lens_tensor: Optional[torch.Tensor]
@@ -146,11 +133,62 @@ class FlashAttentionMetadata(AttentionMetadata):
133 # Whether or not if cuda graph is enabled.
134 # Cuda-graph is currently enabled for decoding only.
135 # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
136 +
137 use_cuda_graph: bool
138
139 + # Maximum query length in the batch.
140 + max_query_len: Optional[int] = None
141 +
142 + # Max number of query tokens among request in the batch.
143 + max_decode_query_len: Optional[int] = None
144 +
145 + # (batch_size + 1,). The cumulative subquery lengths of the sequences in
146 + # the batch, used to index into subquery. E.g., if the subquery length
147 + # is [4, 6], it is [0, 4, 10].
148 + query_start_loc: Optional[torch.Tensor] = None
149 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
150 + # the batch, used to index into sequence. E.g., if the sequence length is
151 + # [4, 6], it is [0, 4, 10].
152 + seq_start_loc: Optional[torch.Tensor] = None
153 +
154 _cached_prefill_metadata: Optional["FlashAttentionMetadata"] = None
155 _cached_decode_metadata: Optional["FlashAttentionMetadata"] = None
156
157 + # Begin encoder attn & enc/dec cross-attn fields...
158 +
159 + # Encoder sequence lengths representation
160 + encoder_seq_lens: Optional[List[int]] = None
161 + encoder_seq_lens_tensor: Optional[torch.Tensor] = None
162 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
163 + # the batch, used to index into sequence. E.g., if the sequence length is
164 + # [4, 6], it is [0, 4, 10].
165 + encoder_seq_start_loc: Optional[torch.Tensor] = None
166 + # Maximum sequence length among encoder sequences
167 + max_encoder_seq_len: Optional[int] = None
168 + # Number of tokens input to encoder
169 + num_encoder_tokens: Optional[int] = None
170 +
171 + # Cross-attention memory-mapping data structures: slot mapping
172 + # and block tables
173 + cross_slot_mapping: Optional[torch.Tensor] = None
174 + cross_block_tables: Optional[torch.Tensor] = None
175 +
176 + @property
177 + def is_all_encoder_attn_metadata_set(self):
178 + '''
179 + All attention metadata required for encoder attention is set.
180 + '''
181 + return is_all_encoder_attn_metadata_set(self)
182 +
183 + @property
184 + def is_all_cross_attn_metadata_set(self):
185 + '''
186 + All attention metadata required for enc/dec cross-attention is set.
187 +
188 + Superset of encoder attention required metadata.
189 + '''
190 + return is_all_cross_attn_metadata_set(self)
191 +
192 @property
193 def prefill_metadata(self) -> Optional["FlashAttentionMetadata"]:
194 if self.num_prefills == 0:
@@ -159,32 +197,52 @@ class FlashAttentionMetadata(AttentionMetadata):
197 if self._cached_prefill_metadata is not None:
198 return self._cached_prefill_metadata
199
162 - assert self.seq_lens is not None
163 - assert self.seq_lens_tensor is not None
164 - assert self.query_start_loc is not None
165 - assert self.context_lens_tensor is not None
166 - assert self.block_tables is not None
167 - assert self.seq_start_loc is not None
200 + assert ((self.seq_lens is not None)
201 + or (self.encoder_seq_lens is not None))
202 + assert ((self.seq_lens_tensor is not None)
203 + or (self.encoder_seq_lens_tensor is not None))
204 +
205 + # Compute some attn_metadata fields which default to None
206 + query_start_loc = (None if self.query_start_loc is None else
207 + self.query_start_loc[:self.num_prefills + 1])
208 + slot_mapping = (None if self.slot_mapping is None else
209 + self.slot_mapping[:self.num_prefill_tokens])
210 + seq_lens = (None if self.seq_lens is None else
211 + self.seq_lens[:self.num_prefills])
212 + seq_lens_tensor = (None if self.seq_lens_tensor is None else
213 + self.seq_lens_tensor[:self.num_prefills])
214 + seq_start_loc = (None if self.seq_start_loc is None else
215 + self.seq_start_loc[:self.num_prefills + 1])
216 + context_lens_tensor = (None if self.context_lens_tensor is None else
217 + self.context_lens_tensor[:self.num_prefills])
218 + block_tables = (None if self.block_tables is None else
219 + self.block_tables[:self.num_prefills])
220
221 self._cached_prefill_metadata = FlashAttentionMetadata(
222 num_prefills=self.num_prefills,
223 num_prefill_tokens=self.num_prefill_tokens,
224 num_decode_tokens=0,
173 - slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
225 + slot_mapping=slot_mapping,
226 multi_modal_placeholder_index_maps=self.
227 multi_modal_placeholder_index_maps,
176 - seq_lens=self.seq_lens[:self.num_prefills],
177 - seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
228 + seq_lens=seq_lens,
229 + seq_lens_tensor=seq_lens_tensor,
230 max_query_len=self.max_query_len,
231 max_prefill_seq_len=self.max_prefill_seq_len,
232 max_decode_query_len=0,
233 max_decode_seq_len=0,
182 - query_start_loc=self.query_start_loc[:self.num_prefills + 1],
183 - seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
184 - context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
185 - block_tables=self.block_tables[:self.num_prefills],
234 + query_start_loc=query_start_loc,
235 + seq_start_loc=seq_start_loc,
236 + context_lens_tensor=context_lens_tensor,
237 + block_tables=block_tables,
238 use_cuda_graph=False,
187 - )
239 + # Begin encoder & cross attn fields below...
240 + encoder_seq_lens=self.encoder_seq_lens,
241 + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
242 + encoder_seq_start_loc=self.encoder_seq_start_loc,
243 + max_encoder_seq_len=self.max_encoder_seq_len,
244 + cross_slot_mapping=self.cross_slot_mapping,
245 + cross_block_tables=self.cross_block_tables)
246 return self._cached_prefill_metadata
247
248 @property
@@ -194,17 +252,25 @@ class FlashAttentionMetadata(AttentionMetadata):
252
253 if self._cached_decode_metadata is not None:
254 return self._cached_decode_metadata
197 - assert self.block_tables is not None
198 - assert self.seq_lens_tensor is not None
255 + assert ((self.seq_lens_tensor is not None)
256 + or (self.encoder_seq_lens_tensor is not None))
257 +
258 + # Compute some attn_metadata fields which default to None
259 + slot_mapping = (None if self.slot_mapping is None else
260 + self.slot_mapping[self.num_prefill_tokens:])
261 + seq_lens_tensor = (None if self.seq_lens_tensor is None else
262 + self.seq_lens_tensor[self.num_prefills:])
263 + block_tables = (None if self.block_tables is None else
264 + self.block_tables[self.num_prefills:])
265
266 self._cached_decode_metadata = FlashAttentionMetadata(
267 num_prefills=0,
268 num_prefill_tokens=0,
269 num_decode_tokens=self.num_decode_tokens,
204 - slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
270 + slot_mapping=slot_mapping,
271 multi_modal_placeholder_index_maps=None,
272 seq_lens=None,
207 - seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
273 + seq_lens_tensor=seq_lens_tensor,
274 max_decode_query_len=self.max_decode_query_len,
275 max_query_len=self.max_query_len,
276 max_prefill_seq_len=0,
@@ -214,9 +280,15 @@ class FlashAttentionMetadata(AttentionMetadata):
280 seq_start_loc=self.seq_start_loc[self.num_prefills:]
281 if self.seq_start_loc is not None else None,
282 context_lens_tensor=None,
217 - block_tables=self.block_tables[self.num_prefills:],
283 + block_tables=block_tables,
284 use_cuda_graph=self.use_cuda_graph,
219 - )
285 + # Begin encoder & cross attn fields below...
286 + encoder_seq_lens=self.encoder_seq_lens,
287 + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
288 + encoder_seq_start_loc=self.encoder_seq_start_loc,
289 + max_encoder_seq_len=self.max_encoder_seq_len,
290 + cross_slot_mapping=self.cross_slot_mapping,
291 + cross_block_tables=self.cross_block_tables)
292 return self._cached_decode_metadata
293
294 def advance_step(self,
@@ -586,16 +658,20 @@ class FlashAttentionImpl(AttentionImpl):
658 Returns:
659 shape = [num_tokens, num_heads * head_size]
660 """
589 - if attn_type != AttentionType.DECODER:
590 - raise NotImplementedError("Encoder self-attention and "
591 - "encoder/decoder cross-attention "
592 - "are not implemented for "
593 - "FlashAttentionImpl")
594 -
661 # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
662 assert k_scale == 1.0 and v_scale == 1.0, (
663 "key/v_scale is not supported in FlashAttention.")
664
665 + if (attn_type == AttentionType.ENCODER
666 + and (not attn_metadata.is_all_encoder_attn_metadata_set)):
667 + raise AttributeError("Encoder attention requires setting "
668 + "encoder metadata attributes.")
669 + elif (attn_type == AttentionType.ENCODER_DECODER
670 + and (not attn_metadata.is_all_cross_attn_metadata_set)):
671 + raise AttributeError("Encoder/decoder cross-attention "
672 + "requires setting cross-attention "
673 + "metadata attributes.")
674 +
675 output = torch.ops.vllm.unified_flash_attention(
676 query,
677 key,
@@ -608,6 +684,7 @@ class FlashAttentionImpl(AttentionImpl):
684 k_scale,
685 v_scale,
686 self.scale,
687 + attn_type.value,
688 self.sliding_window,
689 self.alibi_slopes,
690 self.logits_soft_cap,
@@ -616,6 +693,89 @@ class FlashAttentionImpl(AttentionImpl):
693 return output
694
695
696 +def _get_query_key_seq_metadata(
697 + attn_metadata,
698 + is_prompt: bool,
699 + attn_type: AttentionType,
700 +) -> tuple:
701 + """
702 + Returns sequence metadata for key and query based on the specified
703 + attention type and whether input is a prompt.
704 +
705 + This function computes the starting locations and maximum sequence lengths
706 + for key and query sequences for different attention types.
707 +
708 + Args:
709 + attn_metadata: The attention metadata object
710 + is_prompt (bool): A flag indicating if the input is a prompt
711 + attn_type (AttentionType): The type of attention being used.
712 +
713 + Returns:
714 + tuple: A tuple containing four integers:
715 + - Starting location for the query sequence.
716 + - Maximum sequence length for the query sequence.
717 + - Starting location for the key sequence.
718 + - Maximum sequence length for the key sequence.
719 +
720 + Raises:
721 + AttributeError: If an invalid attention type is provided.
722 + """
723 + if attn_type == AttentionType.DECODER:
724 + # Decoder self-attention
725 + # Choose max_seq_len based on whether we are in prompt_run
726 + if is_prompt:
727 + max_seq_len = attn_metadata.max_prefill_seq_len
728 + else:
729 + max_seq_len = attn_metadata.max_decode_seq_len
730 + return (attn_metadata.seq_start_loc, max_seq_len,
731 + attn_metadata.seq_start_loc, max_seq_len)
732 +
733 + elif attn_type == AttentionType.ENCODER_DECODER:
734 + # This is cross attention between the where the key
735 + # is the precomputed encoder attention and query
736 + # is the input sequence.
737 + # Choose query max length based on whether it is prompt
738 + # or not.
739 + if is_prompt:
740 + max_seq_len = attn_metadata.max_prefill_seq_len
741 + else:
742 + max_seq_len = attn_metadata.max_decode_seq_len
743 + return (attn_metadata.seq_start_loc, max_seq_len,
744 + attn_metadata.encoder_seq_start_loc,
745 + attn_metadata.max_encoder_seq_len)
746 + elif attn_type == AttentionType.ENCODER:
747 + # For encoder attention both the query and the key are same i.e the
748 + # encoder sequence.
749 + return (attn_metadata.encoder_seq_start_loc,
750 + attn_metadata.max_encoder_seq_len,
751 + attn_metadata.encoder_seq_start_loc,
752 + attn_metadata.max_encoder_seq_len)
753 + elif attn_type == AttentionType.ENCODER_ONLY:
754 + assert is_prompt, "Should not have decode for encoder only model."
755 + return (attn_metadata.seq_start_loc, attn_metadata.max_prefill_seq_len,
756 + attn_metadata.seq_start_loc, attn_metadata.max_prefill_seq_len)
757 + else:
758 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
759 +
760 +
761 +def _get_causal_option(attn_type: AttentionType) -> bool:
762 + """
763 + Determine whether the given attention type is suitable for causal
764 + attention mechanisms.
765 +
766 + Args:
767 + attn_type (AttentionType): The type of attention being evaluated
768 +
769 + Returns:
770 + bool: Returns `True` if the attention type is suitable for causal
771 + attention (i.e., not encoder, encoder-only, or encoder-decoder),
772 + otherwise returns `False`.
773 + """
774 + return not (attn_type == AttentionType.ENCODER
775 + or attn_type == AttentionType.ENCODER_ONLY
776 + or attn_type == AttentionType.ENCODER_DECODER)
777 +
778 +
779 def unified_flash_attention(
780 query: torch.Tensor,
781 key: torch.Tensor,
@@ -628,60 +788,76 @@ def unified_flash_attention(
788 k_scale: float,
789 v_scale: float,
790 softmax_scale: float,
791 + attn_type_int_val: int,
792 window_size: Optional[List[int]] = None,
793 alibi_slopes: Optional[torch.Tensor] = None,
794 logits_soft_cap: Optional[float] = None,
795 ) -> torch.Tensor:
796
797 + # Convert integer attn_type to enum
798 + try:
799 + attn_type = AttentionType(attn_type_int_val)
800 + except ValueError as err:
801 + raise AttributeError(
802 + f"Invalid attention type {str(attn_type_int_val)}") from err
803 +
804 current_metadata = get_forward_context()
805 assert current_metadata is not None
806 assert isinstance(current_metadata, FlashAttentionMetadata)
807 attn_metadata: FlashAttentionMetadata = current_metadata
808
809 num_tokens, hidden_size = query.shape
810 +
811 # Reshape the query, key, and value tensors.
812 query = query.view(-1, num_heads, head_size)
644 - key = key.view(-1, num_kv_heads, head_size)
645 - value = value.view(-1, num_kv_heads, head_size)
813 + if (key is not None) and (value is not None):
814 + key = key.view(-1, num_kv_heads, head_size)
815 + value = value.view(-1, num_kv_heads, head_size)
816
817 if kv_cache.numel() > 0:
818 key_cache = kv_cache[0]
819 value_cache = kv_cache[1]
820 + # We skip updating the KV cache under two conditions:
821 + # a. When the Attention Type is ENCODER. In this phase, we compute
822 + # only the encoder attention without updating the cache.
823 + # b. When both Key and Value are None. This occurs during
824 + # cross-attention computation in the decoding phase, where the KV
825 + # cache is already populated with the cross-attention tensor.
826 + # Thus, we skip cache updates during this time.
827 + if (attn_type != AttentionType.ENCODER) and (key is not None) and (
828 + value is not None):
829 + if attn_type == AttentionType.ENCODER_DECODER:
830 + # Update cross-attention KV cache (prefill-only)
831 + updated_slot_mapping = attn_metadata.cross_slot_mapping
832 + else:
833 + # Update self-attention KV cache (prefill/decode)
834 + updated_slot_mapping = attn_metadata.slot_mapping
835 +
836 + # Reshape the input keys and values and store them in the cache.
837 + # If kv_cache is not provided, the new key and value tensors are
838 + # not cached. This happens during the initial memory profiling run.
839 + torch.ops._C_cache_ops.reshape_and_cache_flash(
840 + key,
841 + value,
842 + kv_cache[0],
843 + kv_cache[1],
844 + updated_slot_mapping.flatten(), # type: ignore[union-attr]
845 + kv_cache_dtype,
846 + k_scale,
847 + v_scale,
848 + )
849
651 - # Reshape the input keys and values and store them in the cache.
652 - # If kv_cache is not provided, the new key and value tensors are
653 - # not cached. This happens during the initial memory profiling run.
654 - torch.ops._C_cache_ops.reshape_and_cache_flash(
655 - key,
656 - value,
657 - kv_cache[0],
658 - kv_cache[1],
659 - attn_metadata.slot_mapping.flatten(),
660 - kv_cache_dtype,
661 - k_scale,
662 - v_scale,
663 - )
664 -
665 - num_prefill_tokens = attn_metadata.num_prefill_tokens
666 - num_decode_tokens = attn_metadata.num_decode_tokens
667 - assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
668 - f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
669 - assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
670 - f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
671 -
672 - # Query for decode. KV is not needed because it is already cached.
673 - decode_query = query[num_prefill_tokens:]
850 + (num_prefill_query_tokens, num_prefill_kv_tokens,
851 + num_decode_query_tokens) = \
852 + get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
853 + decode_query = query[num_prefill_query_tokens:]
854 # QKV for prefill.
675 - query = query[:num_prefill_tokens]
676 - key = key[:num_prefill_tokens]
677 - value = value[:num_prefill_tokens]
678 -
679 - assert query.shape[0] == num_prefill_tokens
680 - assert decode_query.shape[0] == num_decode_tokens
855 + query = query[:num_prefill_query_tokens]
856 + assert query.shape[0] == num_prefill_query_tokens
857 + assert decode_query.shape[0] == num_decode_query_tokens
858
859 prefill_output: Optional[torch.Tensor] = None
860 decode_output: Optional[torch.Tensor] = None
684 -
861 if prefill_meta := attn_metadata.prefill_metadata:
862 # Prompt run.
863 if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
@@ -689,22 +865,30 @@ def unified_flash_attention(
865 # normal attention
866 # When block_tables are not filled, it means q and k are the
867 # prompt, and they have the same length.
868 + q_seq_start_loc, q_seq_len, k_seq_start_loc, k_seq_len = \
869 + _get_query_key_seq_metadata(prefill_meta, True, attn_type)
870 +
871 + key = key[:num_prefill_kv_tokens]
872 + value = value[:num_prefill_kv_tokens]
873 +
874 prefill_output = flash_attn_varlen_func(
875 q=query,
876 k=key,
877 v=value,
696 - cu_seqlens_q=prefill_meta.seq_start_loc,
697 - cu_seqlens_k=prefill_meta.seq_start_loc,
698 - max_seqlen_q=prefill_meta.max_prefill_seq_len,
699 - max_seqlen_k=prefill_meta.max_prefill_seq_len,
878 + cu_seqlens_q=q_seq_start_loc,
879 + cu_seqlens_k=k_seq_start_loc,
880 + max_seqlen_q=q_seq_len,
881 + max_seqlen_k=k_seq_len,
882 softmax_scale=softmax_scale,
701 - causal=True,
883 + causal=_get_causal_option(attn_type),
884 window_size=window_size,
885 alibi_slopes=alibi_slopes,
886 softcap=logits_soft_cap,
887 )
888 else:
889 # prefix-enabled attention
890 + assert attn_type == AttentionType.DECODER, (
891 + "Only decoder-only models support prefix caching")
892 assert prefill_meta.seq_lens is not None
893 max_seq_len = max(prefill_meta.seq_lens)
894 prefill_output = flash_attn_varlen_func( # noqa
@@ -729,6 +913,8 @@ def unified_flash_attention(
913 # because different queries might have different lengths.
914 assert decode_meta.max_decode_query_len is not None
915 if decode_meta.max_decode_query_len > 1:
916 + assert attn_type == AttentionType.DECODER, (
917 + "Only decoder-only models support max_decode_query_len > 1")
918 decode_output = flash_attn_varlen_func(
919 q=decode_query,
920 k=key_cache,
@@ -746,12 +932,17 @@ def unified_flash_attention(
932 )
933 else:
934 # Use flash_attn_with_kvcache for normal decoding.
935 + (
936 + seq_lens_arg,
937 + _,
938 + block_tables_arg,
939 + ) = get_seq_len_block_table_args(decode_meta, False, attn_type)
940 decode_output = flash_attn_with_kvcache(
941 q=decode_query.unsqueeze(1),
942 k_cache=key_cache,
943 v_cache=value_cache,
753 - block_table=decode_meta.block_tables,
754 - cache_seqlens=decode_meta.seq_lens_tensor,
944 + block_table=block_tables_arg,
945 + cache_seqlens=seq_lens_arg,
946 softmax_scale=softmax_scale,
947 causal=True,
948 window_size=window_size,
@@ -761,10 +952,10 @@ def unified_flash_attention(
952
953 if prefill_output is None:
954 assert decode_output is not None
764 - return decode_output.view(num_decode_tokens, hidden_size)
955 + return decode_output.view(num_decode_query_tokens, hidden_size)
956 if decode_output is None:
957 assert prefill_output is not None
767 - return prefill_output.view(num_prefill_tokens, hidden_size)
958 + return prefill_output.view(num_prefill_query_tokens, hidden_size)
959
960 # Chunked prefill does not work with speculative decoding.
961 # Therefore, the query length for decode should be 1 in chunked prefill.
@@ -786,6 +977,7 @@ def unified_flash_attention_fake(
977 k_scale: float,
978 v_scale: float,
979 softmax_scale: float,
980 + attn_type_int_val: int,
981 window_size: Optional[List[int]] = None,
982 alibi_slopes: Optional[torch.Tensor] = None,
983 logits_soft_cap: Optional[float] = None,
vllm/attention/backends/utils.py
+143 -16
@@ -1,13 +1,14 @@
1 """Attention backend utils"""
2 from collections import defaultdict
3 from contextlib import contextmanager
4 -from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union
4 +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type, TypeVar, Union
5
6 import numpy as np
7 import torch
8
9 from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder,
10 AttentionState)
11 +from vllm.attention.backends.abstract import AttentionType
12 from vllm.multimodal import MultiModalPlaceholderMap
13 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
14
@@ -336,11 +337,13 @@ class CommonAttentionState(AttentionState):
337 use_cuda_graph=True,
338 )
339 if is_encoder_decoder_model:
339 - # The encoder decoder model works only with XFormers backend.
340 - # Assert the same.
341 - assert self.runner.attn_backend.get_name() == "XFORMERS", \
342 - f"Expected attn_backend name to be 'XFORMERS', but "\
343 - f" got '{self.runner.attn_backend.get_name()}'"
340 + # The encoder decoder model works only with XFormers and
341 + # Flash Attention backend. Assert the same.
342 + assert self.runner.attn_backend.get_name() in\
343 + ["XFORMERS", "FLASH_ATTN"], \
344 + f"Expected attn_backend name to be either 'XFORMERS' or " \
345 + f"'FLASH_ATTN', but "\
346 + f"got '{self.runner.attn_backend.get_name()}'"
347 self._update_captured_metadata_for_enc_dec_model(
348 batch_size=batch_size, attn_metadata=attn_metadata)
349
@@ -356,11 +359,13 @@ class CommonAttentionState(AttentionState):
359 "block_tables": attn_metadata.decode_metadata.block_tables,
360 }
361 if is_encoder_decoder_model:
359 - # The encoder decoder model works only with XFormers backend.
360 - # Assert the same.
361 - assert self.runner.attn_backend.get_name() == "XFORMERS", \
362 - f"Expected attn_backend name to be 'XFORMERS', but "\
363 - f" got '{self.runner.attn_backend.get_name()}'"
362 + # The encoder decoder model works only with XFormers and
363 + # Flash Attention backend. Assert the same.
364 + assert self.runner.attn_backend.get_name() in\
365 + ["XFORMERS", "FLASH_ATTN"], \
366 + f"Expected attn_backend name to be either 'XFORMERS' or "\
367 + f"'FLASH_ATTN', but "\
368 + f"got '{self.runner.attn_backend.get_name()}'"
369 self._add_additonal_input_buffers_for_enc_dec_model(
370 attn_metadata=attn_metadata, input_buffers=input_buffers)
371 return input_buffers
@@ -375,11 +380,13 @@ class CommonAttentionState(AttentionState):
380 input_buffers["block_tables"].copy_(
381 attn_metadata.decode_metadata.block_tables, non_blocking=True)
382 if is_encoder_decoder_model:
378 - # The encoder decoder model works only with XFormers backend.
379 - # Assert the same.
380 - assert self.runner.attn_backend.get_name() == "XFORMERS", \
381 - f"Expected attn_backend name to be 'XFORMERS', but "\
382 - f" got '{self.runner.attn_backend.get_name()}'"
383 + # The encoder decoder model works only with XFormers and
384 + # Flash Attention backend. Assert the same.
385 + assert self.runner.attn_backend.get_name() in\
386 + ["XFORMERS", "FLASH_ATTN"], \
387 + f"Expected attn_backend name to be either 'XFORMERS' or "\
388 + f"'FLASH_ATTN', but "\
389 + f"got '{self.runner.attn_backend.get_name()}'"
390 self._prepare_input_buffers_for_enc_dec_model(
391 attn_metadata, input_buffers)
392
@@ -411,6 +418,7 @@ class CommonAttentionState(AttentionState):
418 attn_metadata.encoder_seq_lens_tensor = torch.full(
419 (batch_size, ), 1, dtype=torch.int).cuda()
420 attn_metadata.max_encoder_seq_len = self.runner.max_seq_len_to_capture
421 + attn_metadata.num_encoder_tokens = 0
422
423 def _add_additonal_input_buffers_for_enc_dec_model(
424 self, attn_metadata, input_buffers: Dict[str, Any]):
@@ -453,3 +461,122 @@ class CommonAttentionState(AttentionState):
461 input_buffers["cross_block_tables"].copy_(
462 attn_metadata.decode_metadata.cross_block_tables,
463 non_blocking=True)
464 +
465 +
466 +def is_all_encoder_attn_metadata_set(attn_metadata):
467 + '''
468 + All attention metadata required for encoder attention is set.
469 + '''
470 + return ((attn_metadata.encoder_seq_lens is not None)
471 + and (attn_metadata.encoder_seq_lens_tensor is not None)
472 + and (attn_metadata.max_encoder_seq_len is not None))
473 +
474 +
475 +def is_all_cross_attn_metadata_set(attn_metadata):
476 + '''
477 + All attention metadata required for enc/dec cross-attention is set.
478 +
479 + Superset of encoder attention required metadata.
480 + '''
481 + return (attn_metadata.is_all_encoder_attn_metadata_set
482 + and (attn_metadata.cross_slot_mapping is not None)
483 + and (attn_metadata.cross_block_tables is not None))
484 +
485 +
486 +def get_seq_len_block_table_args(
487 + attn_metadata,
488 + is_prompt: bool,
489 + attn_type: AttentionType,
490 +) -> tuple:
491 + '''
492 + The particular choice of sequence-length- and block-table-related
493 + attributes which should be extracted from attn_metadata is dependent
494 + on the type of attention operation.
495 +
496 + Decoder attn -> select entirely decoder self-attention-related fields
497 + Encoder/decoder cross-attn -> select encoder sequence lengths &
498 + cross-attn block-tables fields
499 + Encoder attn -> select encoder sequence lengths fields & no block tables
500 +
501 + Arguments:
502 +
503 + * attn_metadata: Attention metadata structure associated with attention op
504 + * is_prompt: True if prefill, False otherwise
505 + * attn_type: encoder attention, decoder self-attention,
506 + encoder/decoder cross-attention
507 +
508 + Returns:
509 +
510 + * Appropriate sequence-lengths tensor
511 + * Appropriate max sequence-length scalar
512 + * Appropriate block tables (or None)
513 + '''
514 +
515 + if attn_type == AttentionType.DECODER:
516 + # Decoder self-attention
517 + # Choose max_seq_len based on whether we are in prompt_run
518 + if is_prompt:
519 + max_seq_len = attn_metadata.max_prefill_seq_len
520 + else:
521 + max_seq_len = attn_metadata.max_decode_seq_len
522 + return (attn_metadata.seq_lens_tensor, max_seq_len,
523 + attn_metadata.block_tables)
524 + elif attn_type == AttentionType.ENCODER_DECODER:
525 + # Enc/dec cross-attention KVs match encoder sequence length;
526 + # cross-attention utilizes special "cross" block tables
527 + return (attn_metadata.encoder_seq_lens_tensor,
528 + attn_metadata.max_encoder_seq_len,
529 + attn_metadata.cross_block_tables)
530 + elif attn_type == AttentionType.ENCODER:
531 + # No block tables associated with encoder attention
532 + return (attn_metadata.encoder_seq_lens_tensor,
533 + attn_metadata.max_encoder_seq_len, None)
534 + else:
535 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
536 +
537 +
538 +def get_num_prefill_decode_query_kv_tokens(
539 + attn_metadata,
540 + attn_type: AttentionType,
541 +) -> Tuple[int, int, int]:
542 + """
543 + Calculate the number of prefill and decode tokens for query, key/value
544 + based on the attention metadata and the specified attention type.
545 +
546 + Args:
547 + attn_metadata (FlashAttentionMetadata): Attention Metadata object.
548 + attn_type (AttentionType): The type of attention being used.
549 + Returns:
550 + Tuple[int, int, int]: A tuple containing three integers:
551 + - The number of prefill query tokens.
552 + - The number of prefill key/value tokens.
553 + - The number of decode query tokens.
554 +
555 + Raises:
556 + AssertionError: If the number of encoder tokens in `attn_metadata`
557 + is `None` when required for the calculations.
558 + """
559 + num_prefill_query_tokens = 0
560 + num_decode_query_tokens = 0
561 + num_prefill_kv_tokens = 0
562 + if attn_type == AttentionType.ENCODER:
563 + # Encoder attention is only invoked during prefill phase.
564 + # The same input servers a both query and key.
565 + assert attn_metadata.num_encoder_tokens is not None
566 + num_prefill_query_tokens = attn_metadata.num_encoder_tokens
567 + num_prefill_kv_tokens = attn_metadata.num_encoder_tokens
568 + num_decode_query_tokens = 0
569 + elif attn_type == AttentionType.ENCODER_DECODER:
570 + assert attn_metadata.num_encoder_tokens is not None
571 + num_prefill_query_tokens = attn_metadata.num_prefill_tokens
572 + # The key is the encoder/cross-attention.
573 + num_prefill_kv_tokens = attn_metadata.num_encoder_tokens
574 + num_decode_query_tokens = attn_metadata.num_decode_tokens
575 + else: # attn_type == AttentionType.DECODER or
576 + # attn_type == AttentionType.ENCODER_ONLY
577 + num_prefill_query_tokens = attn_metadata.num_prefill_tokens
578 + num_prefill_kv_tokens = attn_metadata.num_prefill_tokens
579 + num_decode_query_tokens = attn_metadata.num_decode_tokens
580 +
581 + return (num_prefill_query_tokens, num_prefill_kv_tokens,
582 + num_decode_query_tokens)
vllm/attention/backends/xformers.py
+26 -105
@@ -11,8 +11,10 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
11
12 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 AttentionMetadata, AttentionType)
14 -from vllm.attention.backends.utils import (CommonAttentionState,
15 - CommonMetadataBuilder)
14 +from vllm.attention.backends.utils import (
15 + CommonAttentionState, CommonMetadataBuilder,
16 + get_num_prefill_decode_query_kv_tokens, get_seq_len_block_table_args,
17 + is_all_cross_attn_metadata_set, is_all_encoder_attn_metadata_set)
18 from vllm.attention.ops.paged_attn import (PagedAttention,
19 PagedAttentionMetadata)
20 from vllm.logger import init_logger
@@ -135,6 +137,11 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
137 # Encoder sequence lengths representation
138 encoder_seq_lens: Optional[List[int]] = None
139 encoder_seq_lens_tensor: Optional[torch.Tensor] = None
140 + # FIXME: It is for flash attn.
141 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
142 + # the batch, used to index into sequence. E.g., if the sequence length is
143 + # [4, 6], it is [0, 4, 10].
144 + encoder_seq_start_loc: Optional[torch.Tensor] = None
145
146 # Maximum sequence length among encoder sequences
147 max_encoder_seq_len: Optional[int] = None
@@ -162,9 +169,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
169 '''
170 All attention metadata required for encoder attention is set.
171 '''
165 - return ((self.encoder_seq_lens is not None)
166 - and (self.encoder_seq_lens_tensor is not None)
167 - and (self.max_encoder_seq_len is not None))
172 + return is_all_encoder_attn_metadata_set(self)
173
174 @property
175 def is_all_cross_attn_metadata_set(self):
@@ -173,9 +178,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
178
179 Superset of encoder attention required metadata.
180 '''
176 - return (self.is_all_encoder_attn_metadata_set
177 - and (self.cross_slot_mapping is not None)
178 - and (self.cross_block_tables is not None))
181 + return is_all_cross_attn_metadata_set(self)
182
183 @property
184 def prefill_metadata(self) -> Optional["XFormersMetadata"]:
@@ -329,64 +332,6 @@ def _set_attn_bias(
332 raise AttributeError(f"Invalid attention type {str(attn_type)}")
333
334
332 -def _get_seq_len_block_table_args(
333 - attn_metadata: XFormersMetadata,
334 - is_prompt: bool,
335 - attn_type: AttentionType,
336 -) -> tuple:
337 - '''
338 - The particular choice of sequence-length- and block-table-related
339 - attributes which should be extracted from attn_metadata is dependent
340 - on the type of attention operation.
341 -
342 - Decoder attn -> select entirely decoder self-attention-related fields
343 - Encoder/decoder cross-attn -> select encoder sequence lengths &
344 - cross-attn block-tables fields
345 - Encoder attn -> select encoder sequence lengths fields & no block tables
346 -
347 - Arguments:
348 -
349 - * attn_metadata: Attention metadata structure associated with attention op
350 - * is_prompt: True if prefill, False otherwise
351 - * attn_type: encoder attention, decoder self-attention,
352 - encoder/decoder cross-attention
353 -
354 - Returns:
355 -
356 - * Appropriate sequence-lengths tensor
357 - * Appropriate max sequence-length scalar
358 - * Appropriate block tables (or None)
359 - '''
360 -
361 - if attn_type == AttentionType.DECODER:
362 - # Decoder self-attention
363 - # Choose max_seq_len based on whether we are in prompt_run
364 - if is_prompt:
365 - max_seq_len = attn_metadata.max_prefill_seq_len
366 - else:
367 - max_seq_len = attn_metadata.max_decode_seq_len
368 - return (attn_metadata.seq_lens_tensor, max_seq_len,
369 - attn_metadata.block_tables)
370 - elif attn_type == AttentionType.ENCODER_DECODER:
371 - # Enc/dec cross-attention KVs match encoder sequence length;
372 - # cross-attention utilizes special "cross" block tables
373 - return (attn_metadata.encoder_seq_lens_tensor,
374 - attn_metadata.max_encoder_seq_len,
375 - attn_metadata.cross_block_tables)
376 - elif attn_type == AttentionType.ENCODER:
377 - # No block tables associated with encoder attention
378 - return (attn_metadata.encoder_seq_lens_tensor,
379 - attn_metadata.max_encoder_seq_len, None)
380 - elif attn_type == AttentionType.ENCODER_ONLY:
381 - assert is_prompt, "Should not have decode for encoder only model."
382 -
383 - # No block tables associated with encoder attention
384 - return (attn_metadata.seq_lens_tensor,
385 - attn_metadata.max_prefill_seq_len, None)
386 - else:
387 - raise AttributeError(f"Invalid attention type {str(attn_type)}")
388 -
389 -
335 class XFormersMetadataBuilder(CommonMetadataBuilder[XFormersMetadata]):
336
337 _metadata_cls = XFormersMetadata
@@ -574,45 +519,21 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
519 updated_slot_mapping,
520 self.kv_cache_dtype,
521 k_scale, v_scale)
577 -
578 - if attn_type == AttentionType.ENCODER:
579 - # Encoder attention - chunked prefill is not applicable;
580 - # derive token-count from query shape & and treat them
581 - # as 100% prefill tokens
582 - assert attn_metadata.num_encoder_tokens is not None
583 - num_prefill_tokens = attn_metadata.num_encoder_tokens
584 - num_encoder_tokens = attn_metadata.num_encoder_tokens
585 - num_decode_tokens = 0
586 - elif attn_type == AttentionType.DECODER:
587 - # Decoder self-attention supports chunked prefill.
588 - num_prefill_tokens = attn_metadata.num_prefill_tokens
589 - num_encoder_tokens = attn_metadata.num_prefill_tokens
590 - num_decode_tokens = attn_metadata.num_decode_tokens
591 - # Only enforce this shape-constraint for decoder
592 - # self-attention
593 - assert key.shape[0] == num_prefill_tokens + num_decode_tokens
594 - assert value.shape[0] == num_prefill_tokens + num_decode_tokens
595 - else: # attn_type == AttentionType.ENCODER_DECODER
596 - # Encoder/decoder cross-attention requires no chunked
597 - # prefill (100% prefill or 100% decode tokens, no mix)
598 - num_prefill_tokens = attn_metadata.num_prefill_tokens
599 - if attn_metadata.num_encoder_tokens is not None:
600 - num_encoder_tokens = attn_metadata.num_encoder_tokens
601 - else:
602 - num_encoder_tokens = attn_metadata.num_prefill_tokens
603 - num_decode_tokens = attn_metadata.num_decode_tokens
522 + (num_prefill_query_tokens, num_prefill_kv_tokens,
523 + num_decode_query_tokens) = \
524 + get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
525
526 output = torch.empty_like(query)
527 # Query for decode. KV is not needed because it is already cached.
607 - decode_query = query[num_prefill_tokens:]
528 + decode_query = query[num_prefill_query_tokens:]
529 # QKV for prefill.
609 - query = query[:num_prefill_tokens]
530 + query = query[:num_prefill_query_tokens]
531 if key is not None and value is not None:
611 - key = key[:num_encoder_tokens]
612 - value = value[:num_encoder_tokens]
532 + key = key[:num_prefill_kv_tokens]
533 + value = value[:num_prefill_kv_tokens]
534
614 - assert query.shape[0] == num_prefill_tokens
615 - assert decode_query.shape[0] == num_decode_tokens
535 + assert query.shape[0] == num_prefill_query_tokens
536 + assert decode_query.shape[0] == num_decode_query_tokens
537
538 if prefill_meta := attn_metadata.prefill_metadata:
539 # Prompt run.
@@ -622,8 +543,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
543 # prefix.
544 out = self._run_memory_efficient_xformers_forward(
545 query, key, value, prefill_meta, attn_type=attn_type)
625 - assert out.shape == output[:num_prefill_tokens].shape
626 - output[:num_prefill_tokens] = out
546 + assert out.shape == output[:num_prefill_query_tokens].shape
547 + output[:num_prefill_query_tokens] = out
548 else:
549 assert attn_type != AttentionType.ENCODER_ONLY, (
550 "Encoder-only models should not have prefix attention.")
@@ -652,8 +573,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
573 k_scale,
574 v_scale,
575 )
655 - assert output[:num_prefill_tokens].shape == out.shape
656 - output[:num_prefill_tokens] = out
576 + assert output[:num_prefill_query_tokens].shape == out.shape
577 + output[:num_prefill_query_tokens] = out
578
579 if decode_meta := attn_metadata.decode_metadata:
580 assert attn_type != AttentionType.ENCODER_ONLY, (
@@ -663,9 +584,9 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
584 seq_lens_arg,
585 max_seq_len_arg,
586 block_tables_arg,
666 - ) = _get_seq_len_block_table_args(decode_meta, False, attn_type)
587 + ) = get_seq_len_block_table_args(decode_meta, False, attn_type)
588
668 - output[num_prefill_tokens:] = PagedAttention.forward_decode(
589 + output[num_prefill_query_tokens:] = PagedAttention.forward_decode(
590 decode_query,
591 key_cache,
592 value_cache,
vllm/attention/selector.py
+1 -1
@@ -98,7 +98,6 @@ def get_attn_backend(
98 is_blocksparse: bool = False,
99 ) -> Type[AttentionBackend]:
100 """Selects which attention backend to use and lazily imports it."""
101 -
101 if is_blocksparse:
102 logger.info("Using BlocksparseFlashAttention backend.")
103 from vllm.attention.backends.blocksparse_attn import (
@@ -108,6 +107,7 @@ def get_attn_backend(
107 backend = which_attn_to_use(head_size, dtype, kv_cache_dtype, block_size,
108 is_attention_free)
109 if backend == _Backend.FLASH_ATTN:
110 + logger.info("Using Flash Attention backend.")
111 from vllm.attention.backends.flash_attn import ( # noqa: F401
112 FlashAttentionBackend)
113 return FlashAttentionBackend
vllm/model_executor/models/bart.py
-2
@@ -624,8 +624,6 @@ class BartEncoder(nn.Module):
624 Decoder output torch.Tensor
625 """
626 # retrieve input_ids and inputs_embeds
627 -
628 - input_ids = input_ids.view(-1, input_ids.shape[-1])
627 inputs_embeds = self.embed_tokens(input_ids)
628
629 embed_pos = self.embed_positions(
vllm/utils.py
+2 -2
@@ -80,8 +80,8 @@ STR_NOT_IMPL_ENC_DEC_SPEC_DEC = ("Speculative decoding is not "
80 "currently supported with encoder/"
81 "decoder models.")
82
83 -STR_NOT_IMPL_ENC_DEC_BACKEND = ("XFormers is the only backend "
84 - "currently supported with encoder/"
83 +STR_NOT_IMPL_ENC_DEC_BACKEND = ("XFormers and Flash-Attention are the only "
84 + "backends currently supported with encoder/"
85 "decoder models.")
86
87 STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER = ("Prompt adapters are not "
vllm/worker/enc_dec_model_runner.py
+25 -10
@@ -19,6 +19,7 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry
19 from vllm.logger import init_logger
20 from vllm.model_executor import SamplingMetadata
21 from vllm.model_executor.layers.sampler import SamplerOutput
22 +from vllm.model_executor.model_loader.utils import get_architecture_class_name
23 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
24 MultiModalRegistry)
25 from vllm.sampling_params import SamplingParams
@@ -36,6 +37,11 @@ from vllm.worker.utils import assert_enc_dec_mr_supported_scenario
37
38 logger = init_logger(__name__)
39
40 +# The Mllama model has PagedAttention specific logic because of which it
41 +# can only be run with the XFORMERS backend
42 +# TODO Make Mllama model work with Flash Attention backend.
43 +_XFORMERS_ONLY_ENCODER_DECODER_ARCHS = ["MllamaForConditionalGeneration"]
44 +
45
46 @dataclasses.dataclass(frozen=True)
47 class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata):
@@ -101,9 +107,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
107 models) but these arguments are present here for compatibility with
108 the base-class constructor.
109 '''
104 -
105 - self._maybe_force_supported_attention_backend()
106 -
110 + self._maybe_force_supported_attention_backend(model_config)
111 super().__init__(
112 model_config,
113 parallel_config,
@@ -119,7 +123,12 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
123 # Crash for unsupported encoder/scenarios
124 assert_enc_dec_mr_supported_scenario(self)
125
122 - def _maybe_force_supported_attention_backend(self):
126 + def _is_xformers_only_encoder_decoder_model(self,
127 + model: ModelConfig) -> bool:
128 + return get_architecture_class_name(
129 + model) in _XFORMERS_ONLY_ENCODER_DECODER_ARCHS
130 +
131 + def _maybe_force_supported_attention_backend(self, model: ModelConfig):
132 '''
133 Force vLLM to use the XFormers attention backend,
134 which is currently the only supported option.
@@ -135,22 +144,26 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
144 is_forced_by_global = maybe_global_forced_backend is not None
145 is_forced_by_env_var = maybe_env_var_forced_backend is not None
146
138 - if not (is_forced_by_global or is_forced_by_env_var):
147 + if not (is_forced_by_global or is_forced_by_env_var) \
148 + and self._is_xformers_only_encoder_decoder_model(model):
149 # The user has not already specified an attention backend
150 # override
141 - logger.info("EncoderDecoderModelRunner requires "
142 - "XFormers backend; overriding backend "
143 - "auto-selection and forcing XFormers.")
151 + logger.info(
152 + "Encoder-Decoder Model Architecture %s requires XFormers "
153 + "backend; overriding backend auto-selection and "
154 + "forcing XFormers.", get_architecture_class_name(model))
155 global_force_attn_backend(_Backend.XFORMERS)
156 elif is_forced_by_global:
157 # Backend override enforced by global variable takes
158 # precedence over vLLM backend environment variable.
148 - if maybe_global_forced_backend != _Backend.XFORMERS:
159 + if maybe_global_forced_backend not in\
160 + [_Backend.XFORMERS, _Backend.FLASH_ATTN]:
161 raise_backend_err()
162 elif is_forced_by_env_var:
163 # Backend override enforced by vLLM backend
164 # environment variable
153 - if maybe_env_var_forced_backend != _Backend.XFORMERS:
165 + if maybe_env_var_forced_backend not in\
166 + [_Backend.XFORMERS, _Backend.FLASH_ATTN]:
167 raise_backend_err()
168
169 def _list_to_int32_tensor(
@@ -532,6 +545,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
545 attn_metadata.encoder_seq_lens,
546 attn_metadata.encoder_seq_lens_tensor,
547 attn_metadata.max_encoder_seq_len,
548 + attn_metadata.encoder_seq_start_loc,
549 attn_metadata.cross_slot_mapping,
550 attn_metadata.cross_block_tables,
551 ) = (
@@ -539,6 +553,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
553 encoder_seq_lens,
554 encoder_seq_lens_tensor,
555 max_encoder_seq_len,
556 + encoder_seq_start_loc,
557 cross_slot_mapping_tensor,
558 cross_block_tables,
559 )