@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 978b45f399
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Kernel] Flash Attention 3 Support (#12093)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>

Lucas Wilkinson committed Jan 23, 2025 at 09:45 UTC 978b45f39970f81d2857fbb8aada5b44619d258c
8 files changed +150 -82
CMakeLists.txt
+20 -25
@@ -24,9 +24,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake)
24 # Suppress potential warnings about unused manually-specified variables
25 set(ignoreMe "${VLLM_PYTHON_PATH}")
26
27 -# Prevent installation of dependencies (cutlass) by default.
28 -install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS)
29 -
27 #
28 # Supported python versions. These versions will be searched in order, the
29 # first match will be selected. These should be kept in sync with setup.py.
@@ -535,7 +532,7 @@ if(VLLM_GPU_LANG STREQUAL "HIP")
532 endif()
533
534 # vllm-flash-attn currently only supported on CUDA
538 -if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda")
535 +if (NOT VLLM_GPU_LANG STREQUAL "CUDA")
536 return()
537 endif ()
538
@@ -558,7 +555,7 @@ endif()
555 # They should be identical but if they aren't, this is a massive footgun.
556 #
557 # The vllm-flash-attn install rules are nested under vllm to make sure the library gets installed in the correct place.
561 -# To only install vllm-flash-attn, use --component vllm_flash_attn_c.
558 +# To only install vllm-flash-attn, use --component _vllm_fa2_C (for FA2) or --component _vllm_fa3_C (for FA3).
559 # If no component is specified, vllm-flash-attn is still installed.
560
561 # If VLLM_FLASH_ATTN_SRC_DIR is set, vllm-flash-attn is installed from that directory instead of downloading.
@@ -570,43 +567,41 @@ if (DEFINED ENV{VLLM_FLASH_ATTN_SRC_DIR})
567 endif()
568
569 if(VLLM_FLASH_ATTN_SRC_DIR)
573 - FetchContent_Declare(vllm-flash-attn SOURCE_DIR ${VLLM_FLASH_ATTN_SRC_DIR})
570 + FetchContent_Declare(
571 + vllm-flash-attn SOURCE_DIR
572 + ${VLLM_FLASH_ATTN_SRC_DIR}
573 + BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
574 + )
575 else()
576 FetchContent_Declare(
577 vllm-flash-attn
578 GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
578 - GIT_TAG 96266b1111111f3d11aabefaf3bacbab6a89d03c
579 + GIT_TAG 90eacc1af2a7c3de62ea249e929ed5faccf38954
580 GIT_PROGRESS TRUE
581 # Don't share the vllm-flash-attn build between build types
582 BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
583 )
584 endif()
585
585 -# Set the parent build flag so that the vllm-flash-attn library does not redo compile flag and arch initialization.
586 -set(VLLM_PARENT_BUILD ON)
587 -
588 -# Ensure the vllm/vllm_flash_attn directory exists before installation
589 -install(CODE "file(MAKE_DIRECTORY \"\${CMAKE_INSTALL_PREFIX}/vllm/vllm_flash_attn\")" COMPONENT vllm_flash_attn_c)
590 -
591 -# Make sure vllm-flash-attn install rules are nested under vllm/
592 -install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY FALSE)" COMPONENT vllm_flash_attn_c)
593 -install(CODE "set(OLD_CMAKE_INSTALL_PREFIX \"\${CMAKE_INSTALL_PREFIX}\")" COMPONENT vllm_flash_attn_c)
594 -install(CODE "set(CMAKE_INSTALL_PREFIX \"\${CMAKE_INSTALL_PREFIX}/vllm/\")" COMPONENT vllm_flash_attn_c)
586
587 # Fetch the vllm-flash-attn library
588 FetchContent_MakeAvailable(vllm-flash-attn)
589 message(STATUS "vllm-flash-attn is available at ${vllm-flash-attn_SOURCE_DIR}")
590
600 -# Restore the install prefix
601 -install(CODE "set(CMAKE_INSTALL_PREFIX \"\${OLD_CMAKE_INSTALL_PREFIX}\")" COMPONENT vllm_flash_attn_c)
602 -install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" COMPONENT vllm_flash_attn_c)
591 +# Copy over the vllm-flash-attn python files (duplicated for fa2 and fa3, in
592 +# case only one is built, in the case both are built redundant work is done)
593 +install(
594 + DIRECTORY ${vllm-flash-attn_SOURCE_DIR}/vllm_flash_attn/
595 + DESTINATION vllm_flash_attn
596 + COMPONENT _vllm_fa2_C
597 + FILES_MATCHING PATTERN "*.py"
598 +)
599
604 -# Copy over the vllm-flash-attn python files
600 install(
606 - DIRECTORY ${vllm-flash-attn_SOURCE_DIR}/vllm_flash_attn/
607 - DESTINATION vllm/vllm_flash_attn
608 - COMPONENT vllm_flash_attn_c
609 - FILES_MATCHING PATTERN "*.py"
601 + DIRECTORY ${vllm-flash-attn_SOURCE_DIR}/vllm_flash_attn/
602 + DESTINATION vllm_flash_attn
603 + COMPONENT _vllm_fa3_C
604 + FILES_MATCHING PATTERN "*.py"
605 )
606
607 # Nothing after vllm-flash-attn, see comment about macros above
setup.py
+8 -4
@@ -228,8 +228,11 @@ class cmake_build_ext(build_ext):
228
229 # CMake appends the extension prefix to the install path,
230 # and outdir already contains that prefix, so we need to remove it.
231 + # We assume only the final component of extension prefix is added by
232 + # CMake, this is currently true for current extensions but may not
233 + # always be the case.
234 prefix = outdir
232 - for i in range(ext.name.count('.')):
235 + if '.' in ext.name:
236 prefix = prefix.parent
237
238 # prefix here should actually be the same for all components
@@ -298,7 +301,8 @@ class repackage_wheel(build_ext):
301 files_to_copy = [
302 "vllm/_C.abi3.so",
303 "vllm/_moe_C.abi3.so",
301 - "vllm/vllm_flash_attn/vllm_flash_attn_c.abi3.so",
304 + "vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
305 + "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
306 "vllm/vllm_flash_attn/flash_attn_interface.py",
307 "vllm/vllm_flash_attn/__init__.py",
308 "vllm/cumem_allocator.abi3.so",
@@ -593,8 +597,8 @@ if _is_hip():
597 ext_modules.append(CMakeExtension(name="vllm._rocm_C"))
598
599 if _is_cuda():
596 - ext_modules.append(
597 - CMakeExtension(name="vllm.vllm_flash_attn.vllm_flash_attn_c"))
600 + ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
601 + ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
602 ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
603
604 if _build_custom_ops():
tests/kernels/test_cascade_flash_attn.py
+14 -10
@@ -78,6 +78,7 @@ CASES = [
78 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
79 @pytest.mark.parametrize("soft_cap", [None, 50])
80 @pytest.mark.parametrize("num_blocks", [2048])
81 +@pytest.mark.parametrize("fa_version", [2, 3])
82 @torch.inference_mode()
83 def test_cascade(
84 seq_lens_and_common_prefix: Tuple[List[Tuple[int, int]], int],
@@ -87,8 +88,14 @@ def test_cascade(
88 block_size: int,
89 soft_cap: Optional[float],
90 num_blocks: int,
91 + fa_version: int,
92 ) -> None:
93 torch.set_default_device("cuda")
94 + if fa_version == 3 and (torch.cuda.get_device_capability() == (8, 6)
95 + or torch.cuda.get_device_capability() == (8, 9)):
96 + pytest.skip("Flash attention version 3 fails on 8.6 and 8.9 due to "
97 + "insufficient shared memory for some shapes")
98 +
99 current_platform.seed_everything(0)
100
101 window_size = (-1, -1)
@@ -118,9 +125,7 @@ def test_cascade(
125 cu_query_lens = torch.tensor([0] + query_lens,
126 dtype=torch.int32).cumsum(dim=0,
127 dtype=torch.int32)
121 - cu_kv_lens = torch.tensor([0] + kv_lens,
122 - dtype=torch.int32).cumsum(dim=0,
123 - dtype=torch.int32)
128 + kv_lens_tensor = torch.tensor(kv_lens, dtype=torch.int32)
129 max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
130 block_tables = torch.randint(0,
131 num_blocks,
@@ -140,7 +145,7 @@ def test_cascade(
145 k=key_cache,
146 v=value_cache,
147 cu_seqlens_q=cu_query_lens,
143 - cu_seqlens_k=cu_kv_lens,
148 + seqused_k=kv_lens_tensor,
149 max_seqlen_q=max_query_len,
150 max_seqlen_k=max_kv_len,
151 softmax_scale=scale,
@@ -154,10 +159,8 @@ def test_cascade(
159 assert all(common_prefix_len < kv_len for kv_len in kv_lens)
160 cu_prefix_query_lens = torch.tensor([0, total_num_query_tokens],
161 dtype=torch.int32)
157 - cu_prefix_kv_lens = torch.tensor([0, common_prefix_len], dtype=torch.int32)
158 - cu_suffix_kv_lens = (
159 - cu_kv_lens -
160 - torch.arange(num_seqs + 1, dtype=torch.int32) * common_prefix_len)
162 + prefix_kv_lens = torch.tensor([common_prefix_len], dtype=torch.int32)
163 + suffix_kv_lens = kv_lens_tensor - common_prefix_len
164 output = torch.empty_like(query)
165 cascade_attention(
166 output=output,
@@ -167,8 +170,8 @@ def test_cascade(
170 cu_query_lens=cu_query_lens,
171 max_query_len=max_query_len,
172 cu_prefix_query_lens=cu_prefix_query_lens,
170 - cu_prefix_kv_lens=cu_prefix_kv_lens,
171 - cu_suffix_kv_lens=cu_suffix_kv_lens,
173 + prefix_kv_lens=prefix_kv_lens,
174 + suffix_kv_lens=suffix_kv_lens,
175 max_kv_len=max_kv_len,
176 softmax_scale=scale,
177 alibi_slopes=None,
@@ -176,6 +179,7 @@ def test_cascade(
179 logits_soft_cap=soft_cap if soft_cap is not None else 0,
180 block_table=block_tables,
181 common_prefix_len=common_prefix_len,
182 + fa_version=fa_version,
183 )
184
185 # Compare the results.
tests/kernels/test_flash_attn.py
+18 -4
@@ -80,6 +80,7 @@ def ref_paged_attn(
80 @pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
81 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
82 @pytest.mark.parametrize("sliding_window", [None, 256])
83 +@pytest.mark.parametrize("fa_version", [2, 3])
84 @torch.inference_mode()
85 def test_flash_attn_with_paged_kv(
86 use_out: bool,
@@ -91,8 +92,14 @@ def test_flash_attn_with_paged_kv(
92 soft_cap: Optional[float],
93 num_blocks: int,
94 sliding_window: Optional[int],
95 + fa_version: int,
96 ) -> None:
97 torch.set_default_device("cuda")
98 + if fa_version == 3 and (torch.cuda.get_device_capability() == (8, 6)
99 + or torch.cuda.get_device_capability() == (8, 9)):
100 + pytest.skip("Flash attention version 3 fails on 8.6 and 8.9 due to "
101 + "insufficient shared memory for some shapes")
102 +
103 current_platform.seed_everything(0)
104 num_seqs = len(kv_lens)
105 num_query_heads = num_heads[0]
@@ -131,6 +138,7 @@ def test_flash_attn_with_paged_kv(
138 cache_seqlens=kv_lens_tensor,
139 softcap=soft_cap if soft_cap is not None else 0,
140 window_size=window_size,
141 + fa_version=fa_version,
142 )
143 output = output if not use_out else out
144 output = output.squeeze(1)
@@ -159,6 +167,7 @@ def test_flash_attn_with_paged_kv(
167 @pytest.mark.parametrize("dtype", DTYPES)
168 @pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
169 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
170 +@pytest.mark.parametrize("fa_version", [2, 3])
171 @torch.inference_mode()
172 def test_varlen_with_paged_kv(
173 use_out: bool,
@@ -170,8 +179,14 @@ def test_varlen_with_paged_kv(
179 block_size: int,
180 soft_cap: Optional[float],
181 num_blocks: int,
182 + fa_version: int,
183 ) -> None:
184 torch.set_default_device("cuda")
185 + if fa_version == 3 and (torch.cuda.get_device_capability() == (8, 6)
186 + or torch.cuda.get_device_capability() == (8, 9)):
187 + pytest.skip("Flash attention version 3 fails on 8.6 and 8.9 due to "
188 + "insufficient shared memory for some shapes")
189 +
190 current_platform.seed_everything(0)
191 num_seqs = len(seq_lens)
192 query_lens = [x[0] for x in seq_lens]
@@ -198,9 +213,7 @@ def test_varlen_with_paged_kv(
213 cu_query_lens = torch.tensor([0] + query_lens,
214 dtype=torch.int32).cumsum(dim=0,
215 dtype=torch.int32)
201 - cu_kv_lens = torch.tensor([0] + kv_lens,
202 - dtype=torch.int32).cumsum(dim=0,
203 - dtype=torch.int32)
216 + kv_lens = torch.tensor(kv_lens, dtype=torch.int32)
217
218 max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
219 block_tables = torch.randint(0,
@@ -215,7 +228,7 @@ def test_varlen_with_paged_kv(
228 v=value_cache,
229 out=out,
230 cu_seqlens_q=cu_query_lens,
218 - cu_seqlens_k=cu_kv_lens,
231 + seqused_k=kv_lens,
232 max_seqlen_q=max_query_len,
233 max_seqlen_k=max_kv_len,
234 softmax_scale=scale,
@@ -223,6 +236,7 @@ def test_varlen_with_paged_kv(
236 window_size=window_size,
237 block_table=block_tables,
238 softcap=soft_cap if soft_cap is not None else 0,
239 + fa_version=fa_version,
240 )
241 output = output if not use_out else out
242
vllm/attention/backends/flash_attn.py
+24 -3
@@ -17,7 +17,9 @@ from vllm.attention.backends.utils import (
17 compute_slot_mapping_start_idx, get_num_prefill_decode_query_kv_tokens,
18 get_seq_len_block_table_args, is_all_cross_attn_metadata_set,
19 is_all_encoder_attn_metadata_set, is_block_tables_empty)
20 +from vllm.envs import VLLM_FLASH_ATTN_VERSION
21 from vllm.multimodal import MultiModalPlaceholderMap
22 +from vllm.platforms import current_platform
23 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
24
25 if TYPE_CHECKING:
@@ -25,7 +27,8 @@ if TYPE_CHECKING:
27 ModelInputForGPUWithSamplingMetadata)
28
29 from vllm.vllm_flash_attn import (flash_attn_varlen_func,
28 - flash_attn_with_kvcache)
30 + flash_attn_with_kvcache,
31 + is_fa_version_supported)
32
33
34 class FlashAttentionBackend(AttentionBackend):
@@ -634,6 +637,20 @@ class FlashAttentionImpl(AttentionImpl):
637 f"Supported head sizes are: {support_head_sizes}.")
638 self.attn_type = attn_type
639
640 + # if hopper default to FA3, otherwise stick to FA2 for now
641 + # TODO(lucas): profile FA3 on ampere to see if it makes sense to
642 + # use FA3 as default for both
643 + if current_platform.get_device_capability()[0] >= 9:
644 + self.fa_version = 3 if is_fa_version_supported(3) else 2
645 + else:
646 + self.fa_version = 2
647 +
648 + if VLLM_FLASH_ATTN_VERSION is not None:
649 + assert VLLM_FLASH_ATTN_VERSION in [2, 3]
650 + self.fa_version = VLLM_FLASH_ATTN_VERSION
651 +
652 + assert is_fa_version_supported(self.fa_version)
653 +
654 def forward(
655 self,
656 layer: AttentionLayer,
@@ -752,6 +769,7 @@ class FlashAttentionImpl(AttentionImpl):
769 alibi_slopes=alibi_slopes,
770 softcap=logits_soft_cap,
771 out=prefill_output,
772 + fa_version=self.fa_version,
773 )
774 else:
775 # prefix-enabled attention
@@ -765,7 +783,7 @@ class FlashAttentionImpl(AttentionImpl):
783 v=value_cache,
784 cu_seqlens_q=prefill_meta.query_start_loc,
785 max_seqlen_q=prefill_meta.max_query_len,
768 - cu_seqlens_k=prefill_meta.seq_start_loc,
786 + seqused_k=prefill_meta.seq_lens_tensor,
787 max_seqlen_k=max_seq_len,
788 softmax_scale=softmax_scale,
789 causal=True,
@@ -774,6 +792,7 @@ class FlashAttentionImpl(AttentionImpl):
792 block_table=prefill_meta.block_tables,
793 softcap=logits_soft_cap,
794 out=prefill_output,
795 + fa_version=self.fa_version,
796 )
797
798 if decode_meta := attn_metadata.decode_metadata:
@@ -793,7 +812,7 @@ class FlashAttentionImpl(AttentionImpl):
812 v=value_cache,
813 cu_seqlens_q=decode_meta.query_start_loc,
814 max_seqlen_q=decode_meta.max_decode_query_len,
796 - cu_seqlens_k=decode_meta.seq_start_loc,
815 + seqused_k=decode_meta.seq_lens_tensor,
816 max_seqlen_k=decode_meta.max_decode_seq_len,
817 softmax_scale=softmax_scale,
818 causal=True,
@@ -802,6 +821,7 @@ class FlashAttentionImpl(AttentionImpl):
821 softcap=logits_soft_cap,
822 block_table=decode_meta.block_tables,
823 out=decode_output,
824 + fa_version=self.fa_version,
825 )
826 else:
827 # Use flash_attn_with_kvcache for normal decoding.
@@ -822,6 +842,7 @@ class FlashAttentionImpl(AttentionImpl):
842 alibi_slopes=alibi_slopes,
843 softcap=logits_soft_cap,
844 out=decode_output.unsqueeze(1),
845 + fa_version=self.fa_version,
846 )
847 return output
848
vllm/envs.py
+12
@@ -11,6 +11,7 @@ if TYPE_CHECKING:
11 VLLM_NCCL_SO_PATH: Optional[str] = None
12 LD_LIBRARY_PATH: Optional[str] = None
13 VLLM_USE_TRITON_FLASH_ATTN: bool = False
14 + VLLM_FLASH_ATTN_VERSION: Optional[int] = None
15 LOCAL_RANK: int = 0
16 CUDA_VISIBLE_DEVICES: Optional[str] = None
17 VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60
@@ -90,6 +91,12 @@ def get_default_config_root():
91 )
92
93
94 +def maybe_convert_int(value: Optional[str]) -> Optional[int]:
95 + if value is None:
96 + return None
97 + return int(value)
98 +
99 +
100 # The begin-* and end* here are used by the documentation generator
101 # to extract the used env vars.
102
@@ -203,6 +210,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
210 lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
211 ("true", "1")),
212
213 + # Force vllm to use a specific flash-attention version (2 or 3), only valid
214 + # when using the flash-attention backend.
215 + "VLLM_FLASH_ATTN_VERSION":
216 + lambda: maybe_convert_int(os.environ.get("VLLM_FLASH_ATTN_VERSION", None)),
217 +
218 # Internal flag to enable Dynamo fullgraph capture
219 "VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
220 lambda: bool(
vllm/v1/attention/backends/flash_attn.py
+33 -11
@@ -9,8 +9,11 @@ import triton.language as tl
9
10 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11 AttentionMetadata, AttentionType)
12 +from vllm.envs import VLLM_FLASH_ATTN_VERSION
13 +from vllm.platforms import current_platform
14 from vllm.utils import cdiv
13 -from vllm.vllm_flash_attn import flash_attn_varlen_func
15 +from vllm.vllm_flash_attn import (flash_attn_varlen_func,
16 + is_fa_version_supported)
17
18
19 class FlashAttentionBackend(AttentionBackend):
@@ -63,7 +66,7 @@ class FlashAttentionMetadata:
66 max_query_len: int
67 query_start_loc: torch.Tensor
68 max_seq_len: int
66 - seq_start_loc: torch.Tensor
69 + seq_lens: torch.Tensor
70 block_table: torch.Tensor
71 slot_mapping: torch.Tensor
72
@@ -71,8 +74,8 @@ class FlashAttentionMetadata:
74 use_cascade: bool
75 common_prefix_len: int
76 cu_prefix_query_lens: Optional[torch.Tensor]
74 - cu_prefix_kv_lens: Optional[torch.Tensor]
75 - cu_suffix_kv_lens: Optional[torch.Tensor]
77 + prefix_kv_lens: Optional[torch.Tensor]
78 + suffix_kv_lens: Optional[torch.Tensor]
79
80 # For logging.
81 num_input_tokens: int = 0 # Number of tokens including padding.
@@ -128,6 +131,20 @@ class FlashAttentionImpl(AttentionImpl):
131 "are not implemented for "
132 "FlashAttentionImpl")
133
134 + # if hopper default to FA3, otherwise stick to FA2 for now
135 + # TODO(lucas): profile FA3 on ampere to see if it makes sense to
136 + # use FA3 as default for both
137 + if current_platform.get_device_capability()[0] >= 9:
138 + self.fa_version = 3 if is_fa_version_supported(3) else 2
139 + else:
140 + self.fa_version = 2
141 +
142 + if VLLM_FLASH_ATTN_VERSION is not None:
143 + assert VLLM_FLASH_ATTN_VERSION in [2, 3]
144 + self.fa_version = VLLM_FLASH_ATTN_VERSION
145 +
146 + assert is_fa_version_supported(self.fa_version)
147 +
148 def forward(
149 self,
150 layer: torch.nn.Module,
@@ -196,7 +213,7 @@ class FlashAttentionImpl(AttentionImpl):
213 out=output[:num_actual_tokens],
214 cu_seqlens_q=attn_metadata.query_start_loc,
215 max_seqlen_q=attn_metadata.max_query_len,
199 - cu_seqlens_k=attn_metadata.seq_start_loc,
216 + seqused_k=attn_metadata.seq_lens,
217 max_seqlen_k=attn_metadata.max_seq_len,
218 softmax_scale=self.scale,
219 causal=True,
@@ -204,6 +221,7 @@ class FlashAttentionImpl(AttentionImpl):
221 window_size=self.sliding_window,
222 block_table=attn_metadata.block_table,
223 softcap=self.logits_soft_cap,
224 + fa_version=self.fa_version,
225 )
226 return output
227
@@ -216,8 +234,8 @@ class FlashAttentionImpl(AttentionImpl):
234 cu_query_lens=attn_metadata.query_start_loc,
235 max_query_len=attn_metadata.max_query_len,
236 cu_prefix_query_lens=attn_metadata.cu_prefix_query_lens,
219 - cu_prefix_kv_lens=attn_metadata.cu_prefix_kv_lens,
220 - cu_suffix_kv_lens=attn_metadata.cu_suffix_kv_lens,
237 + prefix_kv_lens=attn_metadata.prefix_kv_lens,
238 + suffix_kv_lens=attn_metadata.suffix_kv_lens,
239 max_kv_len=attn_metadata.max_seq_len,
240 softmax_scale=self.scale,
241 alibi_slopes=self.alibi_slopes,
@@ -225,6 +243,7 @@ class FlashAttentionImpl(AttentionImpl):
243 logits_soft_cap=self.logits_soft_cap,
244 block_table=attn_metadata.block_table,
245 common_prefix_len=attn_metadata.common_prefix_len,
246 + fa_version=self.fa_version,
247 )
248 return output
249
@@ -305,8 +324,8 @@ def cascade_attention(
324 cu_query_lens: torch.Tensor,
325 max_query_len: int,
326 cu_prefix_query_lens: torch.Tensor,
308 - cu_prefix_kv_lens: torch.Tensor,
309 - cu_suffix_kv_lens: torch.Tensor,
327 + prefix_kv_lens: torch.Tensor,
328 + suffix_kv_lens: torch.Tensor,
329 max_kv_len: int,
330 softmax_scale: float,
331 alibi_slopes: Optional[torch.Tensor],
@@ -314,6 +333,7 @@ def cascade_attention(
333 logits_soft_cap: float,
334 block_table: torch.Tensor,
335 common_prefix_len: int,
336 + fa_version: int,
337 ) -> torch.Tensor:
338 assert alibi_slopes is None, ("Cascade attention does not support ALiBi.")
339 # TODO: Support sliding window.
@@ -332,7 +352,7 @@ def cascade_attention(
352 k=key_cache,
353 v=value_cache,
354 cu_seqlens_q=cu_prefix_query_lens,
335 - cu_seqlens_k=cu_prefix_kv_lens,
355 + seqused_k=prefix_kv_lens,
356 max_seqlen_q=num_tokens,
357 max_seqlen_k=common_prefix_len,
358 softmax_scale=softmax_scale,
@@ -341,6 +361,7 @@ def cascade_attention(
361 block_table=block_table[:1],
362 softcap=logits_soft_cap,
363 return_softmax_lse=True,
364 + fa_version=fa_version,
365 )
366
367 # Process suffix per query.
@@ -349,7 +370,7 @@ def cascade_attention(
370 k=key_cache,
371 v=value_cache,
372 cu_seqlens_q=cu_query_lens,
352 - cu_seqlens_k=cu_suffix_kv_lens,
373 + seqused_k=suffix_kv_lens,
374 max_seqlen_q=max_query_len,
375 max_seqlen_k=max_kv_len - common_prefix_len,
376 softmax_scale=softmax_scale,
@@ -358,6 +379,7 @@ def cascade_attention(
379 block_table=block_table[:, num_common_kv_blocks:],
380 softcap=logits_soft_cap,
381 return_softmax_lse=True,
382 + fa_version=fa_version,
383 )
384
385 # Merge prefix and suffix outputs, and store the result in output.
vllm/v1/worker/gpu_model_runner.py
+21 -25
@@ -199,11 +199,11 @@ class GPUModelRunner:
199 device="cpu",
200 pin_memory=self.pin_memory)
201 self.query_start_loc_np = self.query_start_loc_cpu.numpy()
202 - self.seq_start_loc_cpu = torch.zeros(self.max_num_reqs + 1,
203 - dtype=torch.int32,
204 - device="cpu",
205 - pin_memory=self.pin_memory)
206 - self.seq_start_loc_np = self.seq_start_loc_cpu.numpy()
202 + self.seq_lens_cpu = torch.zeros(self.max_num_reqs,
203 + dtype=torch.int32,
204 + device="cpu",
205 + pin_memory=self.pin_memory)
206 + self.seq_lens_np = self.seq_lens_cpu.numpy()
207
208 def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
209 # Remove stopped requests from the cached states.
@@ -412,11 +412,10 @@ class GPUModelRunner:
412 np.cumsum(num_scheduled_tokens,
413 out=self.query_start_loc_np[1:num_reqs + 1])
414
415 - seq_lens = (self.input_batch.num_computed_tokens_cpu[:num_reqs] +
416 - num_scheduled_tokens)
417 - max_seq_len = seq_lens.max()
418 - self.seq_start_loc_np[0] = 0
419 - np.cumsum(seq_lens, out=self.seq_start_loc_np[1:num_reqs + 1])
415 + self.seq_lens_np[:num_reqs] = (
416 + self.input_batch.num_computed_tokens_cpu[:num_reqs] +
417 + num_scheduled_tokens)
418 + max_seq_len = self.seq_lens_np[:num_reqs].max()
419
420 # Copy the tensors to the GPU.
421 self.input_ids[:total_num_scheduled_tokens].copy_(
@@ -433,8 +432,8 @@ class GPUModelRunner:
432 non_blocking=True)
433 query_start_loc = self.query_start_loc_cpu[:num_reqs + 1].to(
434 self.device, non_blocking=True)
436 - seq_start_loc = self.seq_start_loc_cpu[:num_reqs + 1].to(
437 - self.device, non_blocking=True)
435 + seq_lens = self.seq_lens_cpu[:num_reqs].to(self.device,
436 + non_blocking=True)
437 slot_mapping = self.slot_mapping_cpu[:total_num_scheduled_tokens].to(
438 self.device, non_blocking=True).long()
439
@@ -506,33 +505,30 @@ class GPUModelRunner:
505 [0, total_num_scheduled_tokens],
506 dtype=torch.int32,
507 device=self.device)
509 - cu_prefix_kv_lens = torch.tensor([0, common_prefix_len],
510 - dtype=torch.int32,
511 - device=self.device)
512 - cu_suffix_kv_lens = (
513 - self.seq_start_loc_np[:num_reqs + 1] -
514 - self.arange_np[:num_reqs + 1] * common_prefix_len)
515 - cu_suffix_kv_lens = torch.from_numpy(cu_suffix_kv_lens).to(
516 - self.device)
508 + prefix_kv_lens = torch.tensor([common_prefix_len],
509 + dtype=torch.int32,
510 + device=self.device)
511 + suffix_kv_lens = (self.seq_lens_np[:num_reqs] - common_prefix_len)
512 + suffix_kv_lens = torch.from_numpy(suffix_kv_lens).to(self.device)
513 else:
514 cu_prefix_query_lens = None
519 - cu_prefix_kv_lens = None
520 - cu_suffix_kv_lens = None
515 + prefix_kv_lens = None
516 + suffix_kv_lens = None
517
518 attn_metadata = FlashAttentionMetadata(
519 num_actual_tokens=total_num_scheduled_tokens,
520 max_query_len=max_num_scheduled_tokens,
521 query_start_loc=query_start_loc,
522 max_seq_len=max_seq_len,
527 - seq_start_loc=seq_start_loc,
523 + seq_lens=seq_lens,
524 block_table=(
525 self.input_batch.block_table.get_device_tensor()[:num_reqs]),
526 slot_mapping=slot_mapping,
527 use_cascade=use_cascade,
528 common_prefix_len=common_prefix_len,
529 cu_prefix_query_lens=cu_prefix_query_lens,
534 - cu_prefix_kv_lens=cu_prefix_kv_lens,
535 - cu_suffix_kv_lens=cu_suffix_kv_lens,
530 + prefix_kv_lens=prefix_kv_lens,
531 + suffix_kv_lens=suffix_kv_lens,
532 )
533 # NOTE(woosuk): Due to chunked prefills, there can be at most 1 partial
534 # request in the batch. While we should not sample any token from this