@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 073a4bd1c0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Kernel] Use `out` arg in flash_attn_varlen_func (#10811)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 1, 2024 at 17:55 UTC 073a4bd1c04164af29843cb5478740e9839d2d8a
3 files changed +21 -7
CMakeLists.txt
+1 -1
@@ -522,7 +522,7 @@ else()
522 FetchContent_Declare(
523 vllm-flash-attn
524 GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
525 - GIT_TAG fdf6d72b48aea41f4ae6a89139a453dae554abc8
525 + GIT_TAG 04325b6798bcc326c86fb35af62d05a9c8c8eceb
526 GIT_PROGRESS TRUE
527 # Don't share the vllm-flash-attn build between build types
528 BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
tests/kernels/test_flash_attn.py
+17 -3
@@ -71,6 +71,7 @@ def ref_paged_attn(
71 return torch.cat(outputs, dim=0)
72
73
74 +@pytest.mark.parametrize("use_out", [True, False])
75 @pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
76 @pytest.mark.parametrize("num_heads", NUM_HEADS)
77 @pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -81,6 +82,7 @@ def ref_paged_attn(
82 @pytest.mark.parametrize("sliding_window", [None, 256])
83 @torch.inference_mode()
84 def test_flash_attn_with_paged_kv(
85 + use_out: bool,
86 kv_lens: List[int],
87 num_heads: Tuple[int, int],
88 head_size: int,
@@ -116,17 +118,22 @@ def test_flash_attn_with_paged_kv(
118 (num_seqs, max_num_blocks_per_seq),
119 dtype=torch.int32)
120
121 + q = query.unsqueeze(1)
122 + out = torch.empty_like(q) if use_out else None
123 output = flash_attn_with_kvcache(
120 - q=query.unsqueeze(1),
124 + q=q,
125 k_cache=key_cache,
126 v_cache=value_cache,
127 + out=out,
128 softmax_scale=scale,
129 causal=True,
130 block_table=block_tables,
131 cache_seqlens=kv_lens_tensor,
132 softcap=soft_cap if soft_cap is not None else 0,
133 window_size=window_size,
129 - ).squeeze(1)
134 + )
135 + output = output if not use_out else out
136 + output = output.squeeze(1)
137
138 ref_output = ref_paged_attn(query=query,
139 key_cache=key_cache,
@@ -141,7 +148,10 @@ def test_flash_attn_with_paged_kv(
148 f"{torch.max(torch.abs(output - ref_output))}"
149
150
144 -@pytest.mark.parametrize("seq_lens", [[(1, 1328), (5, 18), (129, 463)]])
151 +@pytest.mark.parametrize("use_out", [True, False])
152 +@pytest.mark.parametrize("seq_lens",
153 + [[(1, 1328), (5, 18),
154 + (129, 463)], [(1, 523), (1, 37), (1, 2011)]])
155 @pytest.mark.parametrize("num_heads", NUM_HEADS)
156 @pytest.mark.parametrize("head_size", HEAD_SIZES)
157 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
@@ -151,6 +161,7 @@ def test_flash_attn_with_paged_kv(
161 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
162 @torch.inference_mode()
163 def test_varlen_with_paged_kv(
164 + use_out: bool,
165 seq_lens: List[Tuple[int, int]],
166 num_heads: Tuple[int, int],
167 head_size: int,
@@ -197,10 +208,12 @@ def test_varlen_with_paged_kv(
208 (num_seqs, max_num_blocks_per_seq),
209 dtype=torch.int32)
210
211 + out = torch.empty_like(query) if use_out else None
212 output = flash_attn_varlen_func(
213 q=query,
214 k=key_cache,
215 v=value_cache,
216 + out=out,
217 cu_seqlens_q=cu_query_lens,
218 cu_seqlens_k=cu_kv_lens,
219 max_seqlen_q=max_query_len,
@@ -211,6 +224,7 @@ def test_varlen_with_paged_kv(
224 block_table=block_tables,
225 softcap=soft_cap if soft_cap is not None else 0,
226 )
227 + output = output if not use_out else out
228
229 ref_output = ref_paged_attn(
230 query=query,
vllm/v1/attention/backends/flash_attn.py
+3 -3
@@ -205,10 +205,12 @@ def unified_v1_flash_attention(
205 v_scale,
206 )
207
208 - attn_output = flash_attn_varlen_func(
208 + # Compute attention and update output up to `num_actual_tokens`.
209 + flash_attn_varlen_func(
210 q=query[:num_actual_tokens],
211 k=key_cache,
212 v=value_cache,
213 + out=output[:num_actual_tokens],
214 cu_seqlens_q=attn_metadata.query_start_loc,
215 max_seqlen_q=attn_metadata.max_query_len,
216 cu_seqlens_k=attn_metadata.seq_start_loc,
@@ -220,8 +222,6 @@ def unified_v1_flash_attention(
222 block_table=attn_metadata.block_table,
223 softcap=logits_soft_cap,
224 )
223 - # TODO(woosuk): Remove this unnecessary copy.
224 - output[:num_actual_tokens].copy_(attn_output)
225
226
227 def unified_v1_flash_attention_fake(