A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Kernel] Use `out` arg in flash_attn_varlen_func (#10811)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 1, 2024 at 17:55 UTC
073a4bd1c04164af29843cb5478740e9839d2d8a
3 files changed
+21
-7
CMakeLists.txt
+1
-1
@@ -522,7 +522,7 @@ else()
522
FetchContent_Declare(
523
vllm-flash-attn
524
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
525
- GIT_TAG fdf6d72b48aea41f4ae6a89139a453dae554abc8
525
+ GIT_TAG 04325b6798bcc326c86fb35af62d05a9c8c8eceb
526
GIT_PROGRESS TRUE
527
# Don't share the vllm-flash-attn build between build types
528
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
tests/kernels/test_flash_attn.py
+17
-3
@@ -71,6 +71,7 @@ def ref_paged_attn(
71
return torch.cat(outputs, dim=0)
72
73
74
+@pytest.mark.parametrize("use_out", [True, False])
75
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
76
@pytest.mark.parametrize("num_heads", NUM_HEADS)
77
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -81,6 +82,7 @@ def ref_paged_attn(
82
@pytest.mark.parametrize("sliding_window", [None, 256])
83
@torch.inference_mode()
84
def test_flash_attn_with_paged_kv(
85
+ use_out: bool,
86
kv_lens: List[int],
87
num_heads: Tuple[int, int],
88
head_size: int,
@@ -116,17 +118,22 @@ def test_flash_attn_with_paged_kv(
118
(num_seqs, max_num_blocks_per_seq),
119
dtype=torch.int32)
120
121
+ q = query.unsqueeze(1)
122
+ out = torch.empty_like(q) if use_out else None
123
output = flash_attn_with_kvcache(
120
- q=query.unsqueeze(1),
124
+ q=q,
125
k_cache=key_cache,
126
v_cache=value_cache,
127
+ out=out,
128
softmax_scale=scale,
129
causal=True,
130
block_table=block_tables,
131
cache_seqlens=kv_lens_tensor,
132
softcap=soft_cap if soft_cap is not None else 0,
133
window_size=window_size,
129
- ).squeeze(1)
134
+ )
135
+ output = output if not use_out else out
136
+ output = output.squeeze(1)
137
138
ref_output = ref_paged_attn(query=query,
139
key_cache=key_cache,
@@ -141,7 +148,10 @@ def test_flash_attn_with_paged_kv(
148
f"{torch.max(torch.abs(output - ref_output))}"
149
150
144
-@pytest.mark.parametrize("seq_lens", [[(1, 1328), (5, 18), (129, 463)]])
151
+@pytest.mark.parametrize("use_out", [True, False])
152
+@pytest.mark.parametrize("seq_lens",
153
+ [[(1, 1328), (5, 18),
154
+ (129, 463)], [(1, 523), (1, 37), (1, 2011)]])
155
@pytest.mark.parametrize("num_heads", NUM_HEADS)
156
@pytest.mark.parametrize("head_size", HEAD_SIZES)
157
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@@ -151,6 +161,7 @@ def test_flash_attn_with_paged_kv(
161
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
162
@torch.inference_mode()
163
def test_varlen_with_paged_kv(
164
+ use_out: bool,
165
seq_lens: List[Tuple[int, int]],
166
num_heads: Tuple[int, int],
167
head_size: int,
@@ -197,10 +208,12 @@ def test_varlen_with_paged_kv(
208
(num_seqs, max_num_blocks_per_seq),
209
dtype=torch.int32)
210
211
+ out = torch.empty_like(query) if use_out else None
212
output = flash_attn_varlen_func(
213
q=query,
214
k=key_cache,
215
v=value_cache,
216
+ out=out,
217
cu_seqlens_q=cu_query_lens,
218
cu_seqlens_k=cu_kv_lens,
219
max_seqlen_q=max_query_len,
@@ -211,6 +224,7 @@ def test_varlen_with_paged_kv(
224
block_table=block_tables,
225
softcap=soft_cap if soft_cap is not None else 0,
226
)
227
+ output = output if not use_out else out
228
229
ref_output = ref_paged_attn(
230
query=query,
vllm/v1/attention/backends/flash_attn.py
+3
-3
@@ -205,10 +205,12 @@ def unified_v1_flash_attention(
205
v_scale,
206
)
207
208
- attn_output = flash_attn_varlen_func(
208
+ # Compute attention and update output up to `num_actual_tokens`.
209
+ flash_attn_varlen_func(
210
q=query[:num_actual_tokens],
211
k=key_cache,
212
v=value_cache,
213
+ out=output[:num_actual_tokens],
214
cu_seqlens_q=attn_metadata.query_start_loc,
215
max_seqlen_q=attn_metadata.max_query_len,
216
cu_seqlens_k=attn_metadata.seq_start_loc,
@@ -220,8 +222,6 @@ def unified_v1_flash_attention(
222
block_table=attn_metadata.block_table,
223
softcap=logits_soft_cap,
224
)
223
- # TODO(woosuk): Remove this unnecessary copy.
224
- output[:num_actual_tokens].copy_(attn_output)
225
226
227
def unified_v1_flash_attention_fake(