A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Optimize the CPU overheads in FlashAttention custom op (#10733)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 28, 2024 at 09:01 UTC
98f47f2a4032f8c395268de80858c64ffcfc60fa
1 file changed
+9
-8
vllm/v1/attention/backends/flash_attn.py
+9
-8
@@ -135,6 +135,13 @@ class FlashAttentionImpl(AttentionImpl):
135
assert k_scale == 1.0 and v_scale == 1.0, (
136
"key/v_scale is not supported in FlashAttention.")
137
138
+ # Reshape the query, key, and value tensors.
139
+ # NOTE(woosuk): We do this outside the custom op to minimize the CPU
140
+ # overheads from the non-CUDA-graph regions.
141
+ query = query.view(-1, self.num_heads, self.head_size)
142
+ key = key.view(-1, self.num_kv_heads, self.head_size)
143
+ value = value.view(-1, self.num_kv_heads, self.head_size)
144
+
145
output = torch.empty_like(query)
146
torch.ops.vllm.unified_v1_flash_attention(
147
output,
@@ -153,7 +160,7 @@ class FlashAttentionImpl(AttentionImpl):
160
self.alibi_slopes,
161
self.logits_soft_cap,
162
)
156
- return output
163
+ return output.view(-1, self.num_heads * self.head_size)
164
165
166
def unified_v1_flash_attention(
@@ -184,11 +191,6 @@ def unified_v1_flash_attention(
191
attn_metadata: FlashAttentionMetadata = current_metadata
192
num_actual_tokens = attn_metadata.num_actual_tokens
193
187
- # Reshape the query, key, and value tensors.
188
- query = query.view(-1, num_heads, head_size)
189
- key = key.view(-1, num_kv_heads, head_size)
190
- value = value.view(-1, num_kv_heads, head_size)
191
-
194
# Reshape the input keys and values and store them in the cache.
195
key_cache = kv_cache[0]
196
value_cache = kv_cache[1]
@@ -218,8 +220,7 @@ def unified_v1_flash_attention(
220
block_table=attn_metadata.block_table,
221
softcap=logits_soft_cap,
222
)
221
- attn_output = attn_output.view(num_actual_tokens, -1)
222
- # TODO(woosuk): Optimize this.
223
+ # TODO(woosuk): Remove this unnecessary copy.
224
output[:num_actual_tokens].copy_(attn_output)
225
226