A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
Revert "[Core] Remove unnecessary copies in flash attn backend" (#5478)
Antoni Baum committed
Jun 13, 2024 at 11:22 UTC
6b0511a57bdba85efe2b4d5588dd16280c8fdc78
1 file changed
+6
-7
vllm/attention/backends/flash_attn.py
+6
-7
@@ -317,7 +317,7 @@ class FlashAttentionImpl(AttentionImpl):
317
# normal attention
318
# When block_tables are not filled, it means q and k are the
319
# prompt, and they have the same length.
320
- flash_attn_varlen_func(
320
+ out = flash_attn_varlen_func(
321
q=query,
322
k=key,
323
v=value,
@@ -329,13 +329,14 @@ class FlashAttentionImpl(AttentionImpl):
329
causal=True,
330
window_size=self.sliding_window,
331
alibi_slopes=self.alibi_slopes,
332
- out=output[:num_prefill_tokens],
332
)
333
+ assert output[:num_prefill_tokens].shape == out.shape
334
+ output[:num_prefill_tokens] = out
335
else:
336
# prefix-enabled attention
337
assert prefill_meta.seq_lens is not None
338
max_seq_len = max(prefill_meta.seq_lens)
338
- flash_attn_varlen_func(
339
+ output[:num_prefill_tokens] = flash_attn_varlen_func(
340
q=query,
341
k=key_cache,
342
v=value_cache,
@@ -347,12 +348,11 @@ class FlashAttentionImpl(AttentionImpl):
348
causal=True,
349
alibi_slopes=self.alibi_slopes,
350
block_table=prefill_meta.block_tables,
350
- out=output[:num_prefill_tokens],
351
)
352
353
if decode_meta := attn_metadata.decode_metadata:
354
# Decoding run.
355
- flash_attn_with_kvcache(
355
+ output[num_prefill_tokens:] = flash_attn_with_kvcache(
356
decode_query.unsqueeze(1),
357
key_cache,
358
value_cache,
@@ -361,8 +361,7 @@ class FlashAttentionImpl(AttentionImpl):
361
softmax_scale=self.scale,
362
causal=True,
363
alibi_slopes=self.alibi_slopes,
364
- out=output[num_prefill_tokens:].unsqueeze(1),
365
- )
364
+ ).squeeze(1)
365
366
# Reshape the output tensor.
367
return output.view(num_tokens, hidden_size)