A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Remove unnecessary copies in flash attn backend (#5138)
Antoni Baum committed
Jun 3, 2024 at 09:39 UTC
0ab278ca31028a7623098b3c7d615ad350663d05
2 files changed
+8
-7
requirements-cuda.txt
+1
-1
@@ -6,4 +6,4 @@ ray >= 2.9
6
nvidia-ml-py # for pynvml package
7
torch == 2.3.0
8
xformers == 0.0.26.post1 # Requires PyTorch 2.3.0
9
-vllm-flash-attn == 2.5.8.post2 # Requires PyTorch 2.3.0
9
+vllm-flash-attn == 2.5.9 # Requires PyTorch 2.3.0
vllm/attention/backends/flash_attn.py
+7
-6
@@ -317,7 +317,7 @@ class FlashAttentionImpl(AttentionImpl):
317
# normal attention
318
# When block_tables are not filled, it means q and k are the
319
# prompt, and they have the same length.
320
- out = flash_attn_varlen_func(
320
+ flash_attn_varlen_func(
321
q=query,
322
k=key,
323
v=value,
@@ -329,14 +329,13 @@ class FlashAttentionImpl(AttentionImpl):
329
causal=True,
330
window_size=self.sliding_window,
331
alibi_slopes=self.alibi_slopes,
332
+ out=output[:num_prefill_tokens],
333
)
333
- assert output[:num_prefill_tokens].shape == out.shape
334
- output[:num_prefill_tokens] = out
334
else:
335
# prefix-enabled attention
336
assert prefill_meta.seq_lens is not None
337
max_seq_len = max(prefill_meta.seq_lens)
339
- output[:num_prefill_tokens] = flash_attn_varlen_func(
338
+ flash_attn_varlen_func(
339
q=query,
340
k=key_cache,
341
v=value_cache,
@@ -348,11 +347,12 @@ class FlashAttentionImpl(AttentionImpl):
347
causal=True,
348
alibi_slopes=self.alibi_slopes,
349
block_table=prefill_meta.block_tables,
350
+ out=output[:num_prefill_tokens],
351
)
352
353
if decode_meta := attn_metadata.decode_metadata:
354
# Decoding run.
355
- output[num_prefill_tokens:] = flash_attn_with_kvcache(
355
+ flash_attn_with_kvcache(
356
decode_query.unsqueeze(1),
357
key_cache,
358
value_cache,
@@ -361,7 +361,8 @@ class FlashAttentionImpl(AttentionImpl):
361
softmax_scale=self.scale,
362
causal=True,
363
alibi_slopes=self.alibi_slopes,
364
- ).squeeze(1)
364
+ out=output[num_prefill_tokens:].unsqueeze(1),
365
+ )
366
367
# Reshape the output tensor.
368
return output.view(num_tokens, hidden_size)