@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ab278ca31
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Remove unnecessary copies in flash attn backend (#5138)

Antoni Baum committed Jun 3, 2024 at 09:39 UTC 0ab278ca31028a7623098b3c7d615ad350663d05
2 files changed +8 -7
requirements-cuda.txt
+1 -1
@@ -6,4 +6,4 @@ ray >= 2.9
6 nvidia-ml-py # for pynvml package
7 torch == 2.3.0
8 xformers == 0.0.26.post1 # Requires PyTorch 2.3.0
9 -vllm-flash-attn == 2.5.8.post2 # Requires PyTorch 2.3.0
9 +vllm-flash-attn == 2.5.9 # Requires PyTorch 2.3.0
vllm/attention/backends/flash_attn.py
+7 -6
@@ -317,7 +317,7 @@ class FlashAttentionImpl(AttentionImpl):
317 # normal attention
318 # When block_tables are not filled, it means q and k are the
319 # prompt, and they have the same length.
320 - out = flash_attn_varlen_func(
320 + flash_attn_varlen_func(
321 q=query,
322 k=key,
323 v=value,
@@ -329,14 +329,13 @@ class FlashAttentionImpl(AttentionImpl):
329 causal=True,
330 window_size=self.sliding_window,
331 alibi_slopes=self.alibi_slopes,
332 + out=output[:num_prefill_tokens],
333 )
333 - assert output[:num_prefill_tokens].shape == out.shape
334 - output[:num_prefill_tokens] = out
334 else:
335 # prefix-enabled attention
336 assert prefill_meta.seq_lens is not None
337 max_seq_len = max(prefill_meta.seq_lens)
339 - output[:num_prefill_tokens] = flash_attn_varlen_func(
338 + flash_attn_varlen_func(
339 q=query,
340 k=key_cache,
341 v=value_cache,
@@ -348,11 +347,12 @@ class FlashAttentionImpl(AttentionImpl):
347 causal=True,
348 alibi_slopes=self.alibi_slopes,
349 block_table=prefill_meta.block_tables,
350 + out=output[:num_prefill_tokens],
351 )
352
353 if decode_meta := attn_metadata.decode_metadata:
354 # Decoding run.
355 - output[num_prefill_tokens:] = flash_attn_with_kvcache(
355 + flash_attn_with_kvcache(
356 decode_query.unsqueeze(1),
357 key_cache,
358 value_cache,
@@ -361,7 +361,8 @@ class FlashAttentionImpl(AttentionImpl):
361 softmax_scale=self.scale,
362 causal=True,
363 alibi_slopes=self.alibi_slopes,
364 - ).squeeze(1)
364 + out=output[num_prefill_tokens:].unsqueeze(1),
365 + )
366
367 # Reshape the output tensor.
368 return output.view(num_tokens, hidden_size)