@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6b0511a57b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

Revert "[Core] Remove unnecessary copies in flash attn backend" (#5478)

Antoni Baum committed Jun 13, 2024 at 11:22 UTC 6b0511a57bdba85efe2b4d5588dd16280c8fdc78
1 file changed +6 -7
vllm/attention/backends/flash_attn.py
+6 -7
@@ -317,7 +317,7 @@ class FlashAttentionImpl(AttentionImpl):
317 # normal attention
318 # When block_tables are not filled, it means q and k are the
319 # prompt, and they have the same length.
320 - flash_attn_varlen_func(
320 + out = flash_attn_varlen_func(
321 q=query,
322 k=key,
323 v=value,
@@ -329,13 +329,14 @@ class FlashAttentionImpl(AttentionImpl):
329 causal=True,
330 window_size=self.sliding_window,
331 alibi_slopes=self.alibi_slopes,
332 - out=output[:num_prefill_tokens],
332 )
333 + assert output[:num_prefill_tokens].shape == out.shape
334 + output[:num_prefill_tokens] = out
335 else:
336 # prefix-enabled attention
337 assert prefill_meta.seq_lens is not None
338 max_seq_len = max(prefill_meta.seq_lens)
338 - flash_attn_varlen_func(
339 + output[:num_prefill_tokens] = flash_attn_varlen_func(
340 q=query,
341 k=key_cache,
342 v=value_cache,
@@ -347,12 +348,11 @@ class FlashAttentionImpl(AttentionImpl):
348 causal=True,
349 alibi_slopes=self.alibi_slopes,
350 block_table=prefill_meta.block_tables,
350 - out=output[:num_prefill_tokens],
351 )
352
353 if decode_meta := attn_metadata.decode_metadata:
354 # Decoding run.
355 - flash_attn_with_kvcache(
355 + output[num_prefill_tokens:] = flash_attn_with_kvcache(
356 decode_query.unsqueeze(1),
357 key_cache,
358 value_cache,
@@ -361,8 +361,7 @@ class FlashAttentionImpl(AttentionImpl):
361 softmax_scale=self.scale,
362 causal=True,
363 alibi_slopes=self.alibi_slopes,
364 - out=output[num_prefill_tokens:].unsqueeze(1),
365 - )
364 + ).squeeze(1)
365
366 # Reshape the output tensor.
367 return output.view(num_tokens, hidden_size)