@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a4c4daf364
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[misc] use out argument for flash attention (#10822)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 2, 2024 at 02:50 UTC a4c4daf3642ae2629608d5181487739b044fabe8
13 files changed +144 -157
vllm/attention/backends/abstract.py
+1
@@ -247,5 +247,6 @@ class AttentionImpl(ABC, Generic[T]):
247 k_scale: float = 1.0,
248 v_scale: float = 1.0,
249 attn_type: str = AttentionType.DECODER,
250 + output: Optional[torch.Tensor] = None,
251 ) -> torch.Tensor:
252 raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+2
@@ -360,6 +360,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
360 k_scale: float = 1.0,
361 v_scale: float = 1.0,
362 attn_type: str = AttentionType.DECODER,
363 + output: Optional[torch.Tensor] = None,
364 ) -> torch.Tensor:
365 """Forward pass with FlashAttention and PagedAttention.
366
@@ -448,5 +449,6 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
449 blocksparse_head_sliding_step=self.head_sliding_step,
450 )
451
452 + assert output is not None
453 # Reshape the output tensor.
454 return output.view(num_tokens, hidden_size)
vllm/attention/backends/flash_attn.py
+19 -36
@@ -638,24 +638,27 @@ class FlashAttentionImpl(AttentionImpl):
638 k_scale: float = 1.0,
639 v_scale: float = 1.0,
640 attn_type: str = AttentionType.DECODER,
641 + output: Optional[torch.Tensor] = None,
642 ) -> torch.Tensor:
643 """Forward pass with FlashAttention.
644
645 Args:
645 - query: shape = [num_tokens, num_heads * head_size]
646 - key: shape = [num_tokens, num_kv_heads * head_size]
647 - value: shape = [num_tokens, num_kv_heads * head_size]
646 + query: shape = [num_tokens, num_heads, head_size]
647 + key: shape = [num_tokens, num_kv_heads, head_size]
648 + value: shape = [num_tokens, num_kv_heads, head_size]
649 + output: shape = [num_tokens, num_heads, head_size]
650 kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
651 NOTE: kv_cache will be an empty tensor with shape [0]
652 for profiling run.
653 attn_metadata: Metadata for attention.
652 - Returns:
653 - shape = [num_tokens, num_heads * head_size]
654 + NOTE: It in-place updates the output tensor.
655 """
656 # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
657 assert k_scale == 1.0 and v_scale == 1.0, (
658 "key/v_scale is not supported in FlashAttention.")
659
660 + assert output is not None, "Output tensor must be provided."
661 +
662 if (attn_type == AttentionType.ENCODER
663 and (not attn_metadata.is_all_encoder_attn_metadata_set)):
664 raise AttributeError("Encoder attention requires setting "
@@ -666,23 +669,12 @@ class FlashAttentionImpl(AttentionImpl):
669 "requires setting cross-attention "
670 "metadata attributes.")
671
669 - num_heads: int = self.num_heads
670 - head_size: int = self.head_size
671 - num_kv_heads: int = self.num_kv_heads
672 kv_cache_dtype: str = self.kv_cache_dtype
673 softmax_scale: float = self.scale
674 window_size = self.sliding_window
675 alibi_slopes: Optional[torch.Tensor] = self.alibi_slopes
676 logits_soft_cap: Optional[float] = self.logits_soft_cap
677
678 - num_tokens, hidden_size = query.shape
679 -
680 - # Reshape the query, key, and value tensors.
681 - query = query.view(-1, num_heads, head_size)
682 - if (key is not None) and (value is not None):
683 - key = key.view(-1, num_kv_heads, head_size)
684 - value = value.view(-1, num_kv_heads, head_size)
685 -
678 if kv_cache.numel() > 0:
679 key_cache = kv_cache[0]
680 value_cache = kv_cache[1]
@@ -721,13 +713,13 @@ class FlashAttentionImpl(AttentionImpl):
713 num_decode_query_tokens) = \
714 get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
715 decode_query = query[num_prefill_query_tokens:]
716 + decode_output = output[num_prefill_query_tokens:]
717 # QKV for prefill.
718 query = query[:num_prefill_query_tokens]
719 + prefill_output = output[:num_prefill_query_tokens]
720 assert query.shape[0] == num_prefill_query_tokens
721 assert decode_query.shape[0] == num_decode_query_tokens
722
729 - prefill_output: Optional[torch.Tensor] = None
730 - decode_output: Optional[torch.Tensor] = None
723 if prefill_meta := attn_metadata.prefill_metadata:
724 # Prompt run.
725 if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
@@ -741,7 +733,7 @@ class FlashAttentionImpl(AttentionImpl):
733 key = key[:num_prefill_kv_tokens]
734 value = value[:num_prefill_kv_tokens]
735
744 - prefill_output = flash_attn_varlen_func(
736 + flash_attn_varlen_func(
737 q=query,
738 k=key,
739 v=value,
@@ -754,6 +746,7 @@ class FlashAttentionImpl(AttentionImpl):
746 window_size=window_size,
747 alibi_slopes=alibi_slopes,
748 softcap=logits_soft_cap,
749 + out=prefill_output,
750 )
751 else:
752 # prefix-enabled attention
@@ -761,7 +754,7 @@ class FlashAttentionImpl(AttentionImpl):
754 "Only decoder-only models support prefix caching")
755 assert prefill_meta.seq_lens is not None
756 max_seq_len = max(prefill_meta.seq_lens)
764 - prefill_output = flash_attn_varlen_func( # noqa
757 + flash_attn_varlen_func( # noqa
758 q=query,
759 k=key_cache,
760 v=value_cache,
@@ -775,6 +768,7 @@ class FlashAttentionImpl(AttentionImpl):
768 alibi_slopes=alibi_slopes,
769 block_table=prefill_meta.block_tables,
770 softcap=logits_soft_cap,
771 + out=prefill_output,
772 )
773
774 if decode_meta := attn_metadata.decode_metadata:
@@ -788,7 +782,7 @@ class FlashAttentionImpl(AttentionImpl):
782 assert attn_type == AttentionType.DECODER, (
783 "Only decoder-only models support max_decode_query_len > 1"
784 )
791 - decode_output = flash_attn_varlen_func(
785 + flash_attn_varlen_func(
786 q=decode_query,
787 k=key_cache,
788 v=value_cache,
@@ -802,6 +796,7 @@ class FlashAttentionImpl(AttentionImpl):
796 alibi_slopes=alibi_slopes,
797 softcap=logits_soft_cap,
798 block_table=decode_meta.block_tables,
799 + out=decode_output,
800 )
801 else:
802 # Use flash_attn_with_kvcache for normal decoding.
@@ -810,7 +805,7 @@ class FlashAttentionImpl(AttentionImpl):
805 _,
806 block_tables_arg,
807 ) = get_seq_len_block_table_args(decode_meta, False, attn_type)
813 - decode_output = flash_attn_with_kvcache(
808 + flash_attn_with_kvcache(
809 q=decode_query.unsqueeze(1),
810 k_cache=key_cache,
811 v_cache=value_cache,
@@ -821,20 +816,8 @@ class FlashAttentionImpl(AttentionImpl):
816 window_size=window_size,
817 alibi_slopes=alibi_slopes,
818 softcap=logits_soft_cap,
824 - ).squeeze(1)
825 -
826 - if prefill_output is None:
827 - assert decode_output is not None
828 - return decode_output.view(num_decode_query_tokens, hidden_size)
829 - if decode_output is None:
830 - assert prefill_output is not None
831 - return prefill_output.view(num_prefill_query_tokens, hidden_size)
832 -
833 - assert decode_meta is not None
834 - decode_output = decode_output.squeeze(1)
835 - output = torch.cat([prefill_output, decode_output], dim=0)
836 - return output.view(num_tokens, hidden_size)
837 -
819 + out=decode_output.unsqueeze(1),
820 + )
821 return output
822
823
vllm/attention/backends/flashinfer.py
+4
@@ -774,7 +774,11 @@ class FlashInferImpl(AttentionImpl):
774 k_scale: float = 1.0,
775 v_scale: float = 1.0,
776 attn_type: str = AttentionType.DECODER,
777 + output: Optional[torch.Tensor] = None,
778 ) -> torch.Tensor:
779 +
780 + # TODO: directly write to output tensor
781 +
782 if attn_type != AttentionType.DECODER:
783 raise NotImplementedError("Encoder self-attention and "
784 "encoder/decoder cross-attention "
vllm/attention/backends/hpu_attn.py
+1
@@ -145,6 +145,7 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
145 k_scale: float = 1.0,
146 v_scale: float = 1.0,
147 attn_type: str = AttentionType.DECODER,
148 + output: Optional[torch.Tensor] = None,
149 ) -> torch.Tensor:
150 """Forward pass with xFormers and PagedAttention.
151
vllm/attention/backends/ipex_attn.py
+1
@@ -173,6 +173,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
173 k_scale: float = 1.0,
174 v_scale: float = 1.0,
175 attn_type: str = AttentionType.DECODER,
176 + output: Optional[torch.Tensor] = None,
177 ) -> torch.Tensor:
178 """Forward pass with IPEX varlen_attention and PagedAttention.
179
vllm/attention/backends/pallas.py
+1
@@ -151,6 +151,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
151 k_scale: float = 1.0,
152 v_scale: float = 1.0,
153 attn_type: str = AttentionType.DECODER,
154 + output: Optional[torch.Tensor] = None,
155 ) -> torch.Tensor:
156 """Forward pass with Pallas attention.
157
vllm/attention/backends/rocm_flash_attn.py
+1
@@ -415,6 +415,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
415 k_scale: float = 1.0,
416 v_scale: float = 1.0,
417 attn_type: str = AttentionType.DECODER,
418 + output: Optional[torch.Tensor] = None,
419 ) -> torch.Tensor:
420 """Forward pass with FlashAttention and PagedAttention.
421
vllm/attention/backends/torch_sdpa.py
+1
@@ -431,6 +431,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
431 k_scale: float = 1.0,
432 v_scale: float = 1.0,
433 attn_type: str = AttentionType.DECODER,
434 + output: Optional[torch.Tensor] = None,
435 ) -> torch.Tensor:
436 """Forward pass with torch SDPA and PagedAttention.
437
vllm/attention/backends/xformers.py
+1
@@ -417,6 +417,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
417 k_scale: float = 1.0,
418 v_scale: float = 1.0,
419 attn_type: str = AttentionType.DECODER,
420 + output: Optional[torch.Tensor] = None,
421 ) -> torch.Tensor:
422 """Forward pass with xFormers and PagedAttention.
423
vllm/attention/layer.py
+72 -4
@@ -4,7 +4,6 @@ from typing import Any, Dict, List, Optional
4 import torch
5 import torch.nn as nn
6
7 -import vllm.envs as envs
7 from vllm.attention import AttentionMetadata, AttentionType
8 from vllm.attention.selector import backend_name_to_enum, get_attn_backend
9 from vllm.config import CacheConfig, get_current_vllm_config
@@ -12,7 +11,7 @@ from vllm.forward_context import ForwardContext, get_forward_context
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
15 -from vllm.platforms import current_platform
14 +from vllm.platforms import _Backend, current_platform
15 from vllm.utils import direct_register_custom_op
16
17
@@ -97,14 +96,23 @@ class Attention(nn.Module):
96 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
97 alibi_slopes, sliding_window, kv_cache_dtype,
98 blocksparse_params, logits_soft_cap)
99 + self.num_heads = num_heads
100 + self.head_size = head_size
101 + self.num_kv_heads = num_kv_heads
102 self.backend = backend_name_to_enum(attn_backend.get_name())
103
104 # For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
105 # torch.compile works by registering the attention as one giant
106 # opaque custom op. For other platforms, we directly call them
107 # and let torch.compile handle them.
106 - self.use_direct_call = envs.VLLM_USE_V1 or not (
107 - current_platform.is_cuda_alike() or current_platform.is_cpu())
108 + self.use_direct_call = not current_platform.is_cuda_alike(
109 + ) and not current_platform.is_cpu()
110 +
111 + # For some attention backends, we allocate an output tensor before
112 + # calling the custom op. When piecewise cudagraph is enabled, this
113 + # makes sure the output tensor is allocated inside the cudagraph.
114 + self.use_output = self.backend == _Backend.FLASH_ATTN or \
115 + self.backend == _Backend.FLASH_ATTN_VLLM_V1
116 compilation_config = get_current_vllm_config().compilation_config
117 if prefix in compilation_config.static_forward_context:
118 raise ValueError(f"Duplicate layer name: {prefix}")
@@ -130,6 +138,22 @@ class Attention(nn.Module):
138 self._k_scale,
139 self._v_scale,
140 attn_type=attn_type)
141 + elif self.use_output:
142 + output = torch.empty_like(query)
143 + hidden_size = query.size(-1)
144 + # Reshape the query, key, and value tensors.
145 + # NOTE(woosuk): We do this outside the custom op to minimize the
146 + # CPU overheads from the non-CUDA-graph regions.
147 + query = query.view(-1, self.num_heads, self.head_size)
148 + output = output.view(-1, self.num_heads, self.head_size)
149 + if key is not None:
150 + key = key.view(-1, self.num_kv_heads, self.head_size)
151 + if value is not None:
152 + value = value.view(-1, self.num_kv_heads, self.head_size)
153 + torch.ops.vllm.unified_attention_with_output(
154 + query, key, value, output, kv_cache, attn_type,
155 + self.layer_name)
156 + return output.view(-1, hidden_size)
157 else:
158 return torch.ops.vllm.unified_attention(query, key, value,
159 kv_cache, attn_type,
@@ -183,3 +207,47 @@ direct_register_custom_op(
207 fake_impl=unified_attention_fake,
208 dispatch_key=current_platform.dispatch_key,
209 )
210 +
211 +
212 +def unified_attention_with_output(
213 + query: torch.Tensor,
214 + key: torch.Tensor,
215 + value: torch.Tensor,
216 + output: torch.Tensor,
217 + kv_cache: torch.Tensor,
218 + attn_type: str,
219 + layer_name: str,
220 +) -> None:
221 + forward_context: ForwardContext = get_forward_context()
222 + attn_metadata = forward_context.dynamic_forward_context
223 + self = forward_context.static_forward_context[layer_name]
224 + self.impl.forward(query,
225 + key,
226 + value,
227 + kv_cache,
228 + attn_metadata,
229 + self._k_scale,
230 + self._v_scale,
231 + attn_type=attn_type,
232 + output=output)
233 +
234 +
235 +def unified_attention_with_output_fake(
236 + query: torch.Tensor,
237 + key: torch.Tensor,
238 + value: torch.Tensor,
239 + output: torch.Tensor,
240 + kv_cache: torch.Tensor,
241 + attn_type: str,
242 + layer_name: str,
243 +) -> None:
244 + return
245 +
246 +
247 +direct_register_custom_op(
248 + op_name="unified_attention_with_output",
249 + op_func=unified_attention_with_output,
250 + mutates_args=["kv_cache", "output"],
251 + fake_impl=unified_attention_with_output_fake,
252 + dispatch_key=current_platform.dispatch_key,
253 +)
vllm/config.py
+1 -1
@@ -2238,7 +2238,7 @@ class CompilationConfig(BaseModel):
2238 custom_ops: List[str] = Field(default_factory=list)
2239 splitting_ops: List[str] = Field(default_factory=lambda: [
2240 "vllm.unified_attention",
2241 - "vllm.unified_v1_flash_attention",
2241 + "vllm.unified_attention_with_output",
2242 ])
2243
2244 use_inductor: bool = True
vllm/v1/attention/backends/flash_attn.py
+39 -116
@@ -6,8 +6,6 @@ import torch
6
7 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8 AttentionMetadata, AttentionType)
9 -from vllm.forward_context import get_forward_context
10 -from vllm.utils import direct_register_custom_op
9 from vllm.vllm_flash_attn import flash_attn_varlen_func
10
11
@@ -113,13 +111,14 @@ class FlashAttentionImpl(AttentionImpl):
111 k_scale: float = 1.0,
112 v_scale: float = 1.0,
113 attn_type: AttentionType = AttentionType.DECODER,
114 + output: Optional[torch.Tensor] = None,
115 ) -> torch.Tensor:
116 """Forward pass with FlashAttention.
117
118 Args:
120 - query: shape = [num_tokens, num_heads * head_size]
121 - key: shape = [num_tokens, num_kv_heads * head_size]
122 - value: shape = [num_tokens, num_kv_heads * head_size]
119 + query: shape = [num_tokens, num_heads, head_size]
120 + key: shape = [num_tokens, num_kv_heads, head_size]
121 + value: shape = [num_tokens, num_kv_heads, head_size]
122 kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
123 attn_metadata: Metadata for attention.
124 Returns:
@@ -135,118 +134,42 @@ class FlashAttentionImpl(AttentionImpl):
134 assert k_scale == 1.0 and v_scale == 1.0, (
135 "key/v_scale is not supported in FlashAttention.")
136
138 - # Reshape the query, key, and value tensors.
139 - # NOTE(woosuk): We do this outside the custom op to minimize the CPU
140 - # overheads from the non-CUDA-graph regions.
141 - query = query.view(-1, self.num_heads, self.head_size)
142 - key = key.view(-1, self.num_kv_heads, self.head_size)
143 - value = value.view(-1, self.num_kv_heads, self.head_size)
144 -
145 - output = torch.empty_like(query)
146 - torch.ops.vllm.unified_v1_flash_attention(
147 - output,
148 - query,
149 - key,
150 - value,
151 - self.num_heads,
152 - self.head_size,
153 - self.num_kv_heads,
154 - kv_cache,
137 + if attn_metadata is None:
138 + # Profiling run.
139 + return output
140 +
141 + num_actual_tokens = attn_metadata.num_actual_tokens
142 +
143 + # Reshape the input keys and values and store them in the cache.
144 + key_cache = kv_cache[0]
145 + value_cache = kv_cache[1]
146 + torch.ops._C_cache_ops.reshape_and_cache_flash(
147 + key[:num_actual_tokens],
148 + value[:num_actual_tokens],
149 + key_cache,
150 + value_cache,
151 + attn_metadata.slot_mapping,
152 self.kv_cache_dtype,
153 k_scale,
154 v_scale,
158 - self.scale,
159 - self.sliding_window,
160 - self.alibi_slopes,
161 - self.logits_soft_cap,
155 )
163 - return output.view(-1, self.num_heads * self.head_size)
164 -
165 -
166 -def unified_v1_flash_attention(
167 - output: torch.Tensor,
168 - query: torch.Tensor,
169 - key: torch.Tensor,
170 - value: torch.Tensor,
171 - num_heads: int,
172 - head_size: int,
173 - num_kv_heads: int,
174 - kv_cache: torch.Tensor,
175 - kv_cache_dtype: str,
176 - k_scale: float,
177 - v_scale: float,
178 - softmax_scale: float,
179 - window_size: Optional[List[int]] = None,
180 - alibi_slopes: Optional[torch.Tensor] = None,
181 - logits_soft_cap: Optional[float] = None,
182 -) -> None:
183 - context = get_forward_context()
184 - current_metadata = context.dynamic_forward_context
185 - if current_metadata is None:
186 - # Profiling run.
187 - return
188 -
189 - assert current_metadata is not None
190 - assert isinstance(current_metadata, FlashAttentionMetadata)
191 - attn_metadata: FlashAttentionMetadata = current_metadata
192 - num_actual_tokens = attn_metadata.num_actual_tokens
193 -
194 - # Reshape the input keys and values and store them in the cache.
195 - key_cache = kv_cache[0]
196 - value_cache = kv_cache[1]
197 - torch.ops._C_cache_ops.reshape_and_cache_flash(
198 - key[:num_actual_tokens],
199 - value[:num_actual_tokens],
200 - key_cache,
201 - value_cache,
202 - attn_metadata.slot_mapping,
203 - kv_cache_dtype,
204 - k_scale,
205 - v_scale,
206 - )
207 -
208 - # Compute attention and update output up to `num_actual_tokens`.
209 - flash_attn_varlen_func(
210 - q=query[:num_actual_tokens],
211 - k=key_cache,
212 - v=value_cache,
213 - out=output[:num_actual_tokens],
214 - cu_seqlens_q=attn_metadata.query_start_loc,
215 - max_seqlen_q=attn_metadata.max_query_len,
216 - cu_seqlens_k=attn_metadata.seq_start_loc,
217 - max_seqlen_k=attn_metadata.max_seq_len,
218 - softmax_scale=softmax_scale,
219 - causal=True,
220 - alibi_slopes=alibi_slopes,
221 - window_size=window_size,
222 - block_table=attn_metadata.block_table,
223 - softcap=logits_soft_cap,
224 - )
225 -
226 -
227 -def unified_v1_flash_attention_fake(
228 - output: torch.Tensor,
229 - query: torch.Tensor,
230 - key: torch.Tensor,
231 - value: torch.Tensor,
232 - num_heads: int,
233 - head_size: int,
234 - num_kv_heads: int,
235 - kv_cache: torch.Tensor,
236 - kv_cache_dtype: str,
237 - k_scale: float,
238 - v_scale: float,
239 - softmax_scale: float,
240 - window_size: Optional[List[int]] = None,
241 - alibi_slopes: Optional[torch.Tensor] = None,
242 - logits_soft_cap: Optional[float] = None,
243 -) -> None:
244 - return
245 -
246 -
247 -direct_register_custom_op(
248 - op_name="unified_v1_flash_attention",
249 - op_func=unified_v1_flash_attention,
250 - mutates_args=["kv_cache", "output"],
251 - fake_impl=unified_v1_flash_attention_fake,
252 -)
156 +
157 + # Compute attention and update output up to `num_actual_tokens`.
158 + flash_attn_varlen_func(
159 + q=query[:num_actual_tokens],
160 + k=key_cache,
161 + v=value_cache,
162 + out=output[:num_actual_tokens],
163 + cu_seqlens_q=attn_metadata.query_start_loc,
164 + max_seqlen_q=attn_metadata.max_query_len,
165 + cu_seqlens_k=attn_metadata.seq_start_loc,
166 + max_seqlen_k=attn_metadata.max_seq_len,
167 + softmax_scale=self.scale,
168 + causal=True,
169 + alibi_slopes=self.alibi_slopes,
170 + window_size=self.sliding_window,
171 + block_table=attn_metadata.block_table,
172 + softcap=self.logits_soft_cap,
173 + )
174 +
175 + return output