A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[misc] use out argument for flash attention (#10822)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 2, 2024 at 02:50 UTC
a4c4daf3642ae2629608d5181487739b044fabe8
13 files changed
+144
-157
vllm/attention/backends/abstract.py
+1
@@ -247,5 +247,6 @@ class AttentionImpl(ABC, Generic[T]):
247
k_scale: float = 1.0,
248
v_scale: float = 1.0,
249
attn_type: str = AttentionType.DECODER,
250
+ output: Optional[torch.Tensor] = None,
251
) -> torch.Tensor:
252
raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+2
@@ -360,6 +360,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
360
k_scale: float = 1.0,
361
v_scale: float = 1.0,
362
attn_type: str = AttentionType.DECODER,
363
+ output: Optional[torch.Tensor] = None,
364
) -> torch.Tensor:
365
"""Forward pass with FlashAttention and PagedAttention.
366
@@ -448,5 +449,6 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
449
blocksparse_head_sliding_step=self.head_sliding_step,
450
)
451
452
+ assert output is not None
453
# Reshape the output tensor.
454
return output.view(num_tokens, hidden_size)
vllm/attention/backends/flash_attn.py
+19
-36
@@ -638,24 +638,27 @@ class FlashAttentionImpl(AttentionImpl):
638
k_scale: float = 1.0,
639
v_scale: float = 1.0,
640
attn_type: str = AttentionType.DECODER,
641
+ output: Optional[torch.Tensor] = None,
642
) -> torch.Tensor:
643
"""Forward pass with FlashAttention.
644
645
Args:
645
- query: shape = [num_tokens, num_heads * head_size]
646
- key: shape = [num_tokens, num_kv_heads * head_size]
647
- value: shape = [num_tokens, num_kv_heads * head_size]
646
+ query: shape = [num_tokens, num_heads, head_size]
647
+ key: shape = [num_tokens, num_kv_heads, head_size]
648
+ value: shape = [num_tokens, num_kv_heads, head_size]
649
+ output: shape = [num_tokens, num_heads, head_size]
650
kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
651
NOTE: kv_cache will be an empty tensor with shape [0]
652
for profiling run.
653
attn_metadata: Metadata for attention.
652
- Returns:
653
- shape = [num_tokens, num_heads * head_size]
654
+ NOTE: It in-place updates the output tensor.
655
"""
656
# NOTE(woosuk): FlashAttention does not support FP8 KV cache.
657
assert k_scale == 1.0 and v_scale == 1.0, (
658
"key/v_scale is not supported in FlashAttention.")
659
660
+ assert output is not None, "Output tensor must be provided."
661
+
662
if (attn_type == AttentionType.ENCODER
663
and (not attn_metadata.is_all_encoder_attn_metadata_set)):
664
raise AttributeError("Encoder attention requires setting "
@@ -666,23 +669,12 @@ class FlashAttentionImpl(AttentionImpl):
669
"requires setting cross-attention "
670
"metadata attributes.")
671
669
- num_heads: int = self.num_heads
670
- head_size: int = self.head_size
671
- num_kv_heads: int = self.num_kv_heads
672
kv_cache_dtype: str = self.kv_cache_dtype
673
softmax_scale: float = self.scale
674
window_size = self.sliding_window
675
alibi_slopes: Optional[torch.Tensor] = self.alibi_slopes
676
logits_soft_cap: Optional[float] = self.logits_soft_cap
677
678
- num_tokens, hidden_size = query.shape
679
-
680
- # Reshape the query, key, and value tensors.
681
- query = query.view(-1, num_heads, head_size)
682
- if (key is not None) and (value is not None):
683
- key = key.view(-1, num_kv_heads, head_size)
684
- value = value.view(-1, num_kv_heads, head_size)
685
-
678
if kv_cache.numel() > 0:
679
key_cache = kv_cache[0]
680
value_cache = kv_cache[1]
@@ -721,13 +713,13 @@ class FlashAttentionImpl(AttentionImpl):
713
num_decode_query_tokens) = \
714
get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
715
decode_query = query[num_prefill_query_tokens:]
716
+ decode_output = output[num_prefill_query_tokens:]
717
# QKV for prefill.
718
query = query[:num_prefill_query_tokens]
719
+ prefill_output = output[:num_prefill_query_tokens]
720
assert query.shape[0] == num_prefill_query_tokens
721
assert decode_query.shape[0] == num_decode_query_tokens
722
729
- prefill_output: Optional[torch.Tensor] = None
730
- decode_output: Optional[torch.Tensor] = None
723
if prefill_meta := attn_metadata.prefill_metadata:
724
# Prompt run.
725
if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
@@ -741,7 +733,7 @@ class FlashAttentionImpl(AttentionImpl):
733
key = key[:num_prefill_kv_tokens]
734
value = value[:num_prefill_kv_tokens]
735
744
- prefill_output = flash_attn_varlen_func(
736
+ flash_attn_varlen_func(
737
q=query,
738
k=key,
739
v=value,
@@ -754,6 +746,7 @@ class FlashAttentionImpl(AttentionImpl):
746
window_size=window_size,
747
alibi_slopes=alibi_slopes,
748
softcap=logits_soft_cap,
749
+ out=prefill_output,
750
)
751
else:
752
# prefix-enabled attention
@@ -761,7 +754,7 @@ class FlashAttentionImpl(AttentionImpl):
754
"Only decoder-only models support prefix caching")
755
assert prefill_meta.seq_lens is not None
756
max_seq_len = max(prefill_meta.seq_lens)
764
- prefill_output = flash_attn_varlen_func( # noqa
757
+ flash_attn_varlen_func( # noqa
758
q=query,
759
k=key_cache,
760
v=value_cache,
@@ -775,6 +768,7 @@ class FlashAttentionImpl(AttentionImpl):
768
alibi_slopes=alibi_slopes,
769
block_table=prefill_meta.block_tables,
770
softcap=logits_soft_cap,
771
+ out=prefill_output,
772
)
773
774
if decode_meta := attn_metadata.decode_metadata:
@@ -788,7 +782,7 @@ class FlashAttentionImpl(AttentionImpl):
782
assert attn_type == AttentionType.DECODER, (
783
"Only decoder-only models support max_decode_query_len > 1"
784
)
791
- decode_output = flash_attn_varlen_func(
785
+ flash_attn_varlen_func(
786
q=decode_query,
787
k=key_cache,
788
v=value_cache,
@@ -802,6 +796,7 @@ class FlashAttentionImpl(AttentionImpl):
796
alibi_slopes=alibi_slopes,
797
softcap=logits_soft_cap,
798
block_table=decode_meta.block_tables,
799
+ out=decode_output,
800
)
801
else:
802
# Use flash_attn_with_kvcache for normal decoding.
@@ -810,7 +805,7 @@ class FlashAttentionImpl(AttentionImpl):
805
_,
806
block_tables_arg,
807
) = get_seq_len_block_table_args(decode_meta, False, attn_type)
813
- decode_output = flash_attn_with_kvcache(
808
+ flash_attn_with_kvcache(
809
q=decode_query.unsqueeze(1),
810
k_cache=key_cache,
811
v_cache=value_cache,
@@ -821,20 +816,8 @@ class FlashAttentionImpl(AttentionImpl):
816
window_size=window_size,
817
alibi_slopes=alibi_slopes,
818
softcap=logits_soft_cap,
824
- ).squeeze(1)
825
-
826
- if prefill_output is None:
827
- assert decode_output is not None
828
- return decode_output.view(num_decode_query_tokens, hidden_size)
829
- if decode_output is None:
830
- assert prefill_output is not None
831
- return prefill_output.view(num_prefill_query_tokens, hidden_size)
832
-
833
- assert decode_meta is not None
834
- decode_output = decode_output.squeeze(1)
835
- output = torch.cat([prefill_output, decode_output], dim=0)
836
- return output.view(num_tokens, hidden_size)
837
-
819
+ out=decode_output.unsqueeze(1),
820
+ )
821
return output
822
823
vllm/attention/backends/flashinfer.py
+4
@@ -774,7 +774,11 @@ class FlashInferImpl(AttentionImpl):
774
k_scale: float = 1.0,
775
v_scale: float = 1.0,
776
attn_type: str = AttentionType.DECODER,
777
+ output: Optional[torch.Tensor] = None,
778
) -> torch.Tensor:
779
+
780
+ # TODO: directly write to output tensor
781
+
782
if attn_type != AttentionType.DECODER:
783
raise NotImplementedError("Encoder self-attention and "
784
"encoder/decoder cross-attention "
vllm/attention/backends/hpu_attn.py
+1
@@ -145,6 +145,7 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
145
k_scale: float = 1.0,
146
v_scale: float = 1.0,
147
attn_type: str = AttentionType.DECODER,
148
+ output: Optional[torch.Tensor] = None,
149
) -> torch.Tensor:
150
"""Forward pass with xFormers and PagedAttention.
151
vllm/attention/backends/ipex_attn.py
+1
@@ -173,6 +173,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
173
k_scale: float = 1.0,
174
v_scale: float = 1.0,
175
attn_type: str = AttentionType.DECODER,
176
+ output: Optional[torch.Tensor] = None,
177
) -> torch.Tensor:
178
"""Forward pass with IPEX varlen_attention and PagedAttention.
179
vllm/attention/backends/pallas.py
+1
@@ -151,6 +151,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
151
k_scale: float = 1.0,
152
v_scale: float = 1.0,
153
attn_type: str = AttentionType.DECODER,
154
+ output: Optional[torch.Tensor] = None,
155
) -> torch.Tensor:
156
"""Forward pass with Pallas attention.
157
vllm/attention/backends/rocm_flash_attn.py
+1
@@ -415,6 +415,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
415
k_scale: float = 1.0,
416
v_scale: float = 1.0,
417
attn_type: str = AttentionType.DECODER,
418
+ output: Optional[torch.Tensor] = None,
419
) -> torch.Tensor:
420
"""Forward pass with FlashAttention and PagedAttention.
421
vllm/attention/backends/torch_sdpa.py
+1
@@ -431,6 +431,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
431
k_scale: float = 1.0,
432
v_scale: float = 1.0,
433
attn_type: str = AttentionType.DECODER,
434
+ output: Optional[torch.Tensor] = None,
435
) -> torch.Tensor:
436
"""Forward pass with torch SDPA and PagedAttention.
437
vllm/attention/backends/xformers.py
+1
@@ -417,6 +417,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
417
k_scale: float = 1.0,
418
v_scale: float = 1.0,
419
attn_type: str = AttentionType.DECODER,
420
+ output: Optional[torch.Tensor] = None,
421
) -> torch.Tensor:
422
"""Forward pass with xFormers and PagedAttention.
423
vllm/attention/layer.py
+72
-4
@@ -4,7 +4,6 @@ from typing import Any, Dict, List, Optional
4
import torch
5
import torch.nn as nn
6
7
-import vllm.envs as envs
7
from vllm.attention import AttentionMetadata, AttentionType
8
from vllm.attention.selector import backend_name_to_enum, get_attn_backend
9
from vllm.config import CacheConfig, get_current_vllm_config
@@ -12,7 +11,7 @@ from vllm.forward_context import ForwardContext, get_forward_context
11
from vllm.model_executor.layers.quantization.base_config import (
12
QuantizationConfig)
13
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
15
-from vllm.platforms import current_platform
14
+from vllm.platforms import _Backend, current_platform
15
from vllm.utils import direct_register_custom_op
16
17
@@ -97,14 +96,23 @@ class Attention(nn.Module):
96
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
97
alibi_slopes, sliding_window, kv_cache_dtype,
98
blocksparse_params, logits_soft_cap)
99
+ self.num_heads = num_heads
100
+ self.head_size = head_size
101
+ self.num_kv_heads = num_kv_heads
102
self.backend = backend_name_to_enum(attn_backend.get_name())
103
104
# For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
105
# torch.compile works by registering the attention as one giant
106
# opaque custom op. For other platforms, we directly call them
107
# and let torch.compile handle them.
106
- self.use_direct_call = envs.VLLM_USE_V1 or not (
107
- current_platform.is_cuda_alike() or current_platform.is_cpu())
108
+ self.use_direct_call = not current_platform.is_cuda_alike(
109
+ ) and not current_platform.is_cpu()
110
+
111
+ # For some attention backends, we allocate an output tensor before
112
+ # calling the custom op. When piecewise cudagraph is enabled, this
113
+ # makes sure the output tensor is allocated inside the cudagraph.
114
+ self.use_output = self.backend == _Backend.FLASH_ATTN or \
115
+ self.backend == _Backend.FLASH_ATTN_VLLM_V1
116
compilation_config = get_current_vllm_config().compilation_config
117
if prefix in compilation_config.static_forward_context:
118
raise ValueError(f"Duplicate layer name: {prefix}")
@@ -130,6 +138,22 @@ class Attention(nn.Module):
138
self._k_scale,
139
self._v_scale,
140
attn_type=attn_type)
141
+ elif self.use_output:
142
+ output = torch.empty_like(query)
143
+ hidden_size = query.size(-1)
144
+ # Reshape the query, key, and value tensors.
145
+ # NOTE(woosuk): We do this outside the custom op to minimize the
146
+ # CPU overheads from the non-CUDA-graph regions.
147
+ query = query.view(-1, self.num_heads, self.head_size)
148
+ output = output.view(-1, self.num_heads, self.head_size)
149
+ if key is not None:
150
+ key = key.view(-1, self.num_kv_heads, self.head_size)
151
+ if value is not None:
152
+ value = value.view(-1, self.num_kv_heads, self.head_size)
153
+ torch.ops.vllm.unified_attention_with_output(
154
+ query, key, value, output, kv_cache, attn_type,
155
+ self.layer_name)
156
+ return output.view(-1, hidden_size)
157
else:
158
return torch.ops.vllm.unified_attention(query, key, value,
159
kv_cache, attn_type,
@@ -183,3 +207,47 @@ direct_register_custom_op(
207
fake_impl=unified_attention_fake,
208
dispatch_key=current_platform.dispatch_key,
209
)
210
+
211
+
212
+def unified_attention_with_output(
213
+ query: torch.Tensor,
214
+ key: torch.Tensor,
215
+ value: torch.Tensor,
216
+ output: torch.Tensor,
217
+ kv_cache: torch.Tensor,
218
+ attn_type: str,
219
+ layer_name: str,
220
+) -> None:
221
+ forward_context: ForwardContext = get_forward_context()
222
+ attn_metadata = forward_context.dynamic_forward_context
223
+ self = forward_context.static_forward_context[layer_name]
224
+ self.impl.forward(query,
225
+ key,
226
+ value,
227
+ kv_cache,
228
+ attn_metadata,
229
+ self._k_scale,
230
+ self._v_scale,
231
+ attn_type=attn_type,
232
+ output=output)
233
+
234
+
235
+def unified_attention_with_output_fake(
236
+ query: torch.Tensor,
237
+ key: torch.Tensor,
238
+ value: torch.Tensor,
239
+ output: torch.Tensor,
240
+ kv_cache: torch.Tensor,
241
+ attn_type: str,
242
+ layer_name: str,
243
+) -> None:
244
+ return
245
+
246
+
247
+direct_register_custom_op(
248
+ op_name="unified_attention_with_output",
249
+ op_func=unified_attention_with_output,
250
+ mutates_args=["kv_cache", "output"],
251
+ fake_impl=unified_attention_with_output_fake,
252
+ dispatch_key=current_platform.dispatch_key,
253
+)
vllm/config.py
+1
-1
@@ -2238,7 +2238,7 @@ class CompilationConfig(BaseModel):
2238
custom_ops: List[str] = Field(default_factory=list)
2239
splitting_ops: List[str] = Field(default_factory=lambda: [
2240
"vllm.unified_attention",
2241
- "vllm.unified_v1_flash_attention",
2241
+ "vllm.unified_attention_with_output",
2242
])
2243
2244
use_inductor: bool = True
vllm/v1/attention/backends/flash_attn.py
+39
-116
@@ -6,8 +6,6 @@ import torch
6
7
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8
AttentionMetadata, AttentionType)
9
-from vllm.forward_context import get_forward_context
10
-from vllm.utils import direct_register_custom_op
9
from vllm.vllm_flash_attn import flash_attn_varlen_func
10
11
@@ -113,13 +111,14 @@ class FlashAttentionImpl(AttentionImpl):
111
k_scale: float = 1.0,
112
v_scale: float = 1.0,
113
attn_type: AttentionType = AttentionType.DECODER,
114
+ output: Optional[torch.Tensor] = None,
115
) -> torch.Tensor:
116
"""Forward pass with FlashAttention.
117
118
Args:
120
- query: shape = [num_tokens, num_heads * head_size]
121
- key: shape = [num_tokens, num_kv_heads * head_size]
122
- value: shape = [num_tokens, num_kv_heads * head_size]
119
+ query: shape = [num_tokens, num_heads, head_size]
120
+ key: shape = [num_tokens, num_kv_heads, head_size]
121
+ value: shape = [num_tokens, num_kv_heads, head_size]
122
kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
123
attn_metadata: Metadata for attention.
124
Returns:
@@ -135,118 +134,42 @@ class FlashAttentionImpl(AttentionImpl):
134
assert k_scale == 1.0 and v_scale == 1.0, (
135
"key/v_scale is not supported in FlashAttention.")
136
138
- # Reshape the query, key, and value tensors.
139
- # NOTE(woosuk): We do this outside the custom op to minimize the CPU
140
- # overheads from the non-CUDA-graph regions.
141
- query = query.view(-1, self.num_heads, self.head_size)
142
- key = key.view(-1, self.num_kv_heads, self.head_size)
143
- value = value.view(-1, self.num_kv_heads, self.head_size)
144
-
145
- output = torch.empty_like(query)
146
- torch.ops.vllm.unified_v1_flash_attention(
147
- output,
148
- query,
149
- key,
150
- value,
151
- self.num_heads,
152
- self.head_size,
153
- self.num_kv_heads,
154
- kv_cache,
137
+ if attn_metadata is None:
138
+ # Profiling run.
139
+ return output
140
+
141
+ num_actual_tokens = attn_metadata.num_actual_tokens
142
+
143
+ # Reshape the input keys and values and store them in the cache.
144
+ key_cache = kv_cache[0]
145
+ value_cache = kv_cache[1]
146
+ torch.ops._C_cache_ops.reshape_and_cache_flash(
147
+ key[:num_actual_tokens],
148
+ value[:num_actual_tokens],
149
+ key_cache,
150
+ value_cache,
151
+ attn_metadata.slot_mapping,
152
self.kv_cache_dtype,
153
k_scale,
154
v_scale,
158
- self.scale,
159
- self.sliding_window,
160
- self.alibi_slopes,
161
- self.logits_soft_cap,
155
)
163
- return output.view(-1, self.num_heads * self.head_size)
164
-
165
-
166
-def unified_v1_flash_attention(
167
- output: torch.Tensor,
168
- query: torch.Tensor,
169
- key: torch.Tensor,
170
- value: torch.Tensor,
171
- num_heads: int,
172
- head_size: int,
173
- num_kv_heads: int,
174
- kv_cache: torch.Tensor,
175
- kv_cache_dtype: str,
176
- k_scale: float,
177
- v_scale: float,
178
- softmax_scale: float,
179
- window_size: Optional[List[int]] = None,
180
- alibi_slopes: Optional[torch.Tensor] = None,
181
- logits_soft_cap: Optional[float] = None,
182
-) -> None:
183
- context = get_forward_context()
184
- current_metadata = context.dynamic_forward_context
185
- if current_metadata is None:
186
- # Profiling run.
187
- return
188
-
189
- assert current_metadata is not None
190
- assert isinstance(current_metadata, FlashAttentionMetadata)
191
- attn_metadata: FlashAttentionMetadata = current_metadata
192
- num_actual_tokens = attn_metadata.num_actual_tokens
193
-
194
- # Reshape the input keys and values and store them in the cache.
195
- key_cache = kv_cache[0]
196
- value_cache = kv_cache[1]
197
- torch.ops._C_cache_ops.reshape_and_cache_flash(
198
- key[:num_actual_tokens],
199
- value[:num_actual_tokens],
200
- key_cache,
201
- value_cache,
202
- attn_metadata.slot_mapping,
203
- kv_cache_dtype,
204
- k_scale,
205
- v_scale,
206
- )
207
-
208
- # Compute attention and update output up to `num_actual_tokens`.
209
- flash_attn_varlen_func(
210
- q=query[:num_actual_tokens],
211
- k=key_cache,
212
- v=value_cache,
213
- out=output[:num_actual_tokens],
214
- cu_seqlens_q=attn_metadata.query_start_loc,
215
- max_seqlen_q=attn_metadata.max_query_len,
216
- cu_seqlens_k=attn_metadata.seq_start_loc,
217
- max_seqlen_k=attn_metadata.max_seq_len,
218
- softmax_scale=softmax_scale,
219
- causal=True,
220
- alibi_slopes=alibi_slopes,
221
- window_size=window_size,
222
- block_table=attn_metadata.block_table,
223
- softcap=logits_soft_cap,
224
- )
225
-
226
-
227
-def unified_v1_flash_attention_fake(
228
- output: torch.Tensor,
229
- query: torch.Tensor,
230
- key: torch.Tensor,
231
- value: torch.Tensor,
232
- num_heads: int,
233
- head_size: int,
234
- num_kv_heads: int,
235
- kv_cache: torch.Tensor,
236
- kv_cache_dtype: str,
237
- k_scale: float,
238
- v_scale: float,
239
- softmax_scale: float,
240
- window_size: Optional[List[int]] = None,
241
- alibi_slopes: Optional[torch.Tensor] = None,
242
- logits_soft_cap: Optional[float] = None,
243
-) -> None:
244
- return
245
-
246
-
247
-direct_register_custom_op(
248
- op_name="unified_v1_flash_attention",
249
- op_func=unified_v1_flash_attention,
250
- mutates_args=["kv_cache", "output"],
251
- fake_impl=unified_v1_flash_attention_fake,
252
-)
156
+
157
+ # Compute attention and update output up to `num_actual_tokens`.
158
+ flash_attn_varlen_func(
159
+ q=query[:num_actual_tokens],
160
+ k=key_cache,
161
+ v=value_cache,
162
+ out=output[:num_actual_tokens],
163
+ cu_seqlens_q=attn_metadata.query_start_loc,
164
+ max_seqlen_q=attn_metadata.max_query_len,
165
+ cu_seqlens_k=attn_metadata.seq_start_loc,
166
+ max_seqlen_k=attn_metadata.max_seq_len,
167
+ softmax_scale=self.scale,
168
+ causal=True,
169
+ alibi_slopes=self.alibi_slopes,
170
+ window_size=self.sliding_window,
171
+ block_table=attn_metadata.block_table,
172
+ softcap=self.logits_soft_cap,
173
+ )
174
+
175
+ return output