A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Misc] Pass `attention` to impl backend (#12218)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
wangxiyuan committed
Jan 20, 2025 at 23:25 UTC
86bfb6dba7c6e0650e7d7498cbd46b49155b2a42
12 files changed
+86
-78
vllm/attention/backends/abstract.py
+19
-4
@@ -1,8 +1,8 @@
1
from abc import ABC, abstractmethod
2
from contextlib import contextmanager
3
from dataclasses import dataclass, fields
4
-from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
5
- Tuple, Type, TypeVar)
4
+from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional,
5
+ Protocol, Set, Tuple, Type, TypeVar)
6
7
import torch
8
@@ -223,6 +223,22 @@ class AttentionMetadataBuilder(ABC, Generic[T]):
223
raise NotImplementedError
224
225
226
+class AttentionLayer(Protocol):
227
+
228
+ _k_scale: float
229
+ _v_scale: float
230
+
231
+ def forward(
232
+ self,
233
+ query: torch.Tensor,
234
+ key: torch.Tensor,
235
+ value: torch.Tensor,
236
+ kv_cache: torch.Tensor,
237
+ attn_metadata: AttentionMetadata,
238
+ ) -> torch.Tensor:
239
+ ...
240
+
241
+
242
class AttentionImpl(ABC, Generic[T]):
243
244
@abstractmethod
@@ -244,13 +260,12 @@ class AttentionImpl(ABC, Generic[T]):
260
@abstractmethod
261
def forward(
262
self,
263
+ layer: AttentionLayer,
264
query: torch.Tensor,
265
key: torch.Tensor,
266
value: torch.Tensor,
267
kv_cache: torch.Tensor,
268
attn_metadata: T,
252
- k_scale: float = 1.0,
253
- v_scale: float = 1.0,
269
output: Optional[torch.Tensor] = None,
270
) -> torch.Tensor:
271
raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+6
-6
@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4
import torch
5
6
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7
+ AttentionLayer,
8
AttentionMetadata, AttentionType)
9
from vllm.attention.backends.utils import (CommonAttentionState,
10
CommonMetadataBuilder)
@@ -358,13 +359,12 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
359
360
def forward(
361
self,
362
+ layer: AttentionLayer,
363
query: torch.Tensor,
364
key: torch.Tensor,
365
value: torch.Tensor,
366
kv_cache: torch.Tensor,
367
attn_metadata: BlocksparseFlashAttentionMetadata,
366
- k_scale: float = 1.0,
367
- v_scale: float = 1.0,
368
output: Optional[torch.Tensor] = None,
369
) -> torch.Tensor:
370
"""Forward pass with FlashAttention and PagedAttention.
@@ -401,8 +401,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
401
value_cache,
402
attn_metadata.slot_mapping,
403
self.kv_cache_dtype,
404
- k_scale,
405
- v_scale,
404
+ layer._k_scale,
405
+ layer._v_scale,
406
)
407
408
if prefill_meta := attn_metadata.prefill_metadata:
@@ -439,8 +439,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
439
self.num_kv_heads,
440
self.scale,
441
self.alibi_slopes,
442
- k_scale,
443
- v_scale,
442
+ layer._k_scale,
443
+ layer._v_scale,
444
tp_rank=self.tp_rank,
445
blocksparse_local_blocks=self.local_blocks,
446
blocksparse_vert_stride=self.vert_stride,
vllm/attention/backends/flash_attn.py
+5
-5
@@ -8,6 +8,7 @@ import torch
8
9
from vllm import _custom_ops as ops
10
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11
+ AttentionLayer,
12
AttentionMetadata,
13
AttentionMetadataBuilder,
14
AttentionType)
@@ -634,13 +635,12 @@ class FlashAttentionImpl(AttentionImpl):
635
636
def forward(
637
self,
638
+ layer: AttentionLayer,
639
query: torch.Tensor,
640
key: torch.Tensor,
641
value: torch.Tensor,
642
kv_cache: torch.Tensor,
643
attn_metadata: FlashAttentionMetadata,
642
- k_scale: float = 1.0,
643
- v_scale: float = 1.0,
644
output: Optional[torch.Tensor] = None,
645
) -> torch.Tensor:
646
"""Forward pass with FlashAttention.
@@ -657,7 +657,7 @@ class FlashAttentionImpl(AttentionImpl):
657
NOTE: It in-place updates the output tensor.
658
"""
659
# NOTE(woosuk): FlashAttention does not support FP8 KV cache.
660
- assert k_scale == 1.0 and v_scale == 1.0, (
660
+ assert layer._k_scale == 1.0 and layer._v_scale == 1.0, (
661
"key/v_scale is not supported in FlashAttention.")
662
663
assert output is not None, "Output tensor must be provided."
@@ -709,8 +709,8 @@ class FlashAttentionImpl(AttentionImpl):
709
kv_cache[1],
710
updated_slot_mapping.flatten(), # type: ignore[union-attr]
711
kv_cache_dtype,
712
- k_scale,
713
- v_scale,
712
+ layer._k_scale,
713
+ layer._v_scale,
714
)
715
716
(num_prefill_query_tokens, num_prefill_kv_tokens,
vllm/attention/backends/flashinfer.py
+8
-8
@@ -23,6 +23,7 @@ import torch
23
import vllm.envs as envs
24
from vllm import _custom_ops as ops
25
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
26
+ AttentionLayer,
27
AttentionMetadata,
28
AttentionMetadataBuilder,
29
AttentionState, AttentionType)
@@ -792,13 +793,12 @@ class FlashInferImpl(AttentionImpl):
793
794
def forward(
795
self,
796
+ layer: AttentionLayer,
797
query: torch.Tensor,
798
key: torch.Tensor,
799
value: torch.Tensor,
800
kv_cache: torch.Tensor,
801
attn_metadata: FlashInferMetadata,
800
- k_scale: float = 1.0,
801
- v_scale: float = 1.0,
802
output: Optional[torch.Tensor] = None,
803
) -> torch.Tensor:
804
@@ -826,8 +826,8 @@ class FlashInferImpl(AttentionImpl):
826
kv_cache[:, 1],
827
attn_metadata.slot_mapping.flatten(),
828
kv_cache_dtype,
829
- k_scale,
830
- v_scale,
829
+ layer._k_scale,
830
+ layer._v_scale,
831
)
832
# The FlashInfer api requires data to be in fp8_e4m3 or fp8_e5m2
833
# to process the cache when the kv_cache_dtype is fp8
@@ -886,8 +886,8 @@ class FlashInferImpl(AttentionImpl):
886
kv_cache,
887
logits_soft_cap=logits_soft_cap,
888
causal=True,
889
- k_scale=k_scale,
890
- v_scale=v_scale,
889
+ k_scale=layer._k_scale,
890
+ v_scale=layer._v_scale,
891
window_left=window_left)
892
if decode_meta := attn_metadata.decode_metadata:
893
assert decode_meta is not None
@@ -897,8 +897,8 @@ class FlashInferImpl(AttentionImpl):
897
kv_cache,
898
sm_scale=softmax_scale,
899
logits_soft_cap=logits_soft_cap,
900
- k_scale=k_scale,
901
- v_scale=v_scale,
900
+ k_scale=layer._k_scale,
901
+ v_scale=layer._v_scale,
902
window_left=window_left)
903
904
if prefill_output is None and decode_output is not None:
vllm/attention/backends/hpu_attn.py
+2
-2
@@ -11,6 +11,7 @@ import vllm_hpu_extension.ops as ops
11
from vllm_hpu_extension.utils import Matmul, Softmax, VLLMKVCache
12
13
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
+ AttentionLayer,
15
AttentionMetadata, AttentionType)
16
from vllm.attention.backends.utils import CommonAttentionState
17
from vllm.attention.ops.hpu_paged_attn import (HPUPagedAttention,
@@ -152,13 +153,12 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
153
154
def forward(
155
self,
156
+ layer: AttentionLayer,
157
query: torch.Tensor,
158
key: torch.Tensor,
159
value: torch.Tensor,
160
kv_cache: torch.Tensor,
161
attn_metadata: HPUAttentionMetadata,
160
- k_scale: float = 1.0,
161
- v_scale: float = 1.0,
162
output: Optional[torch.Tensor] = None,
163
) -> torch.Tensor:
164
"""Forward pass with xFormers and PagedAttention.
vllm/attention/backends/ipex_attn.py
+9
-9
@@ -7,6 +7,7 @@ import torch
7
8
from vllm._ipex_ops import ipex_ops
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
+ AttentionLayer,
11
AttentionMetadata, AttentionType)
12
from vllm.attention.backends.utils import CommonAttentionState
13
from vllm.attention.ops.paged_attn import (PagedAttention,
@@ -171,13 +172,12 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
172
173
def forward(
174
self,
175
+ layer: AttentionLayer,
176
query: torch.Tensor,
177
key: torch.Tensor,
178
value: torch.Tensor,
179
kv_cache: torch.Tensor,
180
attn_metadata: IpexAttnMetadata, # type: ignore
179
- k_scale: float = 1.0,
180
- v_scale: float = 1.0,
181
output: Optional[torch.Tensor] = None,
182
) -> torch.Tensor:
183
"""Forward pass with IPEX varlen_attention and PagedAttention.
@@ -193,7 +193,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
193
Returns:
194
shape = [num_tokens, num_heads * head_size]
195
"""
196
- assert k_scale == 1.0 and v_scale == 1.0
196
+ assert layer._k_scale == 1.0 and layer._v_scale == 1.0
197
num_tokens, hidden_size = query.shape
198
# Reshape the query, key, and value tensors.
199
query = query.view(-1, self.num_heads, self.head_size)
@@ -210,8 +210,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
210
value_cache,
211
attn_metadata.slot_mapping.flatten(),
212
self.kv_cache_dtype,
213
- k_scale,
214
- v_scale,
213
+ layer._k_scale,
214
+ layer._v_scale,
215
)
216
217
if attn_metadata.is_prompt:
@@ -296,8 +296,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
296
max_seq_len,
297
self.alibi_slopes,
298
self.kv_cache_dtype,
299
- k_scale,
300
- v_scale,
299
+ layer._k_scale,
300
+ layer._v_scale,
301
)
302
else:
303
# Run PagedAttention V2.
@@ -329,8 +329,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
329
max_seq_len,
330
self.alibi_slopes,
331
self.kv_cache_dtype,
332
- k_scale,
333
- v_scale,
332
+ layer._k_scale,
333
+ layer._v_scale,
334
)
335
336
# Reshape the output tensor.
vllm/attention/backends/pallas.py
+3
-3
@@ -5,6 +5,7 @@ import torch
5
import torch_xla.experimental.custom_kernel # Required to register custom ops.
6
7
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8
+ AttentionLayer,
9
AttentionMetadata, AttentionType)
10
from vllm.attention.backends.utils import CommonAttentionState
11
@@ -150,13 +151,12 @@ class PallasAttentionBackendImpl(AttentionImpl):
151
152
def forward(
153
self,
154
+ layer: AttentionLayer,
155
query: torch.Tensor,
156
key: torch.Tensor,
157
value: torch.Tensor,
158
kv_cache: Tuple[torch.Tensor, torch.Tensor],
159
attn_metadata: PallasMetadata,
158
- k_scale: float = 1.0,
159
- v_scale: float = 1.0,
160
output: Optional[torch.Tensor] = None,
161
) -> torch.Tensor:
162
"""Forward pass with Pallas attention.
@@ -173,7 +173,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
173
Returns:
174
shape = [batch_size, seq_len, num_heads * head_size]
175
"""
176
- assert k_scale == 1.0 and v_scale == 1.0
176
+ assert layer._k_scale == 1.0 and layer._v_scale == 1.0
177
batch_size, seq_len, hidden_size = query.shape
178
query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
179
key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
vllm/attention/backends/rocm_flash_attn.py
+10
-10
@@ -7,6 +7,7 @@ import torch
7
import vllm.envs as envs
8
from vllm import _custom_ops as ops
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
+ AttentionLayer,
11
AttentionMetadata, AttentionType)
12
from vllm.attention.backends.utils import (CommonAttentionState,
13
CommonMetadataBuilder)
@@ -414,13 +415,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
415
416
def forward(
417
self,
418
+ layer: AttentionLayer,
419
query: torch.Tensor,
420
key: torch.Tensor,
421
value: torch.Tensor,
422
kv_cache: torch.Tensor,
423
attn_metadata: ROCmFlashAttentionMetadata,
422
- k_scale: float = 1.0,
423
- v_scale: float = 1.0,
424
output: Optional[torch.Tensor] = None,
425
) -> torch.Tensor:
426
"""Forward pass with FlashAttention and PagedAttention.
@@ -458,8 +458,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
458
value_cache,
459
attn_metadata.slot_mapping,
460
self.kv_cache_dtype,
461
- k_scale,
462
- v_scale,
461
+ layer._k_scale,
462
+ layer._v_scale,
463
)
464
465
num_prefill_tokens = attn_metadata.num_prefill_tokens
@@ -567,8 +567,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
567
prefill_meta.max_query_len,
568
self.alibi_slopes,
569
self.sliding_window[0],
570
- k_scale,
571
- v_scale,
570
+ layer._k_scale,
571
+ layer._v_scale,
572
)
573
574
if decode_meta := attn_metadata.decode_metadata:
@@ -613,8 +613,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
613
max_seq_len,
614
self.alibi_slopes,
615
self.kv_cache_dtype,
616
- k_scale,
617
- v_scale,
616
+ layer._k_scale,
617
+ layer._v_scale,
618
)
619
else:
620
output[num_prefill_tokens:] = PagedAttention.forward_decode(
@@ -628,8 +628,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
628
self.num_kv_heads,
629
self.scale,
630
self.alibi_slopes,
631
- k_scale,
632
- v_scale,
631
+ layer._k_scale,
632
+ layer._v_scale,
633
)
634
635
# Reshape the output tensor.
vllm/attention/backends/torch_sdpa.py
+8
-10
@@ -7,6 +7,7 @@ import torch
7
from torch.nn.functional import scaled_dot_product_attention
8
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
+ AttentionLayer,
11
AttentionMetadata,
12
AttentionMetadataBuilder,
13
AttentionType)
@@ -429,13 +430,12 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
430
431
def forward(
432
self,
433
+ layer: AttentionLayer,
434
query: torch.Tensor,
435
key: torch.Tensor,
436
value: torch.Tensor,
437
kv_cache: torch.Tensor,
438
attn_metadata: TorchSDPAMetadata, # type: ignore
437
- k_scale: float = 1.0,
438
- v_scale: float = 1.0,
439
output: Optional[torch.Tensor] = None,
440
) -> torch.Tensor:
441
"""Forward pass with torch SDPA and PagedAttention.
@@ -451,7 +451,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
451
Returns:
452
shape = [num_tokens, num_heads * head_size]
453
"""
454
- assert k_scale == 1.0 and v_scale == 1.0
454
+ assert layer._k_scale == 1.0 and layer._v_scale == 1.0
455
attn_type = self.attn_type
456
if (attn_type == AttentionType.ENCODER
457
and (not attn_metadata.is_all_encoder_attn_metadata_set)):
@@ -493,11 +493,9 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
493
# Update self-attention KV cache (prefill/decode)
494
updated_slot_mapping = attn_metadata.slot_mapping
495
496
- PagedAttention.write_to_paged_cache(key, value, key_cache,
497
- value_cache,
498
- updated_slot_mapping,
499
- self.kv_cache_dtype,
500
- k_scale, v_scale)
496
+ PagedAttention.write_to_paged_cache(
497
+ key, value, key_cache, value_cache, updated_slot_mapping,
498
+ self.kv_cache_dtype, layer._k_scale, layer._v_scale)
499
500
if attn_type != AttentionType.ENCODER:
501
# Decoder self-attention supports chunked prefill.
@@ -571,8 +569,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
569
self.num_kv_heads,
570
self.scale,
571
self.alibi_slopes,
574
- k_scale,
575
- v_scale,
572
+ layer._k_scale,
573
+ layer._v_scale,
574
)
575
576
# Reshape the output tensor.
vllm/attention/backends/xformers.py
+9
-11
@@ -10,6 +10,7 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
10
LowerTriangularMaskWithTensorBias)
11
12
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13
+ AttentionLayer,
14
AttentionMetadata, AttentionType)
15
from vllm.attention.backends.utils import (
16
CommonAttentionState, CommonMetadataBuilder,
@@ -412,13 +413,12 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
413
414
def forward(
415
self,
416
+ layer: AttentionLayer,
417
query: torch.Tensor,
418
key: Optional[torch.Tensor],
419
value: Optional[torch.Tensor],
420
kv_cache: torch.Tensor,
421
attn_metadata: "XFormersMetadata",
420
- k_scale: float = 1.0,
421
- v_scale: float = 1.0,
422
output: Optional[torch.Tensor] = None,
423
) -> torch.Tensor:
424
"""Forward pass with xFormers and PagedAttention.
@@ -524,11 +524,9 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
524
# If kv_cache is not provided, the new key and value tensors are
525
# not cached. This happens during the initial memory
526
# profiling run.
527
- PagedAttention.write_to_paged_cache(key, value, key_cache,
528
- value_cache,
529
- updated_slot_mapping,
530
- self.kv_cache_dtype,
531
- k_scale, v_scale)
527
+ PagedAttention.write_to_paged_cache(
528
+ key, value, key_cache, value_cache, updated_slot_mapping,
529
+ self.kv_cache_dtype, layer._k_scale, layer._v_scale)
530
(num_prefill_query_tokens, num_prefill_kv_tokens,
531
num_decode_query_tokens) = \
532
get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
@@ -580,8 +578,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
578
prefill_meta.max_query_len,
579
self.alibi_slopes,
580
self.sliding_window,
583
- k_scale,
584
- v_scale,
581
+ layer._k_scale,
582
+ layer._v_scale,
583
)
584
assert output[:num_prefill_query_tokens].shape == out.shape
585
output[:num_prefill_query_tokens] = out
@@ -607,8 +605,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
605
self.num_kv_heads,
606
self.scale,
607
self.alibi_slopes,
610
- k_scale,
611
- v_scale,
608
+ layer._k_scale,
609
+ layer._v_scale,
610
)
611
612
# Reshape the output tensor.
vllm/attention/layer.py
+3
-5
@@ -243,8 +243,7 @@ def unified_attention(
243
attn_metadata = forward_context.attn_metadata
244
self = forward_context.attn_layers[layer_name]
245
kv_cache = self.kv_cache[forward_context.virtual_engine]
246
- return self.impl.forward(query, key, value, kv_cache, attn_metadata,
247
- self._k_scale, self._v_scale)
246
+ return self.impl.forward(self, query, key, value, kv_cache, attn_metadata)
247
248
249
def unified_attention_fake(
@@ -276,13 +275,12 @@ def unified_attention_with_output(
275
attn_metadata = forward_context.attn_metadata
276
self = forward_context.attn_layers[layer_name]
277
kv_cache = self.kv_cache[forward_context.virtual_engine]
279
- self.impl.forward(query,
278
+ self.impl.forward(self,
279
+ query,
280
key,
281
value,
282
kv_cache,
283
attn_metadata,
284
- self._k_scale,
285
- self._v_scale,
284
output=output)
285
286
vllm/v1/attention/backends/flash_attn.py
+4
-5
@@ -130,13 +130,12 @@ class FlashAttentionImpl(AttentionImpl):
130
131
def forward(
132
self,
133
+ layer: torch.nn.Module,
134
query: torch.Tensor,
135
key: torch.Tensor,
136
value: torch.Tensor,
137
kv_cache: torch.Tensor,
138
attn_metadata: FlashAttentionMetadata,
138
- k_scale: float = 1.0,
139
- v_scale: float = 1.0,
139
output: Optional[torch.Tensor] = None,
140
) -> torch.Tensor:
141
"""Forward pass with FlashAttention.
@@ -151,7 +150,7 @@ class FlashAttentionImpl(AttentionImpl):
150
shape = [num_tokens, num_heads * head_size]
151
"""
152
# NOTE(woosuk): FlashAttention does not support FP8 KV cache.
154
- assert k_scale == 1.0 and v_scale == 1.0, (
153
+ assert layer._k_scale == 1.0 and layer._v_scale == 1.0, (
154
"key/v_scale is not supported in FlashAttention.")
155
156
assert output is not None, "Output tensor must be provided."
@@ -183,8 +182,8 @@ class FlashAttentionImpl(AttentionImpl):
182
value_cache,
183
attn_metadata.slot_mapping,
184
self.kv_cache_dtype,
186
- k_scale,
187
- v_scale,
185
+ layer._k_scale,
186
+ layer._v_scale,
187
)
188
189
# Compute attention and update output up to `num_actual_tokens`.