@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 86bfb6dba7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc] Pass `attention` to impl backend (#12218)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>

wangxiyuan committed Jan 20, 2025 at 23:25 UTC 86bfb6dba7c6e0650e7d7498cbd46b49155b2a42
12 files changed +86 -78
vllm/attention/backends/abstract.py
+19 -4
@@ -1,8 +1,8 @@
1 from abc import ABC, abstractmethod
2 from contextlib import contextmanager
3 from dataclasses import dataclass, fields
4 -from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
5 - Tuple, Type, TypeVar)
4 +from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional,
5 + Protocol, Set, Tuple, Type, TypeVar)
6
7 import torch
8
@@ -223,6 +223,22 @@ class AttentionMetadataBuilder(ABC, Generic[T]):
223 raise NotImplementedError
224
225
226 +class AttentionLayer(Protocol):
227 +
228 + _k_scale: float
229 + _v_scale: float
230 +
231 + def forward(
232 + self,
233 + query: torch.Tensor,
234 + key: torch.Tensor,
235 + value: torch.Tensor,
236 + kv_cache: torch.Tensor,
237 + attn_metadata: AttentionMetadata,
238 + ) -> torch.Tensor:
239 + ...
240 +
241 +
242 class AttentionImpl(ABC, Generic[T]):
243
244 @abstractmethod
@@ -244,13 +260,12 @@ class AttentionImpl(ABC, Generic[T]):
260 @abstractmethod
261 def forward(
262 self,
263 + layer: AttentionLayer,
264 query: torch.Tensor,
265 key: torch.Tensor,
266 value: torch.Tensor,
267 kv_cache: torch.Tensor,
268 attn_metadata: T,
252 - k_scale: float = 1.0,
253 - v_scale: float = 1.0,
269 output: Optional[torch.Tensor] = None,
270 ) -> torch.Tensor:
271 raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+6 -6
@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4 import torch
5
6 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7 + AttentionLayer,
8 AttentionMetadata, AttentionType)
9 from vllm.attention.backends.utils import (CommonAttentionState,
10 CommonMetadataBuilder)
@@ -358,13 +359,12 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
359
360 def forward(
361 self,
362 + layer: AttentionLayer,
363 query: torch.Tensor,
364 key: torch.Tensor,
365 value: torch.Tensor,
366 kv_cache: torch.Tensor,
367 attn_metadata: BlocksparseFlashAttentionMetadata,
366 - k_scale: float = 1.0,
367 - v_scale: float = 1.0,
368 output: Optional[torch.Tensor] = None,
369 ) -> torch.Tensor:
370 """Forward pass with FlashAttention and PagedAttention.
@@ -401,8 +401,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
401 value_cache,
402 attn_metadata.slot_mapping,
403 self.kv_cache_dtype,
404 - k_scale,
405 - v_scale,
404 + layer._k_scale,
405 + layer._v_scale,
406 )
407
408 if prefill_meta := attn_metadata.prefill_metadata:
@@ -439,8 +439,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
439 self.num_kv_heads,
440 self.scale,
441 self.alibi_slopes,
442 - k_scale,
443 - v_scale,
442 + layer._k_scale,
443 + layer._v_scale,
444 tp_rank=self.tp_rank,
445 blocksparse_local_blocks=self.local_blocks,
446 blocksparse_vert_stride=self.vert_stride,
vllm/attention/backends/flash_attn.py
+5 -5
@@ -8,6 +8,7 @@ import torch
8
9 from vllm import _custom_ops as ops
10 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11 + AttentionLayer,
12 AttentionMetadata,
13 AttentionMetadataBuilder,
14 AttentionType)
@@ -634,13 +635,12 @@ class FlashAttentionImpl(AttentionImpl):
635
636 def forward(
637 self,
638 + layer: AttentionLayer,
639 query: torch.Tensor,
640 key: torch.Tensor,
641 value: torch.Tensor,
642 kv_cache: torch.Tensor,
643 attn_metadata: FlashAttentionMetadata,
642 - k_scale: float = 1.0,
643 - v_scale: float = 1.0,
644 output: Optional[torch.Tensor] = None,
645 ) -> torch.Tensor:
646 """Forward pass with FlashAttention.
@@ -657,7 +657,7 @@ class FlashAttentionImpl(AttentionImpl):
657 NOTE: It in-place updates the output tensor.
658 """
659 # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
660 - assert k_scale == 1.0 and v_scale == 1.0, (
660 + assert layer._k_scale == 1.0 and layer._v_scale == 1.0, (
661 "key/v_scale is not supported in FlashAttention.")
662
663 assert output is not None, "Output tensor must be provided."
@@ -709,8 +709,8 @@ class FlashAttentionImpl(AttentionImpl):
709 kv_cache[1],
710 updated_slot_mapping.flatten(), # type: ignore[union-attr]
711 kv_cache_dtype,
712 - k_scale,
713 - v_scale,
712 + layer._k_scale,
713 + layer._v_scale,
714 )
715
716 (num_prefill_query_tokens, num_prefill_kv_tokens,
vllm/attention/backends/flashinfer.py
+8 -8
@@ -23,6 +23,7 @@ import torch
23 import vllm.envs as envs
24 from vllm import _custom_ops as ops
25 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
26 + AttentionLayer,
27 AttentionMetadata,
28 AttentionMetadataBuilder,
29 AttentionState, AttentionType)
@@ -792,13 +793,12 @@ class FlashInferImpl(AttentionImpl):
793
794 def forward(
795 self,
796 + layer: AttentionLayer,
797 query: torch.Tensor,
798 key: torch.Tensor,
799 value: torch.Tensor,
800 kv_cache: torch.Tensor,
801 attn_metadata: FlashInferMetadata,
800 - k_scale: float = 1.0,
801 - v_scale: float = 1.0,
802 output: Optional[torch.Tensor] = None,
803 ) -> torch.Tensor:
804
@@ -826,8 +826,8 @@ class FlashInferImpl(AttentionImpl):
826 kv_cache[:, 1],
827 attn_metadata.slot_mapping.flatten(),
828 kv_cache_dtype,
829 - k_scale,
830 - v_scale,
829 + layer._k_scale,
830 + layer._v_scale,
831 )
832 # The FlashInfer api requires data to be in fp8_e4m3 or fp8_e5m2
833 # to process the cache when the kv_cache_dtype is fp8
@@ -886,8 +886,8 @@ class FlashInferImpl(AttentionImpl):
886 kv_cache,
887 logits_soft_cap=logits_soft_cap,
888 causal=True,
889 - k_scale=k_scale,
890 - v_scale=v_scale,
889 + k_scale=layer._k_scale,
890 + v_scale=layer._v_scale,
891 window_left=window_left)
892 if decode_meta := attn_metadata.decode_metadata:
893 assert decode_meta is not None
@@ -897,8 +897,8 @@ class FlashInferImpl(AttentionImpl):
897 kv_cache,
898 sm_scale=softmax_scale,
899 logits_soft_cap=logits_soft_cap,
900 - k_scale=k_scale,
901 - v_scale=v_scale,
900 + k_scale=layer._k_scale,
901 + v_scale=layer._v_scale,
902 window_left=window_left)
903
904 if prefill_output is None and decode_output is not None:
vllm/attention/backends/hpu_attn.py
+2 -2
@@ -11,6 +11,7 @@ import vllm_hpu_extension.ops as ops
11 from vllm_hpu_extension.utils import Matmul, Softmax, VLLMKVCache
12
13 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14 + AttentionLayer,
15 AttentionMetadata, AttentionType)
16 from vllm.attention.backends.utils import CommonAttentionState
17 from vllm.attention.ops.hpu_paged_attn import (HPUPagedAttention,
@@ -152,13 +153,12 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
153
154 def forward(
155 self,
156 + layer: AttentionLayer,
157 query: torch.Tensor,
158 key: torch.Tensor,
159 value: torch.Tensor,
160 kv_cache: torch.Tensor,
161 attn_metadata: HPUAttentionMetadata,
160 - k_scale: float = 1.0,
161 - v_scale: float = 1.0,
162 output: Optional[torch.Tensor] = None,
163 ) -> torch.Tensor:
164 """Forward pass with xFormers and PagedAttention.
vllm/attention/backends/ipex_attn.py
+9 -9
@@ -7,6 +7,7 @@ import torch
7
8 from vllm._ipex_ops import ipex_ops
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 + AttentionLayer,
11 AttentionMetadata, AttentionType)
12 from vllm.attention.backends.utils import CommonAttentionState
13 from vllm.attention.ops.paged_attn import (PagedAttention,
@@ -171,13 +172,12 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
172
173 def forward(
174 self,
175 + layer: AttentionLayer,
176 query: torch.Tensor,
177 key: torch.Tensor,
178 value: torch.Tensor,
179 kv_cache: torch.Tensor,
180 attn_metadata: IpexAttnMetadata, # type: ignore
179 - k_scale: float = 1.0,
180 - v_scale: float = 1.0,
181 output: Optional[torch.Tensor] = None,
182 ) -> torch.Tensor:
183 """Forward pass with IPEX varlen_attention and PagedAttention.
@@ -193,7 +193,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
193 Returns:
194 shape = [num_tokens, num_heads * head_size]
195 """
196 - assert k_scale == 1.0 and v_scale == 1.0
196 + assert layer._k_scale == 1.0 and layer._v_scale == 1.0
197 num_tokens, hidden_size = query.shape
198 # Reshape the query, key, and value tensors.
199 query = query.view(-1, self.num_heads, self.head_size)
@@ -210,8 +210,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
210 value_cache,
211 attn_metadata.slot_mapping.flatten(),
212 self.kv_cache_dtype,
213 - k_scale,
214 - v_scale,
213 + layer._k_scale,
214 + layer._v_scale,
215 )
216
217 if attn_metadata.is_prompt:
@@ -296,8 +296,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
296 max_seq_len,
297 self.alibi_slopes,
298 self.kv_cache_dtype,
299 - k_scale,
300 - v_scale,
299 + layer._k_scale,
300 + layer._v_scale,
301 )
302 else:
303 # Run PagedAttention V2.
@@ -329,8 +329,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
329 max_seq_len,
330 self.alibi_slopes,
331 self.kv_cache_dtype,
332 - k_scale,
333 - v_scale,
332 + layer._k_scale,
333 + layer._v_scale,
334 )
335
336 # Reshape the output tensor.
vllm/attention/backends/pallas.py
+3 -3
@@ -5,6 +5,7 @@ import torch
5 import torch_xla.experimental.custom_kernel # Required to register custom ops.
6
7 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8 + AttentionLayer,
9 AttentionMetadata, AttentionType)
10 from vllm.attention.backends.utils import CommonAttentionState
11
@@ -150,13 +151,12 @@ class PallasAttentionBackendImpl(AttentionImpl):
151
152 def forward(
153 self,
154 + layer: AttentionLayer,
155 query: torch.Tensor,
156 key: torch.Tensor,
157 value: torch.Tensor,
158 kv_cache: Tuple[torch.Tensor, torch.Tensor],
159 attn_metadata: PallasMetadata,
158 - k_scale: float = 1.0,
159 - v_scale: float = 1.0,
160 output: Optional[torch.Tensor] = None,
161 ) -> torch.Tensor:
162 """Forward pass with Pallas attention.
@@ -173,7 +173,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
173 Returns:
174 shape = [batch_size, seq_len, num_heads * head_size]
175 """
176 - assert k_scale == 1.0 and v_scale == 1.0
176 + assert layer._k_scale == 1.0 and layer._v_scale == 1.0
177 batch_size, seq_len, hidden_size = query.shape
178 query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
179 key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
vllm/attention/backends/rocm_flash_attn.py
+10 -10
@@ -7,6 +7,7 @@ import torch
7 import vllm.envs as envs
8 from vllm import _custom_ops as ops
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 + AttentionLayer,
11 AttentionMetadata, AttentionType)
12 from vllm.attention.backends.utils import (CommonAttentionState,
13 CommonMetadataBuilder)
@@ -414,13 +415,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
415
416 def forward(
417 self,
418 + layer: AttentionLayer,
419 query: torch.Tensor,
420 key: torch.Tensor,
421 value: torch.Tensor,
422 kv_cache: torch.Tensor,
423 attn_metadata: ROCmFlashAttentionMetadata,
422 - k_scale: float = 1.0,
423 - v_scale: float = 1.0,
424 output: Optional[torch.Tensor] = None,
425 ) -> torch.Tensor:
426 """Forward pass with FlashAttention and PagedAttention.
@@ -458,8 +458,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
458 value_cache,
459 attn_metadata.slot_mapping,
460 self.kv_cache_dtype,
461 - k_scale,
462 - v_scale,
461 + layer._k_scale,
462 + layer._v_scale,
463 )
464
465 num_prefill_tokens = attn_metadata.num_prefill_tokens
@@ -567,8 +567,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
567 prefill_meta.max_query_len,
568 self.alibi_slopes,
569 self.sliding_window[0],
570 - k_scale,
571 - v_scale,
570 + layer._k_scale,
571 + layer._v_scale,
572 )
573
574 if decode_meta := attn_metadata.decode_metadata:
@@ -613,8 +613,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
613 max_seq_len,
614 self.alibi_slopes,
615 self.kv_cache_dtype,
616 - k_scale,
617 - v_scale,
616 + layer._k_scale,
617 + layer._v_scale,
618 )
619 else:
620 output[num_prefill_tokens:] = PagedAttention.forward_decode(
@@ -628,8 +628,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
628 self.num_kv_heads,
629 self.scale,
630 self.alibi_slopes,
631 - k_scale,
632 - v_scale,
631 + layer._k_scale,
632 + layer._v_scale,
633 )
634
635 # Reshape the output tensor.
vllm/attention/backends/torch_sdpa.py
+8 -10
@@ -7,6 +7,7 @@ import torch
7 from torch.nn.functional import scaled_dot_product_attention
8
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 + AttentionLayer,
11 AttentionMetadata,
12 AttentionMetadataBuilder,
13 AttentionType)
@@ -429,13 +430,12 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
430
431 def forward(
432 self,
433 + layer: AttentionLayer,
434 query: torch.Tensor,
435 key: torch.Tensor,
436 value: torch.Tensor,
437 kv_cache: torch.Tensor,
438 attn_metadata: TorchSDPAMetadata, # type: ignore
437 - k_scale: float = 1.0,
438 - v_scale: float = 1.0,
439 output: Optional[torch.Tensor] = None,
440 ) -> torch.Tensor:
441 """Forward pass with torch SDPA and PagedAttention.
@@ -451,7 +451,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
451 Returns:
452 shape = [num_tokens, num_heads * head_size]
453 """
454 - assert k_scale == 1.0 and v_scale == 1.0
454 + assert layer._k_scale == 1.0 and layer._v_scale == 1.0
455 attn_type = self.attn_type
456 if (attn_type == AttentionType.ENCODER
457 and (not attn_metadata.is_all_encoder_attn_metadata_set)):
@@ -493,11 +493,9 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
493 # Update self-attention KV cache (prefill/decode)
494 updated_slot_mapping = attn_metadata.slot_mapping
495
496 - PagedAttention.write_to_paged_cache(key, value, key_cache,
497 - value_cache,
498 - updated_slot_mapping,
499 - self.kv_cache_dtype,
500 - k_scale, v_scale)
496 + PagedAttention.write_to_paged_cache(
497 + key, value, key_cache, value_cache, updated_slot_mapping,
498 + self.kv_cache_dtype, layer._k_scale, layer._v_scale)
499
500 if attn_type != AttentionType.ENCODER:
501 # Decoder self-attention supports chunked prefill.
@@ -571,8 +569,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
569 self.num_kv_heads,
570 self.scale,
571 self.alibi_slopes,
574 - k_scale,
575 - v_scale,
572 + layer._k_scale,
573 + layer._v_scale,
574 )
575
576 # Reshape the output tensor.
vllm/attention/backends/xformers.py
+9 -11
@@ -10,6 +10,7 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
10 LowerTriangularMaskWithTensorBias)
11
12 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 + AttentionLayer,
14 AttentionMetadata, AttentionType)
15 from vllm.attention.backends.utils import (
16 CommonAttentionState, CommonMetadataBuilder,
@@ -412,13 +413,12 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
413
414 def forward(
415 self,
416 + layer: AttentionLayer,
417 query: torch.Tensor,
418 key: Optional[torch.Tensor],
419 value: Optional[torch.Tensor],
420 kv_cache: torch.Tensor,
421 attn_metadata: "XFormersMetadata",
420 - k_scale: float = 1.0,
421 - v_scale: float = 1.0,
422 output: Optional[torch.Tensor] = None,
423 ) -> torch.Tensor:
424 """Forward pass with xFormers and PagedAttention.
@@ -524,11 +524,9 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
524 # If kv_cache is not provided, the new key and value tensors are
525 # not cached. This happens during the initial memory
526 # profiling run.
527 - PagedAttention.write_to_paged_cache(key, value, key_cache,
528 - value_cache,
529 - updated_slot_mapping,
530 - self.kv_cache_dtype,
531 - k_scale, v_scale)
527 + PagedAttention.write_to_paged_cache(
528 + key, value, key_cache, value_cache, updated_slot_mapping,
529 + self.kv_cache_dtype, layer._k_scale, layer._v_scale)
530 (num_prefill_query_tokens, num_prefill_kv_tokens,
531 num_decode_query_tokens) = \
532 get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
@@ -580,8 +578,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
578 prefill_meta.max_query_len,
579 self.alibi_slopes,
580 self.sliding_window,
583 - k_scale,
584 - v_scale,
581 + layer._k_scale,
582 + layer._v_scale,
583 )
584 assert output[:num_prefill_query_tokens].shape == out.shape
585 output[:num_prefill_query_tokens] = out
@@ -607,8 +605,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
605 self.num_kv_heads,
606 self.scale,
607 self.alibi_slopes,
610 - k_scale,
611 - v_scale,
608 + layer._k_scale,
609 + layer._v_scale,
610 )
611
612 # Reshape the output tensor.
vllm/attention/layer.py
+3 -5
@@ -243,8 +243,7 @@ def unified_attention(
243 attn_metadata = forward_context.attn_metadata
244 self = forward_context.attn_layers[layer_name]
245 kv_cache = self.kv_cache[forward_context.virtual_engine]
246 - return self.impl.forward(query, key, value, kv_cache, attn_metadata,
247 - self._k_scale, self._v_scale)
246 + return self.impl.forward(self, query, key, value, kv_cache, attn_metadata)
247
248
249 def unified_attention_fake(
@@ -276,13 +275,12 @@ def unified_attention_with_output(
275 attn_metadata = forward_context.attn_metadata
276 self = forward_context.attn_layers[layer_name]
277 kv_cache = self.kv_cache[forward_context.virtual_engine]
279 - self.impl.forward(query,
278 + self.impl.forward(self,
279 + query,
280 key,
281 value,
282 kv_cache,
283 attn_metadata,
284 - self._k_scale,
285 - self._v_scale,
284 output=output)
285
286
vllm/v1/attention/backends/flash_attn.py
+4 -5
@@ -130,13 +130,12 @@ class FlashAttentionImpl(AttentionImpl):
130
131 def forward(
132 self,
133 + layer: torch.nn.Module,
134 query: torch.Tensor,
135 key: torch.Tensor,
136 value: torch.Tensor,
137 kv_cache: torch.Tensor,
138 attn_metadata: FlashAttentionMetadata,
138 - k_scale: float = 1.0,
139 - v_scale: float = 1.0,
139 output: Optional[torch.Tensor] = None,
140 ) -> torch.Tensor:
141 """Forward pass with FlashAttention.
@@ -151,7 +150,7 @@ class FlashAttentionImpl(AttentionImpl):
150 shape = [num_tokens, num_heads * head_size]
151 """
152 # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
154 - assert k_scale == 1.0 and v_scale == 1.0, (
153 + assert layer._k_scale == 1.0 and layer._v_scale == 1.0, (
154 "key/v_scale is not supported in FlashAttention.")
155
156 assert output is not None, "Output tensor must be provided."
@@ -183,8 +182,8 @@ class FlashAttentionImpl(AttentionImpl):
182 value_cache,
183 attn_metadata.slot_mapping,
184 self.kv_cache_dtype,
186 - k_scale,
187 - v_scale,
185 + layer._k_scale,
186 + layer._v_scale,
187 )
188
189 # Compute attention and update output up to `num_actual_tokens`.