A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Platform] Add output for Attention Backend (#11981)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
wangxiyuan committed
Jan 14, 2025 at 21:27 UTC
2e0e0176104965c9c8c090609f331f9b70e492f3
4 files changed
+9
-5
vllm/attention/backends/abstract.py
+4
@@ -31,6 +31,10 @@ class AttentionType:
31
32
class AttentionBackend(ABC):
33
"""Abstract class for attention backends."""
34
+ # For some attention backends, we allocate an output tensor before
35
+ # calling the custom op. When piecewise cudagraph is enabled, this
36
+ # makes sure the output tensor is allocated inside the cudagraph.
37
+ accept_output_buffer: bool = False
38
39
@staticmethod
40
@abstractmethod
vllm/attention/backends/flash_attn.py
+2
@@ -29,6 +29,8 @@ from vllm.vllm_flash_attn import (flash_attn_varlen_func,
29
30
class FlashAttentionBackend(AttentionBackend):
31
32
+ accept_output_buffer: bool = True
33
+
34
@staticmethod
35
def get_supported_head_sizes() -> List[int]:
36
return [32, 64, 96, 128, 160, 192, 224, 256]
vllm/attention/layer.py
+1
-5
@@ -110,11 +110,7 @@ class Attention(nn.Module):
110
self.use_direct_call = not current_platform.is_cuda_alike(
111
) and not current_platform.is_cpu()
112
113
- # For some attention backends, we allocate an output tensor before
114
- # calling the custom op. When piecewise cudagraph is enabled, this
115
- # makes sure the output tensor is allocated inside the cudagraph.
116
- self.use_output = self.backend == _Backend.FLASH_ATTN or \
117
- self.backend == _Backend.FLASH_ATTN_VLLM_V1
113
+ self.use_output = attn_backend.accept_output_buffer
114
compilation_config = get_current_vllm_config().compilation_config
115
if prefix in compilation_config.static_forward_context:
116
raise ValueError(f"Duplicate layer name: {prefix}")
vllm/v1/attention/backends/flash_attn.py
+2
@@ -15,6 +15,8 @@ from vllm.vllm_flash_attn import flash_attn_varlen_func
15
16
class FlashAttentionBackend(AttentionBackend):
17
18
+ accept_output_buffer: bool = True
19
+
20
@staticmethod
21
def get_supported_head_sizes() -> List[int]:
22
return [32, 64, 96, 128, 160, 192, 224, 256]