@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2e0e017610
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Platform] Add output for Attention Backend (#11981)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>

wangxiyuan committed Jan 14, 2025 at 21:27 UTC 2e0e0176104965c9c8c090609f331f9b70e492f3
4 files changed +9 -5
vllm/attention/backends/abstract.py
+4
@@ -31,6 +31,10 @@ class AttentionType:
31
32 class AttentionBackend(ABC):
33 """Abstract class for attention backends."""
34 + # For some attention backends, we allocate an output tensor before
35 + # calling the custom op. When piecewise cudagraph is enabled, this
36 + # makes sure the output tensor is allocated inside the cudagraph.
37 + accept_output_buffer: bool = False
38
39 @staticmethod
40 @abstractmethod
vllm/attention/backends/flash_attn.py
+2
@@ -29,6 +29,8 @@ from vllm.vllm_flash_attn import (flash_attn_varlen_func,
29
30 class FlashAttentionBackend(AttentionBackend):
31
32 + accept_output_buffer: bool = True
33 +
34 @staticmethod
35 def get_supported_head_sizes() -> List[int]:
36 return [32, 64, 96, 128, 160, 192, 224, 256]
vllm/attention/layer.py
+1 -5
@@ -110,11 +110,7 @@ class Attention(nn.Module):
110 self.use_direct_call = not current_platform.is_cuda_alike(
111 ) and not current_platform.is_cpu()
112
113 - # For some attention backends, we allocate an output tensor before
114 - # calling the custom op. When piecewise cudagraph is enabled, this
115 - # makes sure the output tensor is allocated inside the cudagraph.
116 - self.use_output = self.backend == _Backend.FLASH_ATTN or \
117 - self.backend == _Backend.FLASH_ATTN_VLLM_V1
113 + self.use_output = attn_backend.accept_output_buffer
114 compilation_config = get_current_vllm_config().compilation_config
115 if prefix in compilation_config.static_forward_context:
116 raise ValueError(f"Duplicate layer name: {prefix}")
vllm/v1/attention/backends/flash_attn.py
+2
@@ -15,6 +15,8 @@ from vllm.vllm_flash_attn import flash_attn_varlen_func
15
16 class FlashAttentionBackend(AttentionBackend):
17
18 + accept_output_buffer: bool = True
19 +
20 @staticmethod
21 def get_supported_head_sizes() -> List[int]:
22 return [32, 64, 96, 128, 160, 192, 224, 256]