A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Add `get_name` method to attention backends (#4685)
Woosuk Kwon committed
May 8, 2024 at 09:59 UTC
5510cf0e8a6a3ee56daefb86b145c7f2a000817f
7 files changed
+30
-12
vllm/attention/backends/abstract.py
+5
@@ -9,6 +9,11 @@ import torch
9
class AttentionBackend(ABC):
10
"""Abstract class for attention backends."""
11
12
+ @staticmethod
13
+ @abstractmethod
14
+ def get_name() -> str:
15
+ raise NotImplementedError
16
+
17
@staticmethod
18
@abstractmethod
19
def get_impl_cls() -> Type["AttentionImpl"]:
vllm/attention/backends/flash_attn.py
+4
@@ -19,6 +19,10 @@ from vllm.attention.ops.paged_attn import (PagedAttention,
19
20
class FlashAttentionBackend(AttentionBackend):
21
22
+ @staticmethod
23
+ def get_name() -> str:
24
+ return "flash-attn"
25
+
26
@staticmethod
27
def get_impl_cls() -> Type["FlashAttentionImpl"]:
28
return FlashAttentionImpl
vllm/attention/backends/flashinfer.py
+7
-9
@@ -1,16 +1,10 @@
1
from dataclasses import dataclass
2
from typing import Any, Dict, List, Optional, Set, Tuple, Type
3
4
-try:
5
- import flashinfer
6
- from flash_attn import flash_attn_varlen_func
7
- from flashinfer import BatchDecodeWithPagedKVCacheWrapper
8
-except ImportError:
9
- flashinfer = None
10
- flash_attn_varlen_func = None
11
- BatchDecodeWithPagedKVCacheWrapper = None
12
-
4
+import flashinfer
5
import torch
6
+from flash_attn import flash_attn_varlen_func
7
+from flashinfer import BatchDecodeWithPagedKVCacheWrapper
8
9
from vllm import _custom_ops as ops
10
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
@@ -20,6 +14,10 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
15
class FlashInferBackend(AttentionBackend):
16
17
+ @staticmethod
18
+ def get_name() -> str:
19
+ return "flashinfer"
20
+
21
@staticmethod
22
def get_impl_cls() -> Type["FlashInferImpl"]:
23
return FlashInferImpl
vllm/attention/backends/rocm_flash_attn.py
+4
@@ -17,6 +17,10 @@ logger = init_logger(__name__)
17
18
class ROCmFlashAttentionBackend(AttentionBackend):
19
20
+ @staticmethod
21
+ def get_name() -> str:
22
+ return "rocm-flash-attn"
23
+
24
@staticmethod
25
def get_impl_cls() -> Type["ROCmFlashAttentionImpl"]:
26
return ROCmFlashAttentionImpl
vllm/attention/backends/torch_sdpa.py
+4
@@ -15,6 +15,10 @@ from vllm.attention.ops.paged_attn import (PagedAttention,
15
16
class TorchSDPABackend(AttentionBackend):
17
18
+ @staticmethod
19
+ def get_name() -> str:
20
+ return "torch-sdpa"
21
+
22
@staticmethod
23
def get_impl_cls() -> Type["TorchSDPABackendImpl"]:
24
return TorchSDPABackendImpl
vllm/attention/backends/xformers.py
+4
@@ -20,6 +20,10 @@ logger = init_logger(__name__)
20
21
class XFormersBackend(AttentionBackend):
22
23
+ @staticmethod
24
+ def get_name() -> str:
25
+ return "xformers"
26
+
27
@staticmethod
28
def get_impl_cls() -> Type["XFormersImpl"]:
29
return XFormersImpl
vllm/worker/model_runner.py
+2
-3
@@ -9,7 +9,6 @@ import torch.nn as nn
9
10
from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11
get_attn_backend)
12
-from vllm.attention.backends.flashinfer import FlashInferBackend
12
from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
13
ParallelConfig, SchedulerConfig, VisionLanguageConfig)
14
from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
@@ -395,7 +394,7 @@ class ModelRunner:
394
dtype=seq_start_loc.dtype,
395
out=seq_start_loc[1:])
396
398
- if self.attn_backend is FlashInferBackend:
397
+ if self.attn_backend.get_name() == "flashinfer":
398
attn_metadata = self.attn_backend.make_metadata(
399
is_prompt=True,
400
use_cuda_graph=False,
@@ -556,7 +555,7 @@ class ModelRunner:
555
device=self.device,
556
)
557
559
- if self.attn_backend is FlashInferBackend:
558
+ if self.attn_backend.get_name() == "flashinfer":
559
if not hasattr(self, "flashinfer_workspace_buffer"):
560
# Allocate 16MB workspace buffer
561
# Follow the example of flashinfer: https://docs.flashinfer.ai/api/python/decode.html