@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5510cf0e8a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Add `get_name` method to attention backends (#4685)

Woosuk Kwon committed May 8, 2024 at 09:59 UTC 5510cf0e8a6a3ee56daefb86b145c7f2a000817f
7 files changed +30 -12
vllm/attention/backends/abstract.py
+5
@@ -9,6 +9,11 @@ import torch
9 class AttentionBackend(ABC):
10 """Abstract class for attention backends."""
11
12 + @staticmethod
13 + @abstractmethod
14 + def get_name() -> str:
15 + raise NotImplementedError
16 +
17 @staticmethod
18 @abstractmethod
19 def get_impl_cls() -> Type["AttentionImpl"]:
vllm/attention/backends/flash_attn.py
+4
@@ -19,6 +19,10 @@ from vllm.attention.ops.paged_attn import (PagedAttention,
19
20 class FlashAttentionBackend(AttentionBackend):
21
22 + @staticmethod
23 + def get_name() -> str:
24 + return "flash-attn"
25 +
26 @staticmethod
27 def get_impl_cls() -> Type["FlashAttentionImpl"]:
28 return FlashAttentionImpl
vllm/attention/backends/flashinfer.py
+7 -9
@@ -1,16 +1,10 @@
1 from dataclasses import dataclass
2 from typing import Any, Dict, List, Optional, Set, Tuple, Type
3
4 -try:
5 - import flashinfer
6 - from flash_attn import flash_attn_varlen_func
7 - from flashinfer import BatchDecodeWithPagedKVCacheWrapper
8 -except ImportError:
9 - flashinfer = None
10 - flash_attn_varlen_func = None
11 - BatchDecodeWithPagedKVCacheWrapper = None
12 -
4 +import flashinfer
5 import torch
6 +from flash_attn import flash_attn_varlen_func
7 +from flashinfer import BatchDecodeWithPagedKVCacheWrapper
8
9 from vllm import _custom_ops as ops
10 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
@@ -20,6 +14,10 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
15 class FlashInferBackend(AttentionBackend):
16
17 + @staticmethod
18 + def get_name() -> str:
19 + return "flashinfer"
20 +
21 @staticmethod
22 def get_impl_cls() -> Type["FlashInferImpl"]:
23 return FlashInferImpl
vllm/attention/backends/rocm_flash_attn.py
+4
@@ -17,6 +17,10 @@ logger = init_logger(__name__)
17
18 class ROCmFlashAttentionBackend(AttentionBackend):
19
20 + @staticmethod
21 + def get_name() -> str:
22 + return "rocm-flash-attn"
23 +
24 @staticmethod
25 def get_impl_cls() -> Type["ROCmFlashAttentionImpl"]:
26 return ROCmFlashAttentionImpl
vllm/attention/backends/torch_sdpa.py
+4
@@ -15,6 +15,10 @@ from vllm.attention.ops.paged_attn import (PagedAttention,
15
16 class TorchSDPABackend(AttentionBackend):
17
18 + @staticmethod
19 + def get_name() -> str:
20 + return "torch-sdpa"
21 +
22 @staticmethod
23 def get_impl_cls() -> Type["TorchSDPABackendImpl"]:
24 return TorchSDPABackendImpl
vllm/attention/backends/xformers.py
+4
@@ -20,6 +20,10 @@ logger = init_logger(__name__)
20
21 class XFormersBackend(AttentionBackend):
22
23 + @staticmethod
24 + def get_name() -> str:
25 + return "xformers"
26 +
27 @staticmethod
28 def get_impl_cls() -> Type["XFormersImpl"]:
29 return XFormersImpl
vllm/worker/model_runner.py
+2 -3
@@ -9,7 +9,6 @@ import torch.nn as nn
9
10 from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11 get_attn_backend)
12 -from vllm.attention.backends.flashinfer import FlashInferBackend
12 from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
13 ParallelConfig, SchedulerConfig, VisionLanguageConfig)
14 from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
@@ -395,7 +394,7 @@ class ModelRunner:
394 dtype=seq_start_loc.dtype,
395 out=seq_start_loc[1:])
396
398 - if self.attn_backend is FlashInferBackend:
397 + if self.attn_backend.get_name() == "flashinfer":
398 attn_metadata = self.attn_backend.make_metadata(
399 is_prompt=True,
400 use_cuda_graph=False,
@@ -556,7 +555,7 @@ class ModelRunner:
555 device=self.device,
556 )
557
559 - if self.attn_backend is FlashInferBackend:
558 + if self.attn_backend.get_name() == "flashinfer":
559 if not hasattr(self, "flashinfer_workspace_buffer"):
560 # Allocate 16MB workspace buffer
561 # Follow the example of flashinfer: https://docs.flashinfer.ai/api/python/decode.html