A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Platform][Refactor] Extract func `get_default_attn_backend` to `Platform` (#10358)
Signed-off-by: Mengqing Cao <cmq0113@163.com>
Mengqing Cao committed
Nov 19, 2024 at 11:22 UTC
8c1fb507052d385d94ac49a7388fd6db5d0069e7
14 files changed
+99
-69
tests/kernels/test_attention_selector.py
+11
-8
@@ -5,6 +5,7 @@ import torch
5
6
from tests.kernels.utils import override_backend_env_variable
7
from vllm.attention.selector import which_attn_to_use
8
+from vllm.platforms import cpu, cuda, openvino, rocm
9
from vllm.utils import STR_FLASH_ATTN_VAL, STR_INVALID_VAL
10
11
@@ -19,26 +20,28 @@ def test_env(name: str, device: str, monkeypatch):
20
override_backend_env_variable(monkeypatch, name)
21
22
if device == "cpu":
22
- with patch("vllm.attention.selector.current_platform.is_cpu",
23
- return_value=True):
23
+ with patch("vllm.attention.selector.current_platform",
24
+ cpu.CpuPlatform()):
25
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
26
False)
27
assert backend.name == "TORCH_SDPA"
28
elif device == "hip":
28
- with patch("vllm.attention.selector.current_platform.is_rocm",
29
- return_value=True):
29
+ with patch("vllm.attention.selector.current_platform",
30
+ rocm.RocmPlatform()):
31
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
32
False)
33
assert backend.name == "ROCM_FLASH"
34
elif device == "openvino":
34
- with patch("vllm.attention.selector.current_platform.is_openvino",
35
- return_value=True):
35
+ with patch("vllm.attention.selector.current_platform",
36
+ openvino.OpenVinoPlatform()):
37
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
38
False)
39
assert backend.name == "OPENVINO"
40
else:
40
- backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
41
- False)
41
+ with patch("vllm.attention.selector.current_platform",
42
+ cuda.CudaPlatform()):
43
+ backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
44
+ False)
45
assert backend.name == name
46
47
vllm/attention/selector.py
+6
-50
@@ -1,4 +1,3 @@
1
-import enum
1
import os
2
from contextlib import contextmanager
3
from functools import lru_cache
@@ -9,26 +8,12 @@ import torch
8
import vllm.envs as envs
9
from vllm.attention.backends.abstract import AttentionBackend
10
from vllm.logger import init_logger
12
-from vllm.platforms import current_platform
11
+from vllm.platforms import _Backend, current_platform
12
from vllm.utils import STR_BACKEND_ENV_VAR
13
14
logger = init_logger(__name__)
15
16
18
-class _Backend(enum.Enum):
19
- FLASH_ATTN = enum.auto()
20
- FLASH_ATTN_VLLM_V1 = enum.auto()
21
- XFORMERS = enum.auto()
22
- ROCM_FLASH = enum.auto()
23
- TORCH_SDPA = enum.auto()
24
- OPENVINO = enum.auto()
25
- FLASHINFER = enum.auto()
26
- HPU_ATTN = enum.auto()
27
- PALLAS = enum.auto()
28
- IPEX = enum.auto()
29
- NO_ATTENTION = enum.auto()
30
-
31
-
17
def backend_name_to_enum(backend_name: str) -> _Backend:
18
assert backend_name is not None
19
@@ -216,40 +201,11 @@ def which_attn_to_use(head_size: int,
201
if backend_by_env_var is not None:
202
selected_backend = backend_name_to_enum(backend_by_env_var)
203
219
- if current_platform.is_cpu():
220
- if selected_backend != _Backend.TORCH_SDPA:
221
- logger.info("Cannot use %s backend on CPU.", selected_backend)
222
- return _Backend.TORCH_SDPA
223
-
224
- if current_platform.is_openvino():
225
- if selected_backend != _Backend.OPENVINO:
226
- logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
227
- return _Backend.OPENVINO
228
-
229
- if current_platform.is_xpu():
230
- if selected_backend != _Backend.IPEX:
231
- logger.info("Cannot use %s backend on XPU.", selected_backend)
232
- return _Backend.IPEX
233
-
234
- if current_platform.is_tpu():
235
- if selected_backend != _Backend.PALLAS:
236
- logger.info("Cannot use %s backend on TPU.", selected_backend)
237
- return _Backend.PALLAS
238
-
239
- if current_platform.is_rocm():
240
- # AMD GPUs.
241
- selected_backend = (_Backend.ROCM_FLASH if selected_backend
242
- == _Backend.FLASH_ATTN else selected_backend)
243
- if selected_backend == _Backend.ROCM_FLASH:
244
- if not current_platform.has_device_capability(90):
245
- # not Instinct series GPUs.
246
- logger.info("flash_attn is not supported on NAVI GPUs.")
247
- else:
248
- logger.info("%s is not supported in AMD GPUs.", selected_backend)
249
- return _Backend.ROCM_FLASH
250
-
251
- if current_platform.is_hpu():
252
- return _Backend.HPU_ATTN
204
+ # get device-specific default attn_backend
205
+ default_backend = current_platform.get_default_attn_backend(
206
+ selected_backend)
207
+ if default_backend is not None:
208
+ return default_backend
209
210
if use_v1:
211
return _Backend.FLASH_ATTN_VLLM_V1
vllm/model_executor/models/molmo.py
+1
-1
@@ -13,7 +13,6 @@ from torch.nn import functional as F
13
from transformers import PretrainedConfig
14
15
from vllm.attention import Attention, AttentionMetadata
16
-from vllm.attention.selector import _Backend
16
from vllm.compilation.decorators import support_torch_compile
17
from vllm.config import CacheConfig, VllmConfig
18
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
@@ -38,6 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
37
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
39
from vllm.multimodal.utils import cached_get_tokenizer
40
+from vllm.platforms import _Backend
41
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42
SequenceData)
43
from vllm.transformers_utils.processor import get_processor
vllm/model_executor/models/qwen2_vl.py
+1
-1
@@ -39,7 +39,6 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
39
make_batched_images, make_batched_videos, smart_resize)
40
41
from vllm.attention import AttentionMetadata
42
-from vllm.attention.selector import _Backend
42
from vllm.config import VllmConfig
43
from vllm.distributed import get_pp_group, parallel_state
44
from vllm.distributed import utils as dist_utils
@@ -65,6 +64,7 @@ from vllm.multimodal.image import cached_get_image_processor
64
from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
65
MultiModalKwargs)
66
from vllm.multimodal.utils import cached_get_tokenizer
67
+from vllm.platforms import _Backend
68
from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
69
from vllm.transformers_utils.config import uses_mrope
70
from vllm.transformers_utils.processor import cached_get_processor
vllm/model_executor/models/utils.py
+2
-2
@@ -9,13 +9,13 @@ from torch.func import functional_call
9
from transformers import PretrainedConfig
10
11
import vllm.envs as envs
12
-from vllm.attention.selector import (_Backend, backend_name_to_enum,
12
+from vllm.attention.selector import (backend_name_to_enum,
13
get_global_forced_attn_backend)
14
from vllm.config import VllmConfig
15
from vllm.logger import init_logger
16
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
from vllm.multimodal import MultiModalPlaceholderMap, NestedTensors
18
-from vllm.platforms import current_platform
18
+from vllm.platforms import _Backend, current_platform
19
from vllm.sequence import IntermediateTensors
20
from vllm.utils import is_pin_memory_available
21
vllm/platforms/__init__.py
+1
@@ -1,3 +1,4 @@
1
+from .interface import _Backend # noqa: F401
2
from .interface import Platform, PlatformEnum, UnspecifiedPlatform
3
4
current_platform: Platform
vllm/platforms/cpu.py
+9
-1
@@ -5,7 +5,9 @@ import torch
5
6
from vllm.logger import init_logger
7
8
-from .interface import Platform, PlatformEnum
8
+from .interface import Platform, PlatformEnum, _Backend
9
+
10
+logger = init_logger(__name__)
11
12
if TYPE_CHECKING:
13
from vllm.config import VllmConfig
@@ -22,6 +24,12 @@ class CpuPlatform(Platform):
24
def get_device_name(cls, device_id: int = 0) -> str:
25
return "cpu"
26
27
+ @classmethod
28
+ def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
29
+ if selected_backend != _Backend.TORCH_SDPA:
30
+ logger.info("Cannot use %s backend on CPU.", selected_backend)
31
+ return _Backend.TORCH_SDPA
32
+
33
@classmethod
34
def get_device_total_memory(cls, device_id: int = 0) -> int:
35
return psutil.virtual_memory().total
vllm/platforms/hpu.py
+5
-1
@@ -1,11 +1,15 @@
1
import torch
2
3
-from .interface import Platform, PlatformEnum
3
+from .interface import Platform, PlatformEnum, _Backend
4
5
6
class HpuPlatform(Platform):
7
_enum = PlatformEnum.HPU
8
9
+ @classmethod
10
+ def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
11
+ return _Backend.HPU_ATTN
12
+
13
@staticmethod
14
def inference_mode():
15
return torch.no_grad()
vllm/platforms/interface.py
+19
@@ -11,6 +11,20 @@ else:
11
VllmConfig = None
12
13
14
+class _Backend(enum.Enum):
15
+ FLASH_ATTN = enum.auto()
16
+ FLASH_ATTN_VLLM_V1 = enum.auto()
17
+ XFORMERS = enum.auto()
18
+ ROCM_FLASH = enum.auto()
19
+ TORCH_SDPA = enum.auto()
20
+ OPENVINO = enum.auto()
21
+ FLASHINFER = enum.auto()
22
+ HPU_ATTN = enum.auto()
23
+ PALLAS = enum.auto()
24
+ IPEX = enum.auto()
25
+ NO_ATTENTION = enum.auto()
26
+
27
+
28
class PlatformEnum(enum.Enum):
29
CUDA = enum.auto()
30
ROCM = enum.auto()
@@ -71,6 +85,11 @@ class Platform:
85
"""Stateless version of :func:`torch.cuda.is_available`."""
86
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
87
88
+ @classmethod
89
+ def get_default_attn_backend(cls, selected_backend: _Backend):
90
+ """Get the default attention backend of a device."""
91
+ return None
92
+
93
@classmethod
94
def get_device_capability(
95
cls,
vllm/platforms/openvino.py
+7
-1
@@ -3,7 +3,7 @@ import torch
3
import vllm.envs as envs
4
from vllm.logger import init_logger
5
6
-from .interface import Platform, PlatformEnum
6
+from .interface import Platform, PlatformEnum, _Backend
7
8
logger = init_logger(__name__)
9
@@ -11,6 +11,12 @@ logger = init_logger(__name__)
11
class OpenVinoPlatform(Platform):
12
_enum = PlatformEnum.OPENVINO
13
14
+ @classmethod
15
+ def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
16
+ if selected_backend != _Backend.OPENVINO:
17
+ logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
18
+ return _Backend.OPENVINO
19
+
20
@classmethod
21
def get_device_name(self, device_id: int = 0) -> str:
22
return "openvino"
vllm/platforms/rocm.py
+13
-1
@@ -5,7 +5,7 @@ import torch
5
6
from vllm.logger import init_logger
7
8
-from .interface import DeviceCapability, Platform, PlatformEnum
8
+from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
9
10
logger = init_logger(__name__)
11
@@ -19,6 +19,18 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
19
class RocmPlatform(Platform):
20
_enum = PlatformEnum.ROCM
21
22
+ @classmethod
23
+ def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
24
+ selected_backend = (_Backend.ROCM_FLASH if selected_backend
25
+ == _Backend.FLASH_ATTN else selected_backend)
26
+ if selected_backend == _Backend.ROCM_FLASH:
27
+ if not cls.has_device_capability(90):
28
+ # not Instinct series GPUs.
29
+ logger.info("flash_attn is not supported on NAVI GPUs.")
30
+ else:
31
+ logger.info("%s is not supported in AMD GPUs.", selected_backend)
32
+ return _Backend.ROCM_FLASH
33
+
34
@classmethod
35
@lru_cache(maxsize=8)
36
def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
vllm/platforms/tpu.py
+11
-1
@@ -3,17 +3,27 @@ from typing import TYPE_CHECKING
3
4
import torch
5
6
-from .interface import Platform, PlatformEnum
6
+from vllm.logger import init_logger
7
+
8
+from .interface import Platform, PlatformEnum, _Backend
9
10
if TYPE_CHECKING:
11
from vllm.config import VllmConfig
12
else:
13
VllmConfig = None
14
15
+logger = init_logger(__name__)
16
+
17
18
class TpuPlatform(Platform):
19
_enum = PlatformEnum.TPU
20
21
+ @classmethod
22
+ def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
23
+ if selected_backend != _Backend.PALLAS:
24
+ logger.info("Cannot use %s backend on TPU.", selected_backend)
25
+ return _Backend.PALLAS
26
+
27
@classmethod
28
def get_device_name(cls, device_id: int = 0) -> str:
29
raise NotImplementedError
vllm/platforms/xpu.py
+11
-1
@@ -1,11 +1,21 @@
1
import torch
2
3
-from .interface import DeviceCapability, Platform, PlatformEnum
3
+from vllm.logger import init_logger
4
+
5
+from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
6
+
7
+logger = init_logger(__name__)
8
9
10
class XPUPlatform(Platform):
11
_enum = PlatformEnum.XPU
12
13
+ @classmethod
14
+ def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
15
+ if selected_backend != _Backend.IPEX:
16
+ logger.info("Cannot use %s backend on XPU.", selected_backend)
17
+ return _Backend.IPEX
18
+
19
@staticmethod
20
def get_device_capability(device_id: int = 0) -> DeviceCapability:
21
major, minor, *_ = torch.xpu.get_device_capability(
vllm/worker/enc_dec_model_runner.py
+2
-1
@@ -8,7 +8,7 @@ import torch.distributed
8
from vllm.attention.backends.abstract import (AttentionBackend,
9
AttentionMetadata)
10
from vllm.attention.backends.utils import PAD_SLOT_ID
11
-from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
11
+from vllm.attention.selector import (get_env_variable_attn_backend,
12
get_global_forced_attn_backend)
13
from vllm.config import VllmConfig
14
from vllm.forward_context import set_forward_context
@@ -18,6 +18,7 @@ from vllm.model_executor import SamplingMetadata
18
from vllm.model_executor.layers.sampler import SamplerOutput
19
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
20
MultiModalRegistry)
21
+from vllm.platforms import _Backend
22
from vllm.sampling_params import SamplingParams
23
from vllm.sequence import (IntermediateTensors, PoolerOutput,
24
SequenceGroupMetadata)