@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c1fb50705
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Platform][Refactor] Extract func `get_default_attn_backend` to `Platform` (#10358)

Signed-off-by: Mengqing Cao <cmq0113@163.com>

Mengqing Cao committed Nov 19, 2024 at 11:22 UTC 8c1fb507052d385d94ac49a7388fd6db5d0069e7
14 files changed +99 -69
tests/kernels/test_attention_selector.py
+11 -8
@@ -5,6 +5,7 @@ import torch
5
6 from tests.kernels.utils import override_backend_env_variable
7 from vllm.attention.selector import which_attn_to_use
8 +from vllm.platforms import cpu, cuda, openvino, rocm
9 from vllm.utils import STR_FLASH_ATTN_VAL, STR_INVALID_VAL
10
11
@@ -19,26 +20,28 @@ def test_env(name: str, device: str, monkeypatch):
20 override_backend_env_variable(monkeypatch, name)
21
22 if device == "cpu":
22 - with patch("vllm.attention.selector.current_platform.is_cpu",
23 - return_value=True):
23 + with patch("vllm.attention.selector.current_platform",
24 + cpu.CpuPlatform()):
25 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
26 False)
27 assert backend.name == "TORCH_SDPA"
28 elif device == "hip":
28 - with patch("vllm.attention.selector.current_platform.is_rocm",
29 - return_value=True):
29 + with patch("vllm.attention.selector.current_platform",
30 + rocm.RocmPlatform()):
31 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
32 False)
33 assert backend.name == "ROCM_FLASH"
34 elif device == "openvino":
34 - with patch("vllm.attention.selector.current_platform.is_openvino",
35 - return_value=True):
35 + with patch("vllm.attention.selector.current_platform",
36 + openvino.OpenVinoPlatform()):
37 backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
38 False)
39 assert backend.name == "OPENVINO"
40 else:
40 - backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
41 - False)
41 + with patch("vllm.attention.selector.current_platform",
42 + cuda.CudaPlatform()):
43 + backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
44 + False)
45 assert backend.name == name
46
47
vllm/attention/selector.py
+6 -50
@@ -1,4 +1,3 @@
1 -import enum
1 import os
2 from contextlib import contextmanager
3 from functools import lru_cache
@@ -9,26 +8,12 @@ import torch
8 import vllm.envs as envs
9 from vllm.attention.backends.abstract import AttentionBackend
10 from vllm.logger import init_logger
12 -from vllm.platforms import current_platform
11 +from vllm.platforms import _Backend, current_platform
12 from vllm.utils import STR_BACKEND_ENV_VAR
13
14 logger = init_logger(__name__)
15
16
18 -class _Backend(enum.Enum):
19 - FLASH_ATTN = enum.auto()
20 - FLASH_ATTN_VLLM_V1 = enum.auto()
21 - XFORMERS = enum.auto()
22 - ROCM_FLASH = enum.auto()
23 - TORCH_SDPA = enum.auto()
24 - OPENVINO = enum.auto()
25 - FLASHINFER = enum.auto()
26 - HPU_ATTN = enum.auto()
27 - PALLAS = enum.auto()
28 - IPEX = enum.auto()
29 - NO_ATTENTION = enum.auto()
30 -
31 -
17 def backend_name_to_enum(backend_name: str) -> _Backend:
18 assert backend_name is not None
19
@@ -216,40 +201,11 @@ def which_attn_to_use(head_size: int,
201 if backend_by_env_var is not None:
202 selected_backend = backend_name_to_enum(backend_by_env_var)
203
219 - if current_platform.is_cpu():
220 - if selected_backend != _Backend.TORCH_SDPA:
221 - logger.info("Cannot use %s backend on CPU.", selected_backend)
222 - return _Backend.TORCH_SDPA
223 -
224 - if current_platform.is_openvino():
225 - if selected_backend != _Backend.OPENVINO:
226 - logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
227 - return _Backend.OPENVINO
228 -
229 - if current_platform.is_xpu():
230 - if selected_backend != _Backend.IPEX:
231 - logger.info("Cannot use %s backend on XPU.", selected_backend)
232 - return _Backend.IPEX
233 -
234 - if current_platform.is_tpu():
235 - if selected_backend != _Backend.PALLAS:
236 - logger.info("Cannot use %s backend on TPU.", selected_backend)
237 - return _Backend.PALLAS
238 -
239 - if current_platform.is_rocm():
240 - # AMD GPUs.
241 - selected_backend = (_Backend.ROCM_FLASH if selected_backend
242 - == _Backend.FLASH_ATTN else selected_backend)
243 - if selected_backend == _Backend.ROCM_FLASH:
244 - if not current_platform.has_device_capability(90):
245 - # not Instinct series GPUs.
246 - logger.info("flash_attn is not supported on NAVI GPUs.")
247 - else:
248 - logger.info("%s is not supported in AMD GPUs.", selected_backend)
249 - return _Backend.ROCM_FLASH
250 -
251 - if current_platform.is_hpu():
252 - return _Backend.HPU_ATTN
204 + # get device-specific default attn_backend
205 + default_backend = current_platform.get_default_attn_backend(
206 + selected_backend)
207 + if default_backend is not None:
208 + return default_backend
209
210 if use_v1:
211 return _Backend.FLASH_ATTN_VLLM_V1
vllm/model_executor/models/molmo.py
+1 -1
@@ -13,7 +13,6 @@ from torch.nn import functional as F
13 from transformers import PretrainedConfig
14
15 from vllm.attention import Attention, AttentionMetadata
16 -from vllm.attention.selector import _Backend
16 from vllm.compilation.decorators import support_torch_compile
17 from vllm.config import CacheConfig, VllmConfig
18 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
@@ -38,6 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
37 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
39 from vllm.multimodal.utils import cached_get_tokenizer
40 +from vllm.platforms import _Backend
41 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42 SequenceData)
43 from vllm.transformers_utils.processor import get_processor
vllm/model_executor/models/qwen2_vl.py
+1 -1
@@ -39,7 +39,6 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
39 make_batched_images, make_batched_videos, smart_resize)
40
41 from vllm.attention import AttentionMetadata
42 -from vllm.attention.selector import _Backend
42 from vllm.config import VllmConfig
43 from vllm.distributed import get_pp_group, parallel_state
44 from vllm.distributed import utils as dist_utils
@@ -65,6 +64,7 @@ from vllm.multimodal.image import cached_get_image_processor
64 from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
65 MultiModalKwargs)
66 from vllm.multimodal.utils import cached_get_tokenizer
67 +from vllm.platforms import _Backend
68 from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
69 from vllm.transformers_utils.config import uses_mrope
70 from vllm.transformers_utils.processor import cached_get_processor
vllm/model_executor/models/utils.py
+2 -2
@@ -9,13 +9,13 @@ from torch.func import functional_call
9 from transformers import PretrainedConfig
10
11 import vllm.envs as envs
12 -from vllm.attention.selector import (_Backend, backend_name_to_enum,
12 +from vllm.attention.selector import (backend_name_to_enum,
13 get_global_forced_attn_backend)
14 from vllm.config import VllmConfig
15 from vllm.logger import init_logger
16 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 from vllm.multimodal import MultiModalPlaceholderMap, NestedTensors
18 -from vllm.platforms import current_platform
18 +from vllm.platforms import _Backend, current_platform
19 from vllm.sequence import IntermediateTensors
20 from vllm.utils import is_pin_memory_available
21
vllm/platforms/__init__.py
+1
@@ -1,3 +1,4 @@
1 +from .interface import _Backend # noqa: F401
2 from .interface import Platform, PlatformEnum, UnspecifiedPlatform
3
4 current_platform: Platform
vllm/platforms/cpu.py
+9 -1
@@ -5,7 +5,9 @@ import torch
5
6 from vllm.logger import init_logger
7
8 -from .interface import Platform, PlatformEnum
8 +from .interface import Platform, PlatformEnum, _Backend
9 +
10 +logger = init_logger(__name__)
11
12 if TYPE_CHECKING:
13 from vllm.config import VllmConfig
@@ -22,6 +24,12 @@ class CpuPlatform(Platform):
24 def get_device_name(cls, device_id: int = 0) -> str:
25 return "cpu"
26
27 + @classmethod
28 + def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
29 + if selected_backend != _Backend.TORCH_SDPA:
30 + logger.info("Cannot use %s backend on CPU.", selected_backend)
31 + return _Backend.TORCH_SDPA
32 +
33 @classmethod
34 def get_device_total_memory(cls, device_id: int = 0) -> int:
35 return psutil.virtual_memory().total
vllm/platforms/hpu.py
+5 -1
@@ -1,11 +1,15 @@
1 import torch
2
3 -from .interface import Platform, PlatformEnum
3 +from .interface import Platform, PlatformEnum, _Backend
4
5
6 class HpuPlatform(Platform):
7 _enum = PlatformEnum.HPU
8
9 + @classmethod
10 + def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
11 + return _Backend.HPU_ATTN
12 +
13 @staticmethod
14 def inference_mode():
15 return torch.no_grad()
vllm/platforms/interface.py
+19
@@ -11,6 +11,20 @@ else:
11 VllmConfig = None
12
13
14 +class _Backend(enum.Enum):
15 + FLASH_ATTN = enum.auto()
16 + FLASH_ATTN_VLLM_V1 = enum.auto()
17 + XFORMERS = enum.auto()
18 + ROCM_FLASH = enum.auto()
19 + TORCH_SDPA = enum.auto()
20 + OPENVINO = enum.auto()
21 + FLASHINFER = enum.auto()
22 + HPU_ATTN = enum.auto()
23 + PALLAS = enum.auto()
24 + IPEX = enum.auto()
25 + NO_ATTENTION = enum.auto()
26 +
27 +
28 class PlatformEnum(enum.Enum):
29 CUDA = enum.auto()
30 ROCM = enum.auto()
@@ -71,6 +85,11 @@ class Platform:
85 """Stateless version of :func:`torch.cuda.is_available`."""
86 return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
87
88 + @classmethod
89 + def get_default_attn_backend(cls, selected_backend: _Backend):
90 + """Get the default attention backend of a device."""
91 + return None
92 +
93 @classmethod
94 def get_device_capability(
95 cls,
vllm/platforms/openvino.py
+7 -1
@@ -3,7 +3,7 @@ import torch
3 import vllm.envs as envs
4 from vllm.logger import init_logger
5
6 -from .interface import Platform, PlatformEnum
6 +from .interface import Platform, PlatformEnum, _Backend
7
8 logger = init_logger(__name__)
9
@@ -11,6 +11,12 @@ logger = init_logger(__name__)
11 class OpenVinoPlatform(Platform):
12 _enum = PlatformEnum.OPENVINO
13
14 + @classmethod
15 + def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
16 + if selected_backend != _Backend.OPENVINO:
17 + logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
18 + return _Backend.OPENVINO
19 +
20 @classmethod
21 def get_device_name(self, device_id: int = 0) -> str:
22 return "openvino"
vllm/platforms/rocm.py
+13 -1
@@ -5,7 +5,7 @@ import torch
5
6 from vllm.logger import init_logger
7
8 -from .interface import DeviceCapability, Platform, PlatformEnum
8 +from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
9
10 logger = init_logger(__name__)
11
@@ -19,6 +19,18 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
19 class RocmPlatform(Platform):
20 _enum = PlatformEnum.ROCM
21
22 + @classmethod
23 + def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
24 + selected_backend = (_Backend.ROCM_FLASH if selected_backend
25 + == _Backend.FLASH_ATTN else selected_backend)
26 + if selected_backend == _Backend.ROCM_FLASH:
27 + if not cls.has_device_capability(90):
28 + # not Instinct series GPUs.
29 + logger.info("flash_attn is not supported on NAVI GPUs.")
30 + else:
31 + logger.info("%s is not supported in AMD GPUs.", selected_backend)
32 + return _Backend.ROCM_FLASH
33 +
34 @classmethod
35 @lru_cache(maxsize=8)
36 def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
vllm/platforms/tpu.py
+11 -1
@@ -3,17 +3,27 @@ from typing import TYPE_CHECKING
3
4 import torch
5
6 -from .interface import Platform, PlatformEnum
6 +from vllm.logger import init_logger
7 +
8 +from .interface import Platform, PlatformEnum, _Backend
9
10 if TYPE_CHECKING:
11 from vllm.config import VllmConfig
12 else:
13 VllmConfig = None
14
15 +logger = init_logger(__name__)
16 +
17
18 class TpuPlatform(Platform):
19 _enum = PlatformEnum.TPU
20
21 + @classmethod
22 + def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
23 + if selected_backend != _Backend.PALLAS:
24 + logger.info("Cannot use %s backend on TPU.", selected_backend)
25 + return _Backend.PALLAS
26 +
27 @classmethod
28 def get_device_name(cls, device_id: int = 0) -> str:
29 raise NotImplementedError
vllm/platforms/xpu.py
+11 -1
@@ -1,11 +1,21 @@
1 import torch
2
3 -from .interface import DeviceCapability, Platform, PlatformEnum
3 +from vllm.logger import init_logger
4 +
5 +from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
6 +
7 +logger = init_logger(__name__)
8
9
10 class XPUPlatform(Platform):
11 _enum = PlatformEnum.XPU
12
13 + @classmethod
14 + def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
15 + if selected_backend != _Backend.IPEX:
16 + logger.info("Cannot use %s backend on XPU.", selected_backend)
17 + return _Backend.IPEX
18 +
19 @staticmethod
20 def get_device_capability(device_id: int = 0) -> DeviceCapability:
21 major, minor, *_ = torch.xpu.get_device_capability(
vllm/worker/enc_dec_model_runner.py
+2 -1
@@ -8,7 +8,7 @@ import torch.distributed
8 from vllm.attention.backends.abstract import (AttentionBackend,
9 AttentionMetadata)
10 from vllm.attention.backends.utils import PAD_SLOT_ID
11 -from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
11 +from vllm.attention.selector import (get_env_variable_attn_backend,
12 get_global_forced_attn_backend)
13 from vllm.config import VllmConfig
14 from vllm.forward_context import set_forward_context
@@ -18,6 +18,7 @@ from vllm.model_executor import SamplingMetadata
18 from vllm.model_executor.layers.sampler import SamplerOutput
19 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
20 MultiModalRegistry)
21 +from vllm.platforms import _Backend
22 from vllm.sampling_params import SamplingParams
23 from vllm.sequence import (IntermediateTensors, PoolerOutput,
24 SequenceGroupMetadata)