A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[platform] Allow platform specify attention backend (#11609)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com> Signed-off-by: Mengqing Cao <cmq0113@163.com> Co-authored-by: Mengqing Cao <cmq0113@163.com>
wangxiyuan committed
Jan 9, 2025 at 21:46 UTC
405eb8e3967eb9bd263b3919796cb3b45a2931d3
10 files changed
+164
-175
tests/kernels/test_attention_selector.py
+42
-32
@@ -1,10 +1,10 @@
1
-from unittest.mock import patch
1
+from unittest.mock import Mock, patch
2
3
import pytest
4
import torch
5
6
from tests.kernels.utils import override_backend_env_variable
7
-from vllm.attention.selector import which_attn_to_use
7
+from vllm.attention.selector import _cached_get_attn_backend, get_attn_backend
8
from vllm.platforms.cpu import CpuPlatform
9
from vllm.platforms.cuda import CudaPlatform
10
from vllm.platforms.openvino import OpenVinoPlatform
@@ -12,6 +12,13 @@ from vllm.platforms.rocm import RocmPlatform
12
from vllm.utils import STR_FLASH_ATTN_VAL, STR_INVALID_VAL
13
14
15
+@pytest.fixture(autouse=True)
16
+def clear_cache():
17
+ """Clear lru cache to ensure each test case runs without caching.
18
+ """
19
+ _cached_get_attn_backend.cache_clear()
20
+
21
+
22
@pytest.mark.parametrize(
23
"name", ["TORCH_SDPA", "ROCM_FLASH", "XFORMERS", "FLASHINFER", "OPENVINO"])
24
@pytest.mark.parametrize("device", ["cpu", "openvino", "hip", "cuda"])
@@ -24,67 +31,70 @@ def test_env(name: str, device: str, monkeypatch):
31
32
if device == "cpu":
33
with patch("vllm.attention.selector.current_platform", CpuPlatform()):
27
- backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
28
- False)
29
- assert backend.name == "TORCH_SDPA"
34
+ backend = get_attn_backend(16, torch.float16, torch.float16, 16,
35
+ False)
36
+ assert backend.get_name() == "TORCH_SDPA"
37
elif device == "hip":
38
with patch("vllm.attention.selector.current_platform", RocmPlatform()):
32
- backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
33
- False)
34
- assert backend.name == "ROCM_FLASH"
39
+ backend = get_attn_backend(16, torch.float16, torch.float16, 16,
40
+ False)
41
+ assert backend.get_name() == "ROCM_FLASH"
42
elif device == "openvino":
43
with patch("vllm.attention.selector.current_platform",
37
- OpenVinoPlatform()):
38
- backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
39
- False)
40
- assert backend.name == "OPENVINO"
44
+ OpenVinoPlatform()), patch.dict('sys.modules',
45
+ {'openvino': Mock()}):
46
+ backend = get_attn_backend(16, torch.float16, torch.float16, 16,
47
+ False)
48
+ assert backend.get_name() == "OPENVINO"
49
else:
42
- with patch("vllm.attention.selector.current_platform", CudaPlatform()):
43
- backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
44
- False)
45
- assert backend.name == name
50
+ if name in ["XFORMERS", "FLASHINFER"]:
51
+ with patch("vllm.attention.selector.current_platform",
52
+ CudaPlatform()):
53
+ backend = get_attn_backend(16, torch.float16, torch.float16,
54
+ 16, False)
55
+ assert backend.get_name() == name
56
57
58
def test_flash_attn(monkeypatch):
59
"""Test FlashAttn validation."""
60
# TODO: When testing for v1, pipe in `use_v1` as an argument to
51
- # which_attn_to_use
61
+ # get_attn_backend
62
63
override_backend_env_variable(monkeypatch, STR_FLASH_ATTN_VAL)
64
65
# Unsupported CUDA arch
66
with patch("torch.cuda.get_device_capability", return_value=(7, 5)):
57
- backend = which_attn_to_use(16, torch.float16, None, 16, False)
58
- assert backend.name != STR_FLASH_ATTN_VAL
67
+ backend = get_attn_backend(16, torch.float16, None, 16, False)
68
+ assert backend.get_name() != STR_FLASH_ATTN_VAL
69
70
# Unsupported data type
61
- backend = which_attn_to_use(16, torch.float8_e4m3fn, None, 16, False)
62
- assert backend.name != STR_FLASH_ATTN_VAL
71
+ backend = get_attn_backend(16, torch.float8_e4m3fn, None, 16, False)
72
+ assert backend.get_name() != STR_FLASH_ATTN_VAL
73
74
# Unsupported kv cache data type
65
- backend = which_attn_to_use(16, torch.float16, "fp8", 16, False)
66
- assert backend.name != STR_FLASH_ATTN_VAL
75
+ backend = get_attn_backend(16, torch.float16, "fp8", 16, False)
76
+ assert backend.get_name() != STR_FLASH_ATTN_VAL
77
78
# Unsupported block size
69
- backend = which_attn_to_use(16, torch.float16, None, 8, False)
70
- assert backend.name != STR_FLASH_ATTN_VAL
79
+ backend = get_attn_backend(16, torch.float16, None, 8, False)
80
+ assert backend.get_name() != STR_FLASH_ATTN_VAL
81
82
# flash-attn is not installed
83
with patch.dict('sys.modules', {'vllm_flash_attn': None}):
74
- backend = which_attn_to_use(16, torch.float16, None, 16, False)
75
- assert backend.name != STR_FLASH_ATTN_VAL
84
+ backend = get_attn_backend(16, torch.float16, None, 16, False)
85
+ assert backend.get_name() != STR_FLASH_ATTN_VAL
86
87
# Unsupported head size
78
- backend = which_attn_to_use(17, torch.float16, None, 16, False)
79
- assert backend.name != STR_FLASH_ATTN_VAL
88
+ backend = get_attn_backend(17, torch.float16, None, 16, False)
89
+ assert backend.get_name() != STR_FLASH_ATTN_VAL
90
91
# Attention-free models should bypass env and use PlaceholderAttention
82
- backend = which_attn_to_use(16, torch.float16, torch.float16, 16, True)
83
- assert backend.name != STR_FLASH_ATTN_VAL
92
+ backend = get_attn_backend(16, torch.float16, torch.float16, 16, True)
93
+ assert backend.get_name() != STR_FLASH_ATTN_VAL
94
95
96
def test_invalid_env(monkeypatch):
97
"""Throw an exception if the backend name is invalid."""
98
override_backend_env_variable(monkeypatch, STR_INVALID_VAL)
99
with pytest.raises(ValueError):
90
- which_attn_to_use(16, torch.float16, None, 16, False)
100
+ get_attn_backend(16, torch.float16, None, 16, False)
vllm/attention/selector.py
+12
-127
@@ -9,7 +9,7 @@ import vllm.envs as envs
9
from vllm.attention.backends.abstract import AttentionBackend
10
from vllm.logger import init_logger
11
from vllm.platforms import _Backend, current_platform
12
-from vllm.utils import STR_BACKEND_ENV_VAR
12
+from vllm.utils import STR_BACKEND_ENV_VAR, resolve_obj_by_qualname
13
14
logger = init_logger(__name__)
15
@@ -114,83 +114,19 @@ def _cached_get_attn_backend(
114
BlocksparseFlashAttentionBackend)
115
return BlocksparseFlashAttentionBackend
116
117
- backend = which_attn_to_use(head_size, dtype, kv_cache_dtype, block_size,
118
- is_attention_free, use_v1)
119
- if backend == _Backend.FLASH_ATTN:
120
- logger.info("Using Flash Attention backend.")
121
- from vllm.attention.backends.flash_attn import ( # noqa: F401
122
- FlashAttentionBackend)
123
- return FlashAttentionBackend
124
- if backend == _Backend.FLASH_ATTN_VLLM_V1:
125
- from vllm.v1.attention.backends.flash_attn import ( # noqa: F401
126
- FlashAttentionBackend as FlashAttentionBackendV1)
127
- return FlashAttentionBackendV1
128
- if backend == _Backend.XFORMERS:
129
- logger.info("Using XFormers backend.")
130
- from vllm.attention.backends.xformers import ( # noqa: F401
131
- XFormersBackend)
132
- return XFormersBackend
133
- elif backend == _Backend.ROCM_FLASH:
134
- logger.info("Using ROCmFlashAttention backend.")
135
- from vllm.attention.backends.rocm_flash_attn import ( # noqa: F401
136
- ROCmFlashAttentionBackend)
137
- return ROCmFlashAttentionBackend
138
- elif backend == _Backend.TORCH_SDPA:
139
- assert current_platform.is_cpu(), RuntimeError(
140
- "Torch SDPA backend is only used for the CPU device.")
141
- logger.info("Using Torch SDPA backend.")
142
- from vllm.attention.backends.torch_sdpa import TorchSDPABackend
143
- return TorchSDPABackend
144
- elif backend == _Backend.OPENVINO:
145
- logger.info("Using OpenVINO Attention backend.")
146
- from vllm.attention.backends.openvino import OpenVINOAttentionBackend
147
- return OpenVINOAttentionBackend
148
- elif backend == _Backend.IPEX:
149
- assert current_platform.is_xpu(), RuntimeError(
150
- "IPEX attention backend is only used for the XPU device.")
151
- logger.info("Using IPEX attention backend.")
152
- from vllm.attention.backends.ipex_attn import IpexAttnBackend
153
- return IpexAttnBackend
154
- elif backend == _Backend.FLASHINFER:
155
- logger.info("Using Flashinfer backend.")
156
- from vllm.attention.backends.flashinfer import FlashInferBackend
157
- return FlashInferBackend
158
- elif backend == _Backend.HPU_ATTN:
159
- logger.info("Using HPUAttention backend.")
160
- from vllm.attention.backends.hpu_attn import HPUAttentionBackend
161
- return HPUAttentionBackend
162
- elif backend == _Backend.PALLAS:
163
- logger.info("Using Pallas backend.")
164
- from vllm.attention.backends.pallas import PallasAttentionBackend
165
- return PallasAttentionBackend
166
- elif backend == _Backend.NO_ATTENTION:
167
- from vllm.attention.backends.placeholder_attn import (
168
- PlaceholderAttentionBackend)
169
- return PlaceholderAttentionBackend
170
- else:
171
- raise ValueError("Invalid attention backend.")
172
-
173
-
174
-def which_attn_to_use(head_size: int,
175
- dtype: torch.dtype,
176
- kv_cache_dtype: Optional[str],
177
- block_size: int,
178
- is_attention_free: bool,
179
- use_v1: bool = False) -> _Backend:
180
- """Returns which flash attention backend to use."""
181
- # Default case.
182
- selected_backend = _Backend.FLASH_ATTN
183
-
117
# If there are no attention layers (e.g. we are running Mamba),
118
# use the placeholder NO_ATTENTION
119
if is_attention_free:
187
- return _Backend.NO_ATTENTION
120
+ from vllm.attention.backends.placeholder_attn import (
121
+ PlaceholderAttentionBackend)
122
+ return PlaceholderAttentionBackend
123
124
# Check whether a particular choice of backend was
125
# previously forced.
126
#
127
# THIS SELECTION OVERRIDES THE VLLM_ATTENTION_BACKEND
128
# ENVIRONMENT VARIABLE.
129
+ selected_backend = None
130
backend_by_global_setting: Optional[_Backend] = (
131
get_global_forced_attn_backend())
132
if backend_by_global_setting is not None:
@@ -201,64 +137,13 @@ def which_attn_to_use(head_size: int,
137
if backend_by_env_var is not None:
138
selected_backend = backend_name_to_enum(backend_by_env_var)
139
204
- # get device-specific default attn_backend
205
- default_backend = current_platform.get_default_attn_backend(
206
- selected_backend)
207
- if default_backend is not None:
208
- return default_backend
209
-
210
- if use_v1:
211
- return _Backend.FLASH_ATTN_VLLM_V1
212
-
213
- # FlashAttn in NVIDIA GPUs.
214
- if selected_backend == _Backend.FLASH_ATTN:
215
- if not current_platform.has_device_capability(80):
216
- # Volta and Turing NVIDIA GPUs.
217
- logger.info(
218
- "Cannot use FlashAttention-2 backend for Volta and Turing "
219
- "GPUs.")
220
- selected_backend = _Backend.XFORMERS
221
- elif dtype not in (torch.float16, torch.bfloat16):
222
- logger.info(
223
- "Cannot use FlashAttention-2 backend for dtype other than "
224
- "torch.float16 or torch.bfloat16.")
225
- selected_backend = _Backend.XFORMERS
226
- elif kv_cache_dtype is not None and kv_cache_dtype.startswith("fp8"):
227
- logger.info(
228
- "Cannot use FlashAttention-2 backend for FP8 KV cache.")
229
- logger.warning(
230
- "Please use FlashInfer backend with FP8 KV Cache for "
231
- "better performance by setting environment variable "
232
- "VLLM_ATTENTION_BACKEND=FLASHINFER")
233
- selected_backend = _Backend.XFORMERS
234
- elif block_size % 16 != 0:
235
- logger.info(
236
- "Cannot use FlashAttention-2 backend for block size not "
237
- "divisible by 16.")
238
- selected_backend = _Backend.XFORMERS
239
-
240
- # FlashAttn is valid for the model, checking if the package is installed.
241
- if selected_backend == _Backend.FLASH_ATTN:
242
- try:
243
- import vllm.vllm_flash_attn # noqa: F401
244
- from vllm.attention.backends.flash_attn import ( # noqa: F401
245
- FlashAttentionBackend)
246
-
247
- supported_sizes = FlashAttentionBackend.get_supported_head_sizes()
248
- if head_size not in supported_sizes:
249
- logger.info(
250
- "Cannot use FlashAttention-2 backend for head size %d.",
251
- head_size)
252
- selected_backend = _Backend.XFORMERS
253
- except ImportError:
254
- logger.info(
255
- "Cannot use FlashAttention-2 backend because the "
256
- "vllm.vllm_flash_attn package is not found. "
257
- "Make sure that vllm_flash_attn was built and installed "
258
- "(on by default).")
259
- selected_backend = _Backend.XFORMERS
260
-
261
- return selected_backend
140
+ # get device-specific attn_backend
141
+ attention_cls = current_platform.get_attn_backend_cls(
142
+ selected_backend, head_size, dtype, kv_cache_dtype, block_size, use_v1)
143
+ if not attention_cls:
144
+ raise ValueError(
145
+ f"Invalid attention backend for {current_platform.device_name}")
146
+ return resolve_obj_by_qualname(attention_cls)
147
148
149
@contextmanager
vllm/platforms/cpu.py
+5
-2
@@ -28,10 +28,13 @@ class CpuPlatform(Platform):
28
return "cpu"
29
30
@classmethod
31
- def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
31
+ def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
32
+ dtype: torch.dtype, kv_cache_dtype: Optional[str],
33
+ block_size: int, use_v1: bool) -> str:
34
if selected_backend != _Backend.TORCH_SDPA:
35
logger.info("Cannot use %s backend on CPU.", selected_backend)
34
- return _Backend.TORCH_SDPA
36
+ logger.info("Using Torch SDPA backend.")
37
+ return "vllm.attention.backends.torch_sdpa.TorchSDPABackend"
38
39
@classmethod
40
def get_device_total_memory(cls, device_id: int = 0) -> int:
vllm/platforms/cuda.py
+76
-1
@@ -16,7 +16,7 @@ import vllm._C # noqa
16
import vllm.envs as envs
17
from vllm.logger import init_logger
18
19
-from .interface import DeviceCapability, Platform, PlatformEnum
19
+from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
20
21
if TYPE_CHECKING:
22
from vllm.config import VllmConfig
@@ -141,6 +141,81 @@ class CudaPlatformBase(Platform):
141
if cache_config and cache_config.block_size is None:
142
cache_config.block_size = 16
143
144
+ @classmethod
145
+ def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
146
+ kv_cache_dtype, block_size, use_v1) -> str:
147
+ if use_v1:
148
+ logger.info("Using Flash Attention backend on V1 engine.")
149
+ return "vllm.v1.attention.backends.flash_attn.FlashAttentionBackend"
150
+ if selected_backend == _Backend.FLASHINFER:
151
+ logger.info("Using FlashInfer backend.")
152
+ return "vllm.attention.backends.flashinfer.FlashInferBackend"
153
+ elif selected_backend == _Backend.XFORMERS:
154
+ logger.info("Using XFormers backend.")
155
+ return "vllm.attention.backends.xformers.XFormersBackend"
156
+ elif selected_backend == _Backend.FLASH_ATTN:
157
+ pass
158
+ elif selected_backend:
159
+ raise ValueError(
160
+ f"Invalid attention backend for {cls.device_name}")
161
+
162
+ target_backend = _Backend.FLASH_ATTN
163
+ if not cls.has_device_capability(80):
164
+ # Volta and Turing NVIDIA GPUs.
165
+ logger.info(
166
+ "Cannot use FlashAttention-2 backend for Volta and Turing "
167
+ "GPUs.")
168
+ target_backend = _Backend.XFORMERS
169
+ elif dtype not in (torch.float16, torch.bfloat16):
170
+ logger.info(
171
+ "Cannot use FlashAttention-2 backend for dtype other than "
172
+ "torch.float16 or torch.bfloat16.")
173
+ target_backend = _Backend.XFORMERS
174
+ elif kv_cache_dtype is not None and \
175
+ kv_cache_dtype.startswith("fp8"):
176
+ logger.info(
177
+ "Cannot use FlashAttention-2 backend for FP8 KV cache.")
178
+ logger.warning(
179
+ "Please use FlashInfer backend with FP8 KV Cache for "
180
+ "better performance by setting environment variable "
181
+ "VLLM_ATTENTION_BACKEND=FLASHINFER")
182
+ target_backend = _Backend.XFORMERS
183
+ elif block_size % 16 != 0:
184
+ logger.info(
185
+ "Cannot use FlashAttention-2 backend for block size not "
186
+ "divisible by 16.")
187
+ target_backend = _Backend.XFORMERS
188
+
189
+ # FlashAttn is valid for the model, checking if the package is
190
+ # installed.
191
+ if target_backend == _Backend.FLASH_ATTN:
192
+ try:
193
+ import vllm.vllm_flash_attn # noqa: F401
194
+ from vllm.attention.backends.flash_attn import ( # noqa: F401
195
+ FlashAttentionBackend)
196
+
197
+ supported_sizes = \
198
+ FlashAttentionBackend.get_supported_head_sizes()
199
+ if head_size not in supported_sizes:
200
+ logger.info(
201
+ "Cannot use FlashAttention-2 backend for head size %d.",
202
+ head_size)
203
+ target_backend = _Backend.XFORMERS
204
+ except ImportError:
205
+ logger.info(
206
+ "Cannot use FlashAttention-2 backend because the "
207
+ "vllm.vllm_flash_attn package is not found. "
208
+ "Make sure that vllm_flash_attn was built and installed "
209
+ "(on by default).")
210
+ target_backend = _Backend.XFORMERS
211
+
212
+ if target_backend == _Backend.XFORMERS:
213
+ logger.info("Using XFormers backend.")
214
+ return "vllm.attention.backends.xformers.XFormersBackend"
215
+
216
+ logger.info("Using Flash Attention backend.")
217
+ return "vllm.attention.backends.flash_attn.FlashAttentionBackend"
218
+
219
220
# NVML utils
221
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
vllm/platforms/hpu.py
+5
-2
@@ -21,8 +21,11 @@ class HpuPlatform(Platform):
21
dispatch_key: str = "HPU"
22
23
@classmethod
24
- def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
25
- return _Backend.HPU_ATTN
24
+ def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
25
+ dtype: torch.dtype, kv_cache_dtype: Optional[str],
26
+ block_size: int, use_v1: bool) -> str:
27
+ logger.info("Using HPUAttention backend.")
28
+ return "vllm.attention.backends.hpu_attn.HPUAttentionBackend"
29
30
@classmethod
31
def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
vllm/platforms/interface.py
+5
-3
@@ -112,9 +112,11 @@ class Platform:
112
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
113
114
@classmethod
115
- def get_default_attn_backend(cls, selected_backend: _Backend):
116
- """Get the default attention backend of a device."""
117
- return None
115
+ def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
116
+ dtype: torch.dtype, kv_cache_dtype: Optional[str],
117
+ block_size: int, use_v1: bool) -> str:
118
+ """Get the attention backend class of a device."""
119
+ return ""
120
121
@classmethod
122
def get_device_capability(
vllm/platforms/openvino.py
+5
-2
@@ -28,10 +28,13 @@ class OpenVinoPlatform(Platform):
28
dispatch_key: str = "CPU"
29
30
@classmethod
31
- def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
31
+ def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
32
+ dtype: torch.dtype, kv_cache_dtype: Optional[str],
33
+ block_size: int, use_v1: bool) -> str:
34
if selected_backend != _Backend.OPENVINO:
35
logger.info("Cannot use %s backend on OpenVINO.", selected_backend)
34
- return _Backend.OPENVINO
36
+ logger.info("Using OpenVINO Attention backend.")
37
+ return "vllm.attention.backends.openvino.OpenVINOAttentionBackend"
38
39
@classmethod
40
def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/rocm.py
+4
-2
@@ -70,7 +70,8 @@ class RocmPlatform(Platform):
70
]
71
72
@classmethod
73
- def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
73
+ def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
74
+ kv_cache_dtype, block_size, use_v1) -> str:
75
selected_backend = (_Backend.ROCM_FLASH if selected_backend
76
== _Backend.FLASH_ATTN else selected_backend)
77
if selected_backend == _Backend.ROCM_FLASH:
@@ -79,7 +80,8 @@ class RocmPlatform(Platform):
80
logger.info("flash_attn is not supported on NAVI GPUs.")
81
else:
82
logger.info("%s is not supported in AMD GPUs.", selected_backend)
82
- return _Backend.ROCM_FLASH
83
+ logger.info("Using ROCmFlashAttention backend.")
84
+ return "vllm.attention.backends.rocm_flash_attn.ROCmFlashAttentionBackend" # noqa: E501
85
86
@classmethod
87
@lru_cache(maxsize=8)
vllm/platforms/tpu.py
+5
-2
@@ -24,10 +24,13 @@ class TpuPlatform(Platform):
24
]
25
26
@classmethod
27
- def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
27
+ def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
28
+ dtype: torch.dtype, kv_cache_dtype: Optional[str],
29
+ block_size: int, use_v1: bool) -> str:
30
if selected_backend != _Backend.PALLAS:
31
logger.info("Cannot use %s backend on TPU.", selected_backend)
30
- return _Backend.PALLAS
32
+ logger.info("Using Pallas backend.")
33
+ return "vllm.attention.backends.pallas.PallasAttentionBackend"
34
35
@classmethod
36
def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/xpu.py
+5
-2
@@ -21,10 +21,13 @@ class XPUPlatform(Platform):
21
dispatch_key: str = "XPU"
22
23
@classmethod
24
- def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
24
+ def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
25
+ dtype: torch.dtype, kv_cache_dtype: Optional[str],
26
+ block_size: int, use_v1: bool) -> str:
27
if selected_backend != _Backend.IPEX:
28
logger.info("Cannot use %s backend on XPU.", selected_backend)
27
- return _Backend.IPEX
29
+ logger.info("Using IPEX attention backend.")
30
+ return "vllm.attention.backends.ipex_attn.IpexAttnBackend"
31
32
@staticmethod
33
def get_device_capability(device_id: int = 0) -> DeviceCapability: