A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
Check if selected backend is None in get_attn_backend_cls() (#12975)
Signed-off-by: Yuan Tang <terrytangyuan@gmail.com>
Yuan Tang committed
Feb 9, 2025 at 22:45 UTC
44607e07d3baf297efe56d77b3b1ddfbf16dad88
1 file changed
+1
-1
vllm/platforms/cpu.py
+1
-1
@@ -35,7 +35,7 @@ class CpuPlatform(Platform):
35
dtype: torch.dtype, kv_cache_dtype: Optional[str],
36
block_size: int, use_v1: bool,
37
use_mla: bool) -> str:
38
- if selected_backend != _Backend.TORCH_SDPA:
38
+ if selected_backend and selected_backend != _Backend.TORCH_SDPA:
39
logger.info("Cannot use %s backend on CPU.", selected_backend)
40
logger.info("Using Torch SDPA backend.")
41
return "vllm.attention.backends.torch_sdpa.TorchSDPABackend"