@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 44607e07d3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

Check if selected backend is None in get_attn_backend_cls() (#12975)

Signed-off-by: Yuan Tang <terrytangyuan@gmail.com>

Yuan Tang committed Feb 9, 2025 at 22:45 UTC 44607e07d3baf297efe56d77b3b1ddfbf16dad88
1 file changed +1 -1
vllm/platforms/cpu.py
+1 -1
@@ -35,7 +35,7 @@ class CpuPlatform(Platform):
35 dtype: torch.dtype, kv_cache_dtype: Optional[str],
36 block_size: int, use_v1: bool,
37 use_mla: bool) -> str:
38 - if selected_backend != _Backend.TORCH_SDPA:
38 + if selected_backend and selected_backend != _Backend.TORCH_SDPA:
39 logger.info("Cannot use %s backend on CPU.", selected_backend)
40 logger.info("Using Torch SDPA backend.")
41 return "vllm.attention.backends.torch_sdpa.TorchSDPABackend"