A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Add device assertion to TorchSDPA (#5402)
Li, Jiang committed
Jun 13, 2024 at 03:58 UTC
c3c2903e72c6e85a81ff6de8b879f4c82e8ad364
1 file changed
+3
vllm/attention/selector.py
+3
@@ -58,6 +58,9 @@ def get_attn_backend(
58
ROCmFlashAttentionBackend)
59
return ROCmFlashAttentionBackend
60
elif backend == _Backend.TORCH_SDPA:
61
+ # TODO: make XPU backend available here.
62
+ assert is_cpu(), RuntimeError(
63
+ "Torch SDPA backend is only used for the CPU device.")
64
logger.info("Using Torch SDPA backend.")
65
from vllm.attention.backends.torch_sdpa import TorchSDPABackend
66
return TorchSDPABackend