@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c3c2903e72
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Add device assertion to TorchSDPA (#5402)

Li, Jiang committed Jun 13, 2024 at 03:58 UTC c3c2903e72c6e85a81ff6de8b879f4c82e8ad364
1 file changed +3
vllm/attention/selector.py
+3
@@ -58,6 +58,9 @@ def get_attn_backend(
58 ROCmFlashAttentionBackend)
59 return ROCmFlashAttentionBackend
60 elif backend == _Backend.TORCH_SDPA:
61 + # TODO: make XPU backend available here.
62 + assert is_cpu(), RuntimeError(
63 + "Torch SDPA backend is only used for the CPU device.")
64 logger.info("Using Torch SDPA backend.")
65 from vllm.attention.backends.torch_sdpa import TorchSDPABackend
66 return TorchSDPABackend