A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Bugfix] Voxtral on Blackwell GPUs (RTX 50 series) (#21077)
Signed-off-by: hax0r31337 <liulihaocaiqwq@gmail.com>
hax0r31337 committed
Jul 19, 2025 at 00:40 UTC
5782581acfa4dc89491904c7612f076c0cd5a646
1 file changed
+33
vllm/attention/layer.py
+33
@@ -16,6 +16,7 @@ from vllm.distributed.kv_transfer import (get_kv_transfer_group,
16
has_kv_transfer_group,
17
is_v1_kv_transfer_group)
18
from vllm.forward_context import ForwardContext, get_forward_context
19
+from vllm.logger import init_logger
20
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
21
from vllm.model_executor.layers.quantization.base_config import (
22
QuantizationConfig)
@@ -23,6 +24,34 @@ from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
24
from vllm.platforms import _Backend, current_platform
25
from vllm.utils import direct_register_custom_op
26
27
+logger = init_logger(__name__)
28
+USE_XFORMERS_OPS = None
29
+
30
+
31
+def check_xformers_availability():
32
+ global USE_XFORMERS_OPS
33
+ if USE_XFORMERS_OPS is not None:
34
+ return USE_XFORMERS_OPS
35
+
36
+ if current_platform.is_cuda() and current_platform.has_device_capability(
37
+ 100):
38
+ # Xformers FA is not compatible with B200
39
+ USE_XFORMERS_OPS = False
40
+ else:
41
+ try:
42
+ from importlib.util import find_spec
43
+
44
+ find_spec("xformers.ops")
45
+ USE_XFORMERS_OPS = True
46
+ except ImportError:
47
+ USE_XFORMERS_OPS = False
48
+
49
+ # the warning only needs to be shown once
50
+ if not USE_XFORMERS_OPS:
51
+ logger.warning("Xformers is not available, falling back.")
52
+
53
+ return USE_XFORMERS_OPS
54
+
55
56
class Attention(nn.Module):
57
"""Attention layer.
@@ -314,6 +343,10 @@ class MultiHeadAttention(nn.Module):
343
_Backend.TORCH_SDPA, _Backend.XFORMERS, _Backend.PALLAS_VLLM_V1
344
} else _Backend.TORCH_SDPA
345
346
+ if (self.attn_backend == _Backend.XFORMERS
347
+ and not check_xformers_availability()):
348
+ self.attn_backend = _Backend.TORCH_SDPA
349
+
350
def forward(
351
self,
352
query: torch.Tensor,