A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Bugfix] Fix Mistral3 support on SM100/SM120 (#20998)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jul 15, 2025 at 23:08 UTC
f46098335b8111b59e205d5bb0a6de43343fc33c
1 file changed
+7
-2
vllm/model_executor/models/pixtral.py
+7
-2
@@ -43,6 +43,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
43
PromptReplacement, PromptUpdate,
44
PromptUpdateDetails)
45
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
46
+from vllm.platforms import current_platform
47
from vllm.sequence import IntermediateTensors
48
from vllm.transformers_utils.tokenizer import (MistralTokenizer,
49
cached_tokenizer_from_config)
@@ -54,7 +55,12 @@ from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
55
56
try:
57
from xformers import ops as xops
57
- USE_XFORMERS_OPS = True
58
+ if (current_platform.is_cuda()
59
+ and current_platform.has_device_capability(100)):
60
+ # Xformers FA is not compatible with B200
61
+ USE_XFORMERS_OPS = False
62
+ else:
63
+ USE_XFORMERS_OPS = True
64
except ImportError:
65
USE_XFORMERS_OPS = False
66
@@ -1082,7 +1088,6 @@ class PixtralHFAttention(nn.Module):
1088
# Transpose q and k back for attention
1089
q = q.transpose(1, 2).contiguous()
1090
k = k.transpose(1, 2).contiguous()
1085
-
1091
out = xops.memory_efficient_attention(q,
1092
k,
1093
v,