A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Allow Only SDPA Backend for ViT on B200 for Qwen3-VL (#25788)
Wentao Ye committed
Sep 26, 2025 at 23:44 UTC
c242c98031b87d00999e07dbb4aa9b2a70798c6c
2 files changed
+75
-51
vllm/model_executor/models/qwen2_5_vl.py
+37
-36
@@ -274,6 +274,8 @@ class Qwen2_5_VisionAttention(nn.Module):
274
quant_config: Optional[QuantizationConfig] = None,
275
prefix: str = "",
276
use_data_parallel: bool = False,
277
+ attn_backend: _Backend = _Backend.TORCH_SDPA,
278
+ use_upstream_fa: bool = False,
279
) -> None:
280
super().__init__()
281
# Per attention head and per partition values.
@@ -300,25 +302,8 @@ class Qwen2_5_VisionAttention(nn.Module):
302
quant_config=quant_config,
303
prefix=f"{prefix}.proj",
304
disable_tp=use_data_parallel)
303
-
304
- # Detect attention implementation.
305
- self.attn_backend = get_vit_attn_backend(
306
- head_size=self.hidden_size_per_attention_head,
307
- dtype=torch.get_default_dtype())
308
- self.use_upstream_fa = False
309
- if self.attn_backend != _Backend.FLASH_ATTN and \
310
- check_upstream_fa_availability(
311
- torch.get_default_dtype()):
312
- self.attn_backend = _Backend.FLASH_ATTN
313
- self.use_upstream_fa = True
314
-
315
- if self.attn_backend not in {
316
- _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS,
317
- _Backend.ROCM_AITER_FA
318
- }:
319
- raise RuntimeError(
320
- f"Qwen2.5-VL does not support {self.attn_backend} backend now."
321
- )
305
+ self.attn_backend = attn_backend
306
+ self.use_upstream_fa = use_upstream_fa
307
self.is_flash_attn_backend = self.attn_backend in {
308
_Backend.FLASH_ATTN, _Backend.ROCM_AITER_FA
309
}
@@ -443,6 +428,8 @@ class Qwen2_5_VisionBlock(nn.Module):
428
quant_config: Optional[QuantizationConfig] = None,
429
prefix: str = "",
430
use_data_parallel: bool = False,
431
+ attn_backend: _Backend = _Backend.TORCH_SDPA,
432
+ use_upstream_fa: bool = False,
433
) -> None:
434
super().__init__()
435
if norm_layer is None:
@@ -455,7 +442,9 @@ class Qwen2_5_VisionBlock(nn.Module):
442
projection_size=dim,
443
quant_config=quant_config,
444
prefix=f"{prefix}.attn",
458
- use_data_parallel=use_data_parallel)
445
+ use_data_parallel=use_data_parallel,
446
+ attn_backend=attn_backend,
447
+ use_upstream_fa=use_upstream_fa)
448
self.mlp = Qwen2_5_VisionMLP(dim,
449
mlp_hidden_dim,
450
act_fn=act_fn,
@@ -627,17 +616,35 @@ class Qwen2_5_VisionTransformer(nn.Module):
616
head_dim = self.hidden_size // self.num_heads
617
self.rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2)
618
619
+ use_upstream_fa = False
620
+ self.attn_backend = get_vit_attn_backend(
621
+ head_size=head_dim, dtype=torch.get_default_dtype())
622
+ if self.attn_backend != _Backend.FLASH_ATTN and \
623
+ check_upstream_fa_availability(
624
+ torch.get_default_dtype()):
625
+ self.attn_backend = _Backend.FLASH_ATTN
626
+ use_upstream_fa = True
627
+
628
+ if self.attn_backend not in {
629
+ _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS,
630
+ _Backend.ROCM_AITER_FA
631
+ }:
632
+ raise RuntimeError(
633
+ f"Qwen2.5-VL does not support {self.attn_backend} backend now."
634
+ )
635
+
636
self.blocks = nn.ModuleList([
631
- Qwen2_5_VisionBlock(dim=self.hidden_size,
632
- num_heads=self.num_heads,
633
- mlp_hidden_dim=vision_config.intermediate_size,
634
- act_fn=get_act_and_mul_fn(
635
- vision_config.hidden_act),
636
- norm_layer=norm_layer,
637
- quant_config=quant_config,
638
- prefix=f"{prefix}.blocks.{layer_idx}",
639
- use_data_parallel=use_data_parallel)
640
- for layer_idx in range(depth)
637
+ Qwen2_5_VisionBlock(
638
+ dim=self.hidden_size,
639
+ num_heads=self.num_heads,
640
+ mlp_hidden_dim=vision_config.intermediate_size,
641
+ act_fn=get_act_and_mul_fn(vision_config.hidden_act),
642
+ norm_layer=norm_layer,
643
+ quant_config=quant_config,
644
+ prefix=f"{prefix}.blocks.{layer_idx}",
645
+ use_data_parallel=use_data_parallel,
646
+ attn_backend=self.attn_backend,
647
+ use_upstream_fa=use_upstream_fa) for layer_idx in range(depth)
648
])
649
self.merger = Qwen2_5_VisionPatchMerger(
650
d_model=vision_config.out_hidden_size,
@@ -648,12 +655,6 @@ class Qwen2_5_VisionTransformer(nn.Module):
655
prefix=f"{prefix}.merger",
656
use_data_parallel=use_data_parallel,
657
)
651
- self.attn_backend = get_vit_attn_backend(
652
- head_size=head_dim, dtype=torch.get_default_dtype())
653
- if self.attn_backend != _Backend.FLASH_ATTN and \
654
- check_upstream_fa_availability(
655
- torch.get_default_dtype()):
656
- self.attn_backend = _Backend.FLASH_ATTN
658
659
@property
660
def dtype(self) -> torch.dtype:
vllm/model_executor/models/qwen3_vl.py
+38
-15
@@ -63,7 +63,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
63
PromptReplacement, PromptUpdate,
64
PromptUpdateDetails)
65
from vllm.multimodal.profiling import BaseDummyInputsBuilder
66
-from vllm.platforms import _Backend
66
+from vllm.platforms import _Backend, current_platform
67
from vllm.sequence import IntermediateTensors
68
from vllm.transformers_utils.config import uses_mrope
69
from vllm.utils import is_list_of
@@ -158,6 +158,8 @@ class Qwen3_VisionBlock(nn.Module):
158
quant_config: Optional[QuantizationConfig] = None,
159
prefix: str = "",
160
use_data_parallel: bool = False,
161
+ attn_backend: _Backend = _Backend.TORCH_SDPA,
162
+ use_upstream_fa: bool = False,
163
) -> None:
164
super().__init__()
165
if norm_layer is None:
@@ -170,7 +172,9 @@ class Qwen3_VisionBlock(nn.Module):
172
projection_size=dim,
173
quant_config=quant_config,
174
prefix=f"{prefix}.attn",
173
- use_data_parallel=use_data_parallel)
175
+ use_data_parallel=use_data_parallel,
176
+ attn_backend=attn_backend,
177
+ use_upstream_fa=use_upstream_fa)
178
self.mlp = Qwen3_VisionMLP(dim,
179
mlp_hidden_dim,
180
act_fn=act_fn,
@@ -287,19 +291,6 @@ class Qwen3_VisionTransformer(nn.Module):
291
head_dim = self.hidden_size // self.num_heads
292
self.rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2)
293
290
- self.blocks = nn.ModuleList([
291
- Qwen3_VisionBlock(
292
- dim=self.hidden_size,
293
- num_heads=self.num_heads,
294
- mlp_hidden_dim=vision_config.intermediate_size,
295
- act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],
296
- norm_layer=norm_layer,
297
- quant_config=quant_config,
298
- prefix=f"{prefix}.blocks.{layer_idx}",
299
- use_data_parallel=use_data_parallel)
300
- for layer_idx in range(vision_config.depth)
301
- ])
302
-
294
self.merger = Qwen3_VisionPatchMerger(
295
d_model=vision_config.out_hidden_size,
296
context_dim=self.hidden_size,
@@ -325,10 +316,42 @@ class Qwen3_VisionTransformer(nn.Module):
316
317
self.attn_backend = get_vit_attn_backend(
318
head_size=head_dim, dtype=torch.get_default_dtype())
319
+ use_upstream_fa = False
320
if self.attn_backend != _Backend.FLASH_ATTN and \
321
check_upstream_fa_availability(
322
torch.get_default_dtype()):
323
self.attn_backend = _Backend.FLASH_ATTN
324
+ use_upstream_fa = True
325
+
326
+ if self.attn_backend not in {
327
+ _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS,
328
+ _Backend.ROCM_AITER_FA
329
+ }:
330
+ raise RuntimeError(
331
+ f"Qwen3-VL does not support {self.attn_backend} backend now.")
332
+ if current_platform.is_device_capability(
333
+ 100) and self.attn_backend != _Backend.TORCH_SDPA:
334
+ # TODO(Roger/Wentao): remove this after FA
335
+ # or XFORMERS's issue fixed on Blackwell
336
+ logger.info_once("Qwen3-VL vision attention does not support "
337
+ f"{self.attn_backend} backend on Blackwell now. "
338
+ "Vision attention backend is set to TORCH_SDPA.")
339
+ self.attn_backend = _Backend.TORCH_SDPA
340
+
341
+ self.blocks = nn.ModuleList([
342
+ Qwen3_VisionBlock(
343
+ dim=self.hidden_size,
344
+ num_heads=self.num_heads,
345
+ mlp_hidden_dim=vision_config.intermediate_size,
346
+ act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],
347
+ norm_layer=norm_layer,
348
+ quant_config=quant_config,
349
+ prefix=f"{prefix}.blocks.{layer_idx}",
350
+ use_data_parallel=use_data_parallel,
351
+ attn_backend=self.attn_backend,
352
+ use_upstream_fa=use_upstream_fa)
353
+ for layer_idx in range(vision_config.depth)
354
+ ])
355
356
@property
357
def dtype(self) -> torch.dtype: