A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 00:30 UTC
[BugFix][VL] Fix FA selection on Qwen2.5-VL (#27790)
Signed-off-by: zhewenli <zhewenli@meta.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Zhewen Li committed
Oct 30, 2025 at 00:54 UTC
e806178d2a9b65ebd536342d58097a825d066b9e
2 files changed
+20
-12
.buildkite/test-amd.yaml
+1
-1
@@ -318,7 +318,7 @@ steps:
318
319
- label: V1 Test entrypoints # 35min
320
timeout_in_minutes: 50
321
- mirror_hardwares: [amdexperimental]
321
+ mirror_hardwares: [amdexperimental, amdproduction]
322
agent_pool: mi325_1
323
# grade: Blocking
324
source_file_dependencies:
vllm/model_executor/models/qwen2_5_vl.py
+19
-11
@@ -43,10 +43,7 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
43
)
44
45
from vllm.attention.backends.registry import _Backend
46
-from vllm.attention.layer import (
47
- check_upstream_fa_availability,
48
- maybe_get_vit_flash_attn_backend,
49
-)
46
+from vllm.attention.layer import maybe_get_vit_flash_attn_backend
47
from vllm.attention.ops.vit_attn_wrappers import (
48
vit_flash_attn_wrapper,
49
vit_xformers_attn_wrapper,
@@ -318,6 +315,7 @@ class Qwen2_5_VisionAttention(nn.Module):
315
use_data_parallel: bool = False,
316
attn_backend: _Backend = _Backend.TORCH_SDPA,
317
use_upstream_fa: bool = False,
318
+ attn_backend_override: _Backend | None = None,
319
) -> None:
320
super().__init__()
321
# Per attention head and per partition values.
@@ -358,8 +356,14 @@ class Qwen2_5_VisionAttention(nn.Module):
356
maybe_get_vit_flash_attn_backend(
357
self.attn_backend,
358
self.use_upstream_fa,
359
+ attn_backend_override=attn_backend_override,
360
)
361
)
362
+ # On ROCm with FLASH_ATTN backend, upstream flash_attn is used
363
+ from vllm.platforms import current_platform
364
+
365
+ if current_platform.is_rocm() and self.attn_backend == _Backend.FLASH_ATTN:
366
+ self.use_upstream_fa = True
367
self.is_flash_attn_backend = self.attn_backend in {
368
_Backend.FLASH_ATTN,
369
_Backend.ROCM_AITER_FA,
@@ -484,6 +488,7 @@ class Qwen2_5_VisionBlock(nn.Module):
488
use_data_parallel: bool = False,
489
attn_backend: _Backend = _Backend.TORCH_SDPA,
490
use_upstream_fa: bool = False,
491
+ attn_backend_override: _Backend | None = None,
492
) -> None:
493
super().__init__()
494
if norm_layer is None:
@@ -499,6 +504,7 @@ class Qwen2_5_VisionBlock(nn.Module):
504
use_data_parallel=use_data_parallel,
505
attn_backend=attn_backend,
506
use_upstream_fa=use_upstream_fa,
507
+ attn_backend_override=attn_backend_override,
508
)
509
self.mlp = Qwen2_5_VisionMLP(
510
dim,
@@ -698,13 +704,14 @@ class Qwen2_5_VisionTransformer(nn.Module):
704
dtype=torch.get_default_dtype(),
705
attn_backend_override=attn_backend_override,
706
)
701
- if (
702
- self.attn_backend != _Backend.FLASH_ATTN
703
- and self.attn_backend != _Backend.ROCM_AITER_FA
704
- and check_upstream_fa_availability(torch.get_default_dtype())
705
- ):
706
- self.attn_backend = _Backend.FLASH_ATTN
707
- use_upstream_fa = True
707
+
708
+ self.attn_backend, self.flash_attn_varlen_func = (
709
+ maybe_get_vit_flash_attn_backend(
710
+ self.attn_backend,
711
+ use_upstream_fa,
712
+ attn_backend_override=attn_backend_override,
713
+ )
714
+ )
715
716
if self.attn_backend not in {
717
_Backend.FLASH_ATTN,
@@ -730,6 +737,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
737
use_data_parallel=use_data_parallel,
738
attn_backend=self.attn_backend,
739
use_upstream_fa=use_upstream_fa,
740
+ attn_backend_override=attn_backend_override,
741
)
742
for layer_idx in range(depth)
743
]