A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Honor --mm_encoder_attn_backend when used (#27124)
Co-authored-by: Bradley D <4551889+bradleyhd@users.noreply.github.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Bradley D committed
Oct 23, 2025 at 05:09 UTC
570c3e1cd4b325ee4cb240f9ff74876257cef085
6 files changed
+10
-1
vllm/attention/layer.py
+5
-1
@@ -93,12 +93,15 @@ def check_upstream_fa_availability(dtype: torch.dtype):
93
94
95
def maybe_get_vit_flash_attn_backend(
96
- attn_backend: _Backend, use_upstream_fa: bool
96
+ attn_backend: _Backend,
97
+ use_upstream_fa: bool,
98
+ attn_backend_override: _Backend | None = None,
99
) -> tuple[_Backend, Callable]:
100
if (
101
attn_backend != _Backend.FLASH_ATTN
102
and attn_backend != _Backend.ROCM_AITER_FA
103
and check_upstream_fa_availability(torch.get_default_dtype())
104
+ and attn_backend_override is None
105
):
106
attn_backend = _Backend.FLASH_ATTN
107
use_upstream_fa = True
@@ -499,6 +502,7 @@ class MultiHeadAttention(nn.Module):
502
maybe_get_vit_flash_attn_backend(
503
self.attn_backend,
504
use_upstream_fa,
505
+ attn_backend_override=attn_backend_override,
506
)
507
)
508
vllm/model_executor/models/dots_ocr.py
+1
@@ -299,6 +299,7 @@ class DotsVisionAttention(nn.Module):
299
maybe_get_vit_flash_attn_backend(
300
self.attn_backend,
301
self.use_upstream_fa,
302
+ attn_backend_override=attn_backend_override,
303
)
304
)
305
if self.attn_backend not in {
vllm/model_executor/models/ernie45_vl.py
+1
@@ -206,6 +206,7 @@ class Ernie4_5_VisionAttention(nn.Module):
206
maybe_get_vit_flash_attn_backend(
207
self.attn_backend,
208
self.use_upstream_fa,
209
+ attn_backend_override=attn_backend_override,
210
)
211
)
212
vllm/model_executor/models/glm4_1v.py
+1
@@ -296,6 +296,7 @@ class Glm4vVisionAttention(nn.Module):
296
maybe_get_vit_flash_attn_backend(
297
self.attn_backend,
298
self.use_upstream_fa,
299
+ attn_backend_override=attn_backend_override,
300
)
301
)
302
vllm/model_executor/models/qwen2_vl.py
+1
@@ -364,6 +364,7 @@ class Qwen2VisionAttention(nn.Module):
364
maybe_get_vit_flash_attn_backend(
365
self.attn_backend,
366
self.use_upstream_fa,
367
+ attn_backend_override=attn_backend_override,
368
)
369
)
370
vllm/model_executor/models/siglip2navit.py
+1
@@ -259,6 +259,7 @@ class Siglip2Attention(nn.Module):
259
maybe_get_vit_flash_attn_backend(
260
self.attn_backend,
261
self.use_upstream_fa,
262
+ attn_backend_override=attn_backend_override,
263
)
264
)
265