[XPU] allow forcing flash attn for mm_prefix (#48828)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
liuzhenwei committed
Jul 17, 2026 at 17:44 UTC
d5b1ec268431f394efd6bad0f1b2adec657c31af
1 file changed
+12
-6
vllm/platforms/xpu.py
+12
-6
@@ -150,8 +150,18 @@ class XPUPlatform(Platform):
150
return AttentionBackendEnum.TRITON_ATTN.get_path()
151
elif attn_selector_config.use_mm_prefix:
152
# Flash Attention on XPU has no FA4 kernel, so it cannot apply the
153
- # multimodal prefix-LM bidirectional mask. Fall back to Triton
154
- # Attention, which supports mm_prefix.
153
+ # multimodal prefix-LM bidirectional mask. Honor an explicit Flash
154
+ # Attention request (for text-only workloads); otherwise fall back
155
+ # to Triton Attention, which supports mm_prefix.
156
+ if selected_backend == AttentionBackendEnum.FLASH_ATTN:
157
+ logger.warning_once(
158
+ "Using Flash Attention on XPU for a multimodal prefix-LM "
159
+ "model because it was explicitly requested. The prefix-LM "
160
+ "bidirectional mask cannot be applied, so image/video "
161
+ "inputs will produce incorrect results; only use this for "
162
+ "text-only workloads."
163
+ )
164
+ return AttentionBackendEnum.FLASH_ATTN.get_path()
165
logger.warning_once(
166
"Flash Attention on XPU does not support multimodal prefix-LM "
167
"attention. Falling back to Triton Attention backend."
@@ -265,10 +275,6 @@ class XPUPlatform(Platform):
275
if compilation_config.compile_sizes is None:
276
compilation_config.compile_sizes = []
277
268
- attention_config = vllm_config.attention_config
269
- if attention_config.backend is None:
270
- attention_config.backend = AttentionBackendEnum.FLASH_ATTN
271
-
278
# lazy import to avoid circular import
279
from vllm.utils.torch_utils import supports_xpu_graph
280