A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 02:45 UTC
[CI][MM] Gate vision encoder attention mask to MiniCPM only, fixing Aria regression (#36206)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Mar 6, 2026 at 03:17 UTC
2a00d3241f2c5810f4ba6a3c5fe79f7c76a94900
2 files changed
+13
-1
vllm/model_executor/models/idefics2_vision_model.py
+9
-1
@@ -359,6 +359,7 @@ class Idefics2VisionTransformer(nn.Module):
359
*,
360
num_hidden_layers_override: int | None = None,
361
require_post_norm: bool = True,
362
+ apply_encoder_attention_mask: bool = False,
363
prefix: str = "",
364
) -> None:
365
super().__init__()
@@ -366,6 +367,7 @@ class Idefics2VisionTransformer(nn.Module):
367
embed_dim = config.hidden_size
368
self.config = config
369
self.use_data_parallel = is_vit_use_data_parallel()
370
+ self.apply_encoder_attention_mask = apply_encoder_attention_mask
371
self.embeddings = Idefics2VisionEmbeddings(config)
372
self.encoder = Idefics2Encoder(
373
config,
@@ -425,10 +427,16 @@ class Idefics2VisionTransformer(nn.Module):
427
)
428
429
# Align with HuggingFace NaViT SigLIP in MiniCPMV/O:
430
+ # - if apply_encoder_attention_mask is False, skip (not all models
431
+ # sharing this encoder apply masking in attention, e.g. Aria, Phi4)
432
# - if patch_attention_mask was None, skip attention masking
433
# - if any padding exists, create an additive 4D mask and pass it
434
# to attention; else skip mask for performance.
431
- if flat_patch_mask is None or not torch.any(~flat_patch_mask):
435
+ if (
436
+ not self.apply_encoder_attention_mask
437
+ or flat_patch_mask is None
438
+ or not torch.any(~flat_patch_mask)
439
+ ):
440
attention_mask = None
441
else:
442
# Additive mask: masked positions receive a large negative value.
vllm/model_executor/models/minicpmv.py
+4
@@ -1336,6 +1336,7 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
1336
model = Idefics2VisionTransformer(
1337
config.vision_config,
1338
quant_config=quant_config,
1339
+ apply_encoder_attention_mask=True,
1340
prefix=prefix,
1341
)
1342
if self.config.drop_vision_last_layer:
@@ -1428,6 +1429,7 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1429
model = Idefics2VisionTransformer(
1430
config.vision_config,
1431
quant_config=quant_config,
1432
+ apply_encoder_attention_mask=True,
1433
prefix=prefix,
1434
)
1435
if self.config.drop_vision_last_layer:
@@ -1525,6 +1527,7 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):
1527
model = Idefics2VisionTransformer(
1528
config.vision_config,
1529
quant_config=quant_config,
1530
+ apply_encoder_attention_mask=True,
1531
prefix=prefix,
1532
)
1533
if self.config.drop_vision_last_layer:
@@ -1622,6 +1625,7 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):
1625
model = Idefics2VisionTransformer(
1626
config.vision_config,
1627
quant_config=quant_config,
1628
+ apply_encoder_attention_mask=True,
1629
prefix=prefix,
1630
)
1631
if self.config.drop_vision_last_layer: