9ad7f89f55 · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/layers/attention/mm_encoder_attention.py

-9

@@ -4,7 +4,6 @@

4

5 import torch

6

7 -from vllm.config import MultiModalConfig

7 from vllm.logger import init_logger

8 from vllm.model_executor.custom_op import CustomOp

9 from vllm.model_executor.models.vision import get_vit_attn_backend

@@ -32,7 +31,6 @@ class MMEncoderAttention(CustomOp):

31 scale: float | None = None,

32 num_kv_heads: int | None = None,

33 prefix: str = "",

35 - multimodal_config: MultiModalConfig | None = None,

34 ) -> None:

35 """

36 Args:

@@ -42,7 +40,6 @@ class MMEncoderAttention(CustomOp):

40 num_kv_heads: number of kv heads.

41 prefix: This has no effect, it is only here to make it easier to

42 swap between Attention and MultiHeadAttention

45 - multimodal_config: configs for multi-modal.

43 """

44 super().__init__()

45

@@ -62,16 +59,10 @@ class MMEncoderAttention(CustomOp):

59 # weight and activation dtype.

60 dtype = torch.get_default_dtype()

61

65 - # Try to get vision attention backend from multimodal_config.

66 - attn_backend_override = None

67 - if multimodal_config is not None:

68 - attn_backend_override = multimodal_config.mm_encoder_attn_backend

69 -

62 # Get device-specific vision attention backend.

63 self.attn_backend = get_vit_attn_backend(

64 head_size=head_size,

65 dtype=dtype,

74 - attn_backend_override=attn_backend_override,

66 )

67

68 self.is_flash_attn_backend = self.attn_backend in {

vllm/model_executor/models/clip.py

+4 -24

@@ -16,7 +16,7 @@ from transformers import (

16

17 from vllm.attention.layer import Attention

18 from vllm.config import VllmConfig

19 -from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

19 +from vllm.config.multimodal import BaseDummyOptions

20 from vllm.distributed import divide, get_tensor_model_parallel_world_size

21 from vllm.model_executor.layers.activation import get_act_fn

22 from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention

@@ -59,6 +59,7 @@ from .vision import (

59 VisionFeatureSelectStrategy,

60 VisionFeatureSelectStrategyStr,

61 get_num_selected_vision_tokens,

62 + is_vit_use_data_parallel,

63 resolve_visual_encoder_outputs,

64 )

65

@@ -353,7 +354,6 @@ class CLIPAttention(nn.Module):

354 self,

355 config: CLIPTextConfig | CLIPVisionConfig,

356 quant_config: QuantizationConfig | None = None,

356 - multimodal_config: MultiModalConfig | None = None,

357 *,

358 prefix: str = "",

359 attn_cls: type[Attention] | type[MMEncoderAttention],

@@ -372,11 +372,7 @@ class CLIPAttention(nn.Module):

372 )

373 self.scale = self.head_dim**-0.5

374

375 - use_data_parallel = (

376 - multimodal_config.mm_encoder_tp_mode == "data"

377 - if multimodal_config

378 - else False

379 - )

375 + use_data_parallel = is_vit_use_data_parallel()

376 self.qkv_proj = QKVParallelLinear(

377 hidden_size=self.embed_dim,

378 head_size=self.head_dim,

@@ -405,7 +401,6 @@ class CLIPAttention(nn.Module):

401 self.head_dim,

402 self.scale,

403 prefix=f"{prefix}.attn",

408 - multimodal_config=multimodal_config,

404 )

405 else:

406 self.attn = attn_cls(

@@ -434,17 +429,12 @@ class CLIPMLP(nn.Module):

429 self,

430 config: CLIPTextConfig | CLIPVisionConfig,

431 quant_config: QuantizationConfig | None = None,

437 - multimodal_config: MultiModalConfig | None = None,

432 prefix: str = "",

433 ) -> None:

434 super().__init__()

435

436 self.config = config

443 - use_data_parallel = (

444 - multimodal_config.mm_encoder_tp_mode == "data"

445 - if multimodal_config

446 - else False

447 - )

437 + use_data_parallel = is_vit_use_data_parallel()

438 self.activation_fn = get_act_fn(config.hidden_act)

439

440 self.fc1 = ColumnParallelLinear(

@@ -477,7 +467,6 @@ class CLIPEncoderLayer(nn.Module):

467 self,

468 config: CLIPTextConfig | CLIPVisionConfig,

469 quant_config: QuantizationConfig | None = None,

480 - multimodal_config: MultiModalConfig | None = None,

470 *,

471 prefix: str = "",

472 attn_cls: type[Attention] | type[MMEncoderAttention],

@@ -487,7 +476,6 @@ class CLIPEncoderLayer(nn.Module):

476 self.self_attn = CLIPAttention(

477 config,

478 quant_config=quant_config,

490 - multimodal_config=multimodal_config,

479 prefix=f"{prefix}.self_attn",

480 attn_cls=attn_cls,

481 )

@@ -495,7 +483,6 @@ class CLIPEncoderLayer(nn.Module):

483 self.mlp = CLIPMLP(

484 config,

485 quant_config=quant_config,

498 - multimodal_config=multimodal_config,

486 prefix=f"{prefix}.mlp",

487 )

488 self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)

@@ -528,7 +515,6 @@ class CLIPEncoder(nn.Module):

515 self,

516 config: CLIPTextConfig | CLIPVisionConfig,

517 quant_config: QuantizationConfig | None = None,

531 - multimodal_config: MultiModalConfig | None = None,

518 num_hidden_layers_override: int | None = None,

519 *,

520 prefix: str = "",

@@ -548,7 +534,6 @@ class CLIPEncoder(nn.Module):

534 CLIPEncoderLayer(

535 config=config,

536 quant_config=quant_config,

551 - multimodal_config=multimodal_config,

537 prefix=f"{prefix}.layers.{layer_idx}",

538 attn_cls=attn_cls,

539 )

@@ -658,7 +643,6 @@ class CLIPVisionTransformer(nn.Module):

643 self,

644 config: CLIPVisionConfig,

645 quant_config: QuantizationConfig | None = None,

661 - multimodal_config: MultiModalConfig | None = None,

646 *,

647 num_hidden_layers_override: int | None = None,

648 require_post_norm: bool | None = None,

@@ -678,7 +662,6 @@ class CLIPVisionTransformer(nn.Module):

662 self.encoder = CLIPEncoder(

663 config=config,

664 quant_config=quant_config,

681 - multimodal_config=multimodal_config,

665 num_hidden_layers_override=num_hidden_layers_override,

666 prefix=f"{prefix}.encoder",

667 attn_cls=MMEncoderAttention,

@@ -780,7 +763,6 @@ class CLIPVisionModel(nn.Module):

763 self,

764 config: CLIPVisionConfig,

765 quant_config: QuantizationConfig | None = None,

783 - multimodal_config: MultiModalConfig | None = None,

766 *,

767 num_hidden_layers_override: int | None = None,

768 require_post_norm: bool | None = None,

@@ -791,7 +773,6 @@ class CLIPVisionModel(nn.Module):

773 self.vision_model = CLIPVisionTransformer(

774 config=config,

775 quant_config=quant_config,

794 - multimodal_config=multimodal_config,

776 num_hidden_layers_override=num_hidden_layers_override,

777 require_post_norm=require_post_norm,

778 prefix=f"{prefix}.vision_model",

@@ -869,7 +850,6 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

850 self.vision_model = CLIPVisionTransformer(

851 vision_config,

852 quant_config=quant_config,

872 - multimodal_config=multimodal_config,

853 prefix=maybe_prefix(prefix, "vision_model"),

854 )

855 self.visual_projection = nn.Linear(

vllm/model_executor/models/deepencoder.py

-3

@@ -18,7 +18,6 @@ import torch.nn as nn

18 import torch.nn.functional as F

19 from transformers import CLIPVisionConfig

20

21 -from vllm.config import MultiModalConfig

21 from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention

22 from vllm.model_executor.layers.conv import Conv2dLayer

23 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -609,7 +608,6 @@ class DeepCLIPVisionTransformer(nn.Module):

608 self,

609 config: CLIPVisionConfig,

610 quant_config: QuantizationConfig | None = None,

612 - multimodal_config: MultiModalConfig | None = None,

611 *,

612 num_hidden_layers_override: int | None = None,

613 prefix: str = "",

@@ -628,7 +626,6 @@ class DeepCLIPVisionTransformer(nn.Module):

626 self.transformer = CLIPEncoder(

627 config=config,

628 quant_config=quant_config,

631 - multimodal_config=multimodal_config,

629 num_hidden_layers_override=num_hidden_layers_override,

630 prefix=f"{prefix}.encoder",

631 attn_cls=MMEncoderAttention,

vllm/model_executor/models/deepseek_ocr.py

-1

@@ -398,7 +398,6 @@ class DeepseekOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports

398 self.vision_model = DeepCLIPVisionTransformer(

399 config=clip_vision_config,

400 quant_config=quant_config,

401 - multimodal_config=multimodal_config,

401 prefix=maybe_prefix(prefix, "vision_model"),

402 )

403

vllm/model_executor/models/dots_ocr.py

+5 -34

@@ -8,7 +8,7 @@ import torch.nn as nn

8 from torch.nn import LayerNorm

9 from transformers.models.qwen2_vl import Qwen2VLProcessor

10

11 -from vllm.config import MultiModalConfig, VllmConfig

11 +from vllm.config import VllmConfig

12 from vllm.config.multimodal import BaseDummyOptions

13 from vllm.distributed import utils as dist_utils

14 from vllm.distributed.parallel_state import (

@@ -60,7 +60,7 @@ from vllm.transformers_utils.configs.dotsocr import DotsOCRConfig, DotsVisionCon

60 from vllm.utils.tensor_schema import TensorSchema, TensorShape

61 from vllm.v1.attention.backends.registry import AttentionBackendEnum

62

63 -from .vision import run_dp_sharded_mrope_vision_model

63 +from .vision import is_vit_use_data_parallel, run_dp_sharded_mrope_vision_model

64

65 IMAGE_TOKEN = "<|imgpad|>"

66

@@ -183,9 +183,9 @@ class PatchMerger(nn.Module):

183 spatial_merge_size: int = 2,

184 pre_norm="layernorm",

185 prefix: str = "",

186 - use_data_parallel: bool = False,

186 ) -> None:

187 super().__init__()

188 + use_data_parallel = is_vit_use_data_parallel()

189 self.hidden_size = context_dim * (spatial_merge_size**2)

190 self.pre_norm = pre_norm

191 if self.pre_norm == "layernorm":

@@ -230,15 +230,10 @@ class DotsVisionAttention(nn.Module):

230 bias: bool = True,

231 *,

232 quant_config: QuantizationConfig | None = None,

233 - multimodal_config: MultiModalConfig | None = None,

233 prefix: str = "",

234 ) -> None:

235 super().__init__()

237 - use_data_parallel = (

238 - multimodal_config.mm_encoder_tp_mode == "data"

239 - if multimodal_config

240 - else False

241 - )

236 + use_data_parallel = is_vit_use_data_parallel()

237

238 self.embed_dim = dim

239 self.tp_size = (

@@ -272,7 +267,6 @@ class DotsVisionAttention(nn.Module):

267 num_heads=self.num_attention_heads_per_partition,

268 head_size=self.hidden_size_per_attention_head,

269 scale=self.hidden_size_per_attention_head**-0.5,

275 - multimodal_config=multimodal_config,

270 prefix=f"{prefix}.attn",

271 )

272

@@ -329,7 +323,6 @@ class DotsSwiGLUFFN(nn.Module):

323 config,

324 *,

325 quant_config: QuantizationConfig | None = None,

332 - multimodal_config: MultiModalConfig | None = None,

326 prefix: str = "",

327 ):

328 super().__init__()

@@ -337,11 +330,7 @@ class DotsSwiGLUFFN(nn.Module):

330 in_features = config.embed_dim

331 bias = config.use_bias

332

340 - use_data_parallel = (

341 - multimodal_config.mm_encoder_tp_mode == "data"

342 - if multimodal_config

343 - else False

344 - )

333 + use_data_parallel = is_vit_use_data_parallel()

334 # Referenced aimv2.py AIMv2SwiGLUFFN

335 self.fc13 = MergedColumnParallelLinear(

336 in_features,

@@ -447,7 +436,6 @@ class DotsVisionBlock(nn.Module):

436 config,

437 *,

438 quant_config: QuantizationConfig | None = None,

450 - multimodal_config: MultiModalConfig | None = None,

439 prefix: str = "",

440 ):

441 super().__init__()

@@ -458,14 +446,12 @@ class DotsVisionBlock(nn.Module):

446 num_heads=config.num_attention_heads,

447 bias=config.use_bias,

448 quant_config=quant_config,

461 - multimodal_config=multimodal_config,

449 prefix=f"{prefix}.attn",

450 )

451 self.norm1 = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)

452 self.mlp = DotsSwiGLUFFN(

453 config,

454 quant_config=quant_config,

468 - multimodal_config=multimodal_config,

455 prefix=f"{prefix}.mlp",

456 )

457 self.norm2 = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)

@@ -493,7 +479,6 @@ class DotsVisionTransformer(nn.Module):

479 self,

480 config: DotsVisionConfig,

481 quant_config: QuantizationConfig | None = None,

496 - multimodal_config: MultiModalConfig | None = None,

482 *,

483 num_hidden_layers_override: int | None = None,

484 require_post_norm: bool | None = None,

@@ -507,15 +492,9 @@ class DotsVisionTransformer(nn.Module):

492

493 head_dim = config.embed_dim // config.num_attention_heads

494 self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2)

510 - attn_backend_override = (

511 - multimodal_config.mm_encoder_attn_backend

512 - if multimodal_config is not None

513 - else None

514 - )

495 self.attn_backend = get_vit_attn_backend(

496 head_size=head_dim,

497 dtype=torch.get_default_dtype(),

518 - attn_backend_override=attn_backend_override,

498 )

499 self.out_hidden_size = config.hidden_size

500 # Keep blocks for compatibility with other vision towers

@@ -529,7 +508,6 @@ class DotsVisionTransformer(nn.Module):

508 DotsVisionBlock(

509 config,

510 quant_config=quant_config,

532 - multimodal_config=multimodal_config,

511 prefix=f"{prefix}.blocks.{i}",

512 )

513 for i in range(num_layers)

@@ -542,16 +520,10 @@ class DotsVisionTransformer(nn.Module):

520 else:

521 self.post_trunk_norm = None

522

545 - use_data_parallel = (

546 - multimodal_config.mm_encoder_tp_mode == "data"

547 - if multimodal_config

548 - else False

549 - )

523 self.merger = PatchMerger(

524 dim=config.hidden_size,

525 context_dim=config.embed_dim,

526 spatial_merge_size=config.spatial_merge_size,

554 - use_data_parallel=use_data_parallel,

527 )

528

529 @property

@@ -693,7 +665,6 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA

665 self.vision_tower = DotsVisionTransformer(

666 vision_config,

667 quant_config=self.quant_config,

696 - multimodal_config=multimodal_config,

668 prefix=maybe_prefix(prefix, "vision_tower"),

669 )

670

vllm/model_executor/models/eagle2_5_vl.py

-1

@@ -270,7 +270,6 @@ class Eagle2_5_VLForConditionalGeneration(

270 return SiglipVisionModel(

271 vision_config,

272 quant_config=quant_config,

273 - multimodal_config=self.multimodal_config,

273 num_hidden_layers_override=num_hidden_layers,

274 prefix=prefix,

275 )

vllm/model_executor/models/ernie45_vl.py

+1 -12

@@ -36,7 +36,7 @@ import torch.nn.functional as F

36 from einops import rearrange

37 from transformers import BatchFeature

38

39 -from vllm.config import MultiModalConfig, VllmConfig

39 +from vllm.config import VllmConfig

40 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

41 from vllm.distributed import parallel_state

42 from vllm.distributed import utils as dist_utils

@@ -119,7 +119,6 @@ class Ernie4_5_VisionAttention(nn.Module):

119 num_heads: int,

120 projection_size: int,

121 quant_config: QuantizationConfig | None = None,

122 - multimodal_config: MultiModalConfig | None = None,

122 prefix: str = "",

123 ) -> None:

124 super().__init__()

@@ -153,7 +152,6 @@ class Ernie4_5_VisionAttention(nn.Module):

152 num_heads=self.num_attention_heads_per_partition,

153 head_size=self.hidden_size_per_attention_head,

154 scale=self.hidden_size_per_attention_head**-0.5,

156 - multimodal_config=multimodal_config,

155 prefix=f"{prefix}.attn",

156 )

157

@@ -266,7 +264,6 @@ class Ernie4_5_VisionBlock(nn.Module):

264 act_layer: type[nn.Module] = QuickGELU,

265 norm_layer: Callable[[int], nn.Module] | None = None,

266 quant_config: QuantizationConfig | None = None,

269 - multimodal_config: MultiModalConfig | None = None,

267 prefix: str = "",

268 ) -> None:

269 super().__init__()

@@ -282,7 +279,6 @@ class Ernie4_5_VisionBlock(nn.Module):

279 num_heads=num_heads,

280 projection_size=dim,

281 quant_config=quant_config,

285 - multimodal_config=multimodal_config,

282 prefix=f"{prefix}.attn",

283 )

284

@@ -357,7 +353,6 @@ class Ernie4_5_VisionTransformer(nn.Module):

353 vision_config,

354 norm_eps: float = 1e-6,

355 quant_config: QuantizationConfig | None = None,

360 - multimodal_config: MultiModalConfig | None = None,

356 prefix: str = "",

357 ) -> None:

358 super().__init__()

@@ -393,7 +388,6 @@ class Ernie4_5_VisionTransformer(nn.Module):

388 mlp_ratio=mlp_ratio,

389 norm_layer=norm_layer,

390 quant_config=quant_config,

396 - multimodal_config=multimodal_config,

391 prefix=f"{prefix}.blocks.{layer_idx}",

392 )

393 for layer_idx in range(depth)

@@ -405,13 +399,9 @@ class Ernie4_5_VisionTransformer(nn.Module):

399 )

400 self.ln = nn.LayerNorm(hidden_size, eps=1e-6)

401

408 - attn_backend_override = (

409 - multimodal_config.mm_encoder_attn_backend if multimodal_config else None

410 - )

402 self.attn_backend = get_vit_attn_backend(

403 head_size=head_dim,

404 dtype=torch.get_default_dtype(),

414 - attn_backend_override=attn_backend_override,

405 )

406

407 @property

@@ -1308,7 +1298,6 @@ class Ernie4_5_VLMoeForConditionalGeneration(

1298 config.vision_config,

1299 norm_eps=getattr(config, "rms_norm_eps", 1e-6),

1300 quant_config=quant_config,

1311 - multimodal_config=multimodal_config,

1301 prefix=maybe_prefix(prefix, "vision_model"),

1302 )

1303 self.resampler_model = VariableResolutionResamplerModel(

vllm/model_executor/models/glm4_1v.py

+5 -30

@@ -46,7 +46,7 @@ from transformers.models.glm4v.image_processing_glm4v import (

46 from transformers.models.glm4v.video_processing_glm4v import Glm4vVideoProcessor

47 from transformers.video_utils import VideoMetadata

48

49 -from vllm.config import MultiModalConfig, VllmConfig

49 +from vllm.config import VllmConfig

50 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

51 from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state

52 from vllm.distributed import utils as dist_utils

@@ -107,6 +107,7 @@ from .utils import (

107 )

108 from .vision import (

109 get_vit_attn_backend,

110 + is_vit_use_data_parallel,

111 run_dp_sharded_mrope_vision_model,

112 )

113

@@ -196,15 +197,10 @@ class Glm4vVisionMLP(nn.Module):

197 hidden_features: int,

198 bias: bool = False,

199 quant_config: QuantizationConfig | None = None,

199 - multimodal_config: MultiModalConfig | None = None,

200 prefix: str = "",

201 ):

202 super().__init__()

203 - use_data_parallel = (

204 - multimodal_config.mm_encoder_tp_mode == "data"

205 - if multimodal_config

206 - else False

207 - )

203 + use_data_parallel = is_vit_use_data_parallel()

204 self.gate_up_proj = MergedColumnParallelLinear(

205 input_size=in_features,

206 output_sizes=[hidden_features] * 2,

@@ -258,16 +254,11 @@ class Glm4vVisionAttention(nn.Module):

254 num_heads: int,

255 projection_size: int,

256 quant_config: QuantizationConfig | None = None,

261 - multimodal_config: MultiModalConfig | None = None,

257 prefix: str = "",

258 ) -> None:

259 super().__init__()

260 # Per attention head and per partition values.

266 - use_data_parallel = (

267 - multimodal_config.mm_encoder_tp_mode == "data"

268 - if multimodal_config

269 - else False

270 - )

261 + use_data_parallel = is_vit_use_data_parallel()

262 self.tp_size = (

263 1 if use_data_parallel else get_tensor_model_parallel_world_size()

264 )

@@ -305,7 +296,6 @@ class Glm4vVisionAttention(nn.Module):

296 num_heads=self.num_attention_heads_per_partition,

297 head_size=self.hidden_size_per_attention_head,

298 scale=self.hidden_size_per_attention_head**-0.5,

308 - multimodal_config=multimodal_config,

299 )

300

301 self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)

@@ -373,7 +363,6 @@ class Glm4vVisionBlock(nn.Module):

363 mlp_hidden_dim: int,

364 norm_layer: Callable[[int], nn.Module] | None = None,

365 quant_config: QuantizationConfig | None = None,

376 - multimodal_config: MultiModalConfig | None = None,

366 prefix: str = "",

367 ) -> None:

368 super().__init__()

@@ -386,7 +375,6 @@ class Glm4vVisionBlock(nn.Module):

375 num_heads=num_heads,

376 projection_size=dim,

377 quant_config=quant_config,

389 - multimodal_config=multimodal_config,

378 prefix=f"{prefix}.attn",

379 )

380 self.mlp = Glm4vVisionMLP(

@@ -394,7 +382,6 @@ class Glm4vVisionBlock(nn.Module):

382 mlp_hidden_dim,

383 bias=False,

384 quant_config=quant_config,

397 - multimodal_config=multimodal_config,

385 prefix=f"{prefix}.mlp",

386 )

387

@@ -454,16 +441,11 @@ class Glm4vPatchMerger(nn.Module):

441 d_model: int,

442 context_dim: int,

443 quant_config: QuantizationConfig | None = None,

457 - multimodal_config: MultiModalConfig | None = None,

444 bias: bool = False,

445 prefix: str = "",

446 ) -> None:

447 super().__init__()

462 - use_data_parallel = (

463 - multimodal_config.mm_encoder_tp_mode == "data"

464 - if multimodal_config

465 - else False

466 - )

448 + use_data_parallel = is_vit_use_data_parallel()

449 self.hidden_size = d_model

450 self.proj = ColumnParallelLinear(

451 self.hidden_size,

@@ -619,13 +601,10 @@ class Glm4vVisionTransformer(nn.Module):

601 vision_config: Glm4vVisionConfig,

602 norm_eps: float = 1e-6,

603 quant_config: QuantizationConfig | None = None,

622 - multimodal_config: MultiModalConfig | None = None,

604 prefix: str = "",

605 ) -> None:

606 super().__init__()

607

627 - assert multimodal_config is not None, "multimodal_config must be provided"

628 -

608 patch_size = vision_config.patch_size

609 temporal_patch_size = vision_config.temporal_patch_size

610 in_channels = vision_config.in_channels

@@ -660,7 +639,6 @@ class Glm4vVisionTransformer(nn.Module):

639 mlp_hidden_dim=vision_config.out_hidden_size,

640 norm_layer=norm_layer,

641 quant_config=quant_config,

663 - multimodal_config=multimodal_config,

642 prefix=f"{prefix}.blocks.{layer_idx}",

643 )

644 for layer_idx in range(depth)

@@ -670,7 +648,6 @@ class Glm4vVisionTransformer(nn.Module):

648 d_model=vision_config.out_hidden_size,

649 context_dim=vision_config.intermediate_size,

650 quant_config=quant_config,

673 - multimodal_config=multimodal_config,

651 bias=False,

652 prefix=f"{prefix}.merger",

653 )

@@ -692,7 +669,6 @@ class Glm4vVisionTransformer(nn.Module):

669 self.attn_backend = get_vit_attn_backend(

670 head_size=head_dim,

671 dtype=torch.get_default_dtype(),

695 - attn_backend_override=multimodal_config.mm_encoder_attn_backend,

672 )

673

674 @property

@@ -1439,7 +1415,6 @@ class Glm4vForConditionalGeneration(

1415 config.vision_config,

1416 norm_eps=getattr(config, "rms_norm_eps", 1e-5),

1417 quant_config=quant_config,

1442 - multimodal_config=multimodal_config,

1418 prefix=maybe_prefix(prefix, "visual"),

1419 )

1420

vllm/model_executor/models/hunyuan_vision.py

+5 -26

@@ -33,7 +33,7 @@ import torch.nn as nn

33 import torch.nn.functional as F

34 from transformers import BatchFeature

35

36 -from vllm.config import MultiModalConfig, VllmConfig

36 +from vllm.config import VllmConfig

37 from vllm.config.multimodal import BaseDummyOptions

38 from vllm.distributed import parallel_state

39 from vllm.distributed import utils as dist_utils

@@ -80,7 +80,6 @@ from vllm.transformers_utils.configs.hunyuan_vl import (

80 from vllm.transformers_utils.processors.hunyuan_vl import HunYuanVLProcessor

81 from vllm.transformers_utils.processors.hunyuan_vl_image import smart_resize

82 from vllm.utils.tensor_schema import TensorSchema, TensorShape

83 -from vllm.v1.attention.backends.registry import AttentionBackendEnum

83

84 from .interfaces import (

85 MultiModalEmbeddings,

@@ -96,6 +95,7 @@ from .utils import (

95 init_vllm_registered_model,

96 maybe_prefix,

97 )

98 +from .vision import is_vit_use_data_parallel

99

100 logger = init_logger(__name__)

101

@@ -160,9 +160,9 @@ class HunYuanVisionMLP(nn.Module):

160 act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu,

161 quant_config: QuantizationConfig | None = None,

162 prefix: str = "",

163 - use_data_parallel: bool = False,

163 ):

164 super().__init__()

165 + use_data_parallel = is_vit_use_data_parallel()

166 self.dense_h_to_4h = ColumnParallelLinear(

167 in_features,

168 hidden_features,

@@ -194,12 +194,11 @@ class HunYuanVisionAttention(nn.Module):

194 num_heads: int,

195 projection_size: int,

196 quant_config: QuantizationConfig | None = None,

197 - multimodal_config: MultiModalConfig | None = None,

197 prefix: str = "",

199 - use_data_parallel: bool = False,

198 ) -> None:

199 super().__init__()

200 # Per attention head and per partition values.

201 + use_data_parallel = is_vit_use_data_parallel()

202 self.tp_size = (

203 1

204 if use_data_parallel

@@ -237,7 +236,6 @@ class HunYuanVisionAttention(nn.Module):

236 self.hidden_size_per_attention_head,

237 self.scale,

238 prefix=f"{prefix}.attn",

240 - multimodal_config=multimodal_config,

239 )

240

241 def forward(

@@ -260,9 +258,7 @@ class HunYuanVisionBlock(nn.Module):

258 act_fn: Callable[[torch.Tensor], torch.Tensor] = F.gelu,

259 norm_layer: Callable[[int], nn.Module] | None = None,

260 quant_config: QuantizationConfig | None = None,

263 - multimodal_config: MultiModalConfig | None = None,

261 prefix: str = "",

265 - use_data_parallel: bool = False,

262 ) -> None:

263 super().__init__()

264 if norm_layer is None:

@@ -274,9 +270,7 @@ class HunYuanVisionBlock(nn.Module):

270 num_heads=num_heads,

271 projection_size=dim,

272 quant_config=quant_config,

277 - multimodal_config=multimodal_config,

273 prefix=f"{prefix}.self_attn",

279 - use_data_parallel=use_data_parallel,

274 )

275 self.mlp = HunYuanVisionMLP(

276 dim,

@@ -285,7 +279,6 @@ class HunYuanVisionBlock(nn.Module):

279 bias=True,

280 quant_config=quant_config,

281 prefix=f"{prefix}.mlp",

288 - use_data_parallel=use_data_parallel,

282 )

283

284 def forward(

@@ -439,9 +432,6 @@ class HunYuanVisionTransformer(nn.Module):

432 vision_config: HunYuanVLVisionConfig,

433 quant_config: QuantizationConfig | None = None,

434 prefix: str = "",

442 - use_data_parallel: bool = False,

443 - multimodal_config: MultiModalConfig | None = None,

444 - attn_backend_override: AttentionBackendEnum | None = None,

435 ) -> None:

436 super().__init__()

437

@@ -467,9 +457,7 @@ class HunYuanVisionTransformer(nn.Module):

457 act_fn=get_act_fn(vision_config.hidden_act),

458 norm_layer=norm_layer,

459 quant_config=quant_config,

470 - multimodal_config=multimodal_config,

460 prefix=f"{prefix}.layers.{layer_idx}",

472 - use_data_parallel=use_data_parallel,

461 )

462 for layer_idx in range(num_hidden_layers)

463 ]

@@ -872,23 +860,14 @@ class HunYuanVLForConditionalGeneration(

860 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

861 super().__init__()

862 config: HunYuanVLConfig = vllm_config.model_config.hf_config

875 - multimodal_config = vllm_config.model_config.multimodal_config

863

864 self.config = config

878 - self.multimodal_config = multimodal_config

865

866 with self._mark_tower_model(vllm_config, {"image"}):

881 - attn_backend_override = (

882 - multimodal_config.mm_encoder_attn_backend

883 - if multimodal_config is not None

884 - else None

885 - )

867 self.visual = HunYuanVisionTransformer(

868 config.vision_config,

888 - quant_config=self.quant_config,

869 + quant_config=vllm_config.quant_config,

870 prefix=maybe_prefix(prefix, "visual"),

890 - multimodal_config=multimodal_config,

891 - attn_backend_override=attn_backend_override,

871 )

872

873 with self._mark_language_model(vllm_config):

vllm/model_executor/models/hyperclovax_vision.py

+1 -6

@@ -17,7 +17,7 @@ from timm.models.regnet import RegStage

17 from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig

18

19 from vllm.config import VllmConfig

20 -from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

20 +from vllm.config.multimodal import BaseDummyOptions

21 from vllm.model_executor.layers.quantization import QuantizationConfig

22 from vllm.multimodal import MULTIMODAL_REGISTRY

23 from vllm.multimodal.cache import BaseMultiModalProcessorCache

@@ -360,7 +360,6 @@ def _build_hcxvision_hf_processor(

360 def init_vision_tower_for_hcxvision(

361 vision_config,

362 quant_config: QuantizationConfig | None,

363 - multimodal_config: MultiModalConfig | None,

363 *,

364 use_nth_layer: int | None = None,

365 require_post_norm: bool | None = None,

@@ -378,7 +377,6 @@ def init_vision_tower_for_hcxvision(

377 return CLIPVisionModel(

378 vision_config,

379 quant_config=quant_config,

381 - multimodal_config=multimodal_config,

380 num_hidden_layers_override=num_hidden_layers,

381 require_post_norm=require_post_norm,

382 prefix=prefix,

@@ -387,7 +385,6 @@ def init_vision_tower_for_hcxvision(

385 return SiglipVisionModel(

386 vision_config,

387 quant_config=quant_config,

390 - multimodal_config=multimodal_config,

388 num_hidden_layers_override=num_hidden_layers,

389 require_post_norm=require_post_norm,

390 prefix=prefix,

@@ -605,7 +602,6 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

602 # init configs

603 config = vllm_config.model_config.hf_config

604 quant_config = vllm_config.quant_config

608 - multimodal_config = vllm_config.model_config.multimodal_config

605 # text_config

606 text_config = config.text_config

607 if text_config.model_type in ["gpt2", "hyperclovax", "llama"]:

@@ -628,7 +624,6 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

624 self.vision_model = init_vision_tower_for_hcxvision(

625 vision_config,

626 quant_config=quant_config,

631 - multimodal_config=multimodal_config,

627 use_nth_layer=getattr(config, "use_nth_layer", -1),

628 require_post_norm=False,

629 prefix=maybe_prefix(prefix, "vision_model"),

vllm/model_executor/models/isaac.py

+4 -18

@@ -16,7 +16,7 @@ from transformers.image_processing_utils import BatchFeature

16 from transformers.tokenization_utils import TensorType

17 from typing_extensions import TypedDict, Unpack

18

19 -from vllm.config import MultiModalConfig, VllmConfig

19 +from vllm.config import VllmConfig

20 from vllm.config.model import ModelConfig

21 from vllm.distributed import parallel_state

22 from vllm.distributed import utils as dist_utils

@@ -72,6 +72,8 @@ from vllm.transformers_utils.configs import (

72 )

73 from vllm.utils.tensor_schema import TensorSchema, TensorShape

74

75 +from .vision import is_vit_use_data_parallel

76 +

77

78 def create_cumulative_seq_lengths(

79 seq_sizes: torch.Tensor, device: torch.device

@@ -942,15 +944,10 @@ class Siglip2VisionAttention(nn.Module):

944 quant_config: QuantizationConfig | None = None,

945 *,

946 prefix: str = "",

945 - multimodal_config: MultiModalConfig | None = None,

947 ) -> None:

948 super().__init__()

949

949 - use_data_parallel = (

950 - multimodal_config.mm_encoder_tp_mode == "data"

951 - if multimodal_config

952 - else False

953 - )

950 + use_data_parallel = is_vit_use_data_parallel()

951 self.tp_size = (

952 1

953 if use_data_parallel

@@ -987,7 +984,6 @@ class Siglip2VisionAttention(nn.Module):

984 head_size=self.hidden_size_per_attention_head,

985 scale=self.hidden_size_per_attention_head**-0.5,

986 prefix=f"{prefix}.attn",

990 - multimodal_config=multimodal_config,

987 )

988

989 def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:

@@ -1038,7 +1034,6 @@ class Siglip2EncoderLayer(nn.Module):

1034 quant_config: QuantizationConfig | None = None,

1035 *,

1036 prefix: str = "",

1041 - multimodal_config: MultiModalConfig | None = None,

1037 ) -> None:

1038 super().__init__()

1039 self.embed_dim = config.hidden_size

@@ -1047,7 +1042,6 @@ class Siglip2EncoderLayer(nn.Module):

1042 config,

1043 quant_config=quant_config,

1044 prefix=f"{prefix}.self_attn",

1050 - multimodal_config=multimodal_config,

1045 )

1046 self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)

1047 self.mlp = SiglipMLP(

@@ -1088,7 +1082,6 @@ class Siglip2Encoder(nn.Module):

1082 quant_config: QuantizationConfig | None = None,

1083 *,

1084 prefix: str = "",

1091 - multimodal_config: MultiModalConfig | None = None,

1085 ) -> None:

1086 super().__init__()

1087 self.config = config

@@ -1098,7 +1091,6 @@ class Siglip2Encoder(nn.Module):

1091 config,

1092 quant_config=quant_config,

1093 prefix=f"{prefix}.layers.{layer_idx}",

1101 - multimodal_config=multimodal_config,

1094 )

1095 for layer_idx in range(config.num_hidden_layers)

1096 ]

@@ -1127,7 +1119,6 @@ class Siglip2VisionTransformer(nn.Module):

1119 config: PixelShuffleSiglip2VisionConfig,

1120 quant_config: QuantizationConfig | None = None,

1121 prefix: str = "",

1130 - multimodal_config: MultiModalConfig | None = None,

1122 ):

1123 super().__init__()

1124 self.config = config

@@ -1140,7 +1131,6 @@ class Siglip2VisionTransformer(nn.Module):

1131 config,

1132 quant_config=quant_config,

1133 prefix=f"{prefix}.encoder",

1143 - multimodal_config=multimodal_config,

1134 )

1135 self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)

1136

@@ -1221,14 +1211,12 @@ class IsaacVisionEmbedding(nn.Module):

1211 hidden_dim: int,

1212 output_dim: int,

1213 quant_config: QuantizationConfig | None = None,

1224 - multimodal_config: MultiModalConfig | None = None,

1214 prefix: str = "",

1215 ):

1216 super().__init__()

1217 self.transformer = Siglip2VisionTransformer(

1218 vision_cfg,

1219 quant_config=quant_config,

1231 - multimodal_config=multimodal_config,

1220 prefix=maybe_prefix(prefix, "0"),

1221 )

1222 self.linear_fc1 = ColumnParallelLinear(

@@ -1309,7 +1297,6 @@ class IsaacForConditionalGeneration(

1297 config: IsaacConfig = vllm_config.model_config.hf_config

1298 quant_config = vllm_config.quant_config

1299 self.config = config

1312 - self.multimodal_config = vllm_config.model_config.multimodal_config

1300

1301 head_dim = config.head_dim

1302 calculated_mrope_section = [

@@ -1373,7 +1360,6 @@ class IsaacForConditionalGeneration(

1360 hidden_dim=hidden_dim,

1361 output_dim=config.hidden_size,

1362 quant_config=quant_config,

1376 - multimodal_config=self.multimodal_config,

1363 prefix=maybe_prefix(prefix, "vision_embedding"),

1364 )

1365

vllm/model_executor/models/keye.py

+4 -16

@@ -16,7 +16,7 @@ from transformers.feature_extraction_utils import BatchFeature

16 from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling

17 from transformers.utils import torch_int

18

19 -from vllm.config import MultiModalConfig, VllmConfig

19 +from vllm.config import VllmConfig

20 from vllm.config.multimodal import BaseDummyOptions

21 from vllm.distributed import get_tensor_model_parallel_world_size

22 from vllm.logger import init_logger

@@ -80,6 +80,7 @@ from .utils import (

80 is_pp_missing_parameter,

81 maybe_prefix,

82 )

83 +from .vision import is_vit_use_data_parallel

84

85 logger = init_logger(__name__)

86

@@ -358,7 +359,6 @@ class KeyeSiglipAttention(nn.Module):

359 self,

360 config: PretrainedConfig,

361 quant_config: QuantizationConfig | None = None,

361 - multimodal_config: MultiModalConfig | None = None,

362 prefix: str = "",

363 ):

364 super().__init__()

@@ -366,7 +366,8 @@ class KeyeSiglipAttention(nn.Module):

366

367 hidden_size = config.hidden_size

368 self.hidden_size = config.hidden_size

369 - tp_size = get_tensor_model_parallel_world_size()

369 + use_data_parallel = is_vit_use_data_parallel()

370 + tp_size = 1 if use_data_parallel else get_tensor_model_parallel_world_size()

371 self.total_num_heads = config.num_attention_heads

372 assert self.total_num_heads % tp_size == 0

373 self.num_heads = self.total_num_heads // tp_size

@@ -403,7 +404,6 @@ class KeyeSiglipAttention(nn.Module):

404 scale=self.scale,

405 num_kv_heads=self.num_kv_heads,

406 prefix=f"{prefix}.attn",

406 - multimodal_config=multimodal_config,

407 )

408

409 self.apply_rotary_emb = ApplyRotaryEmb(

@@ -497,7 +497,6 @@ class KeyeSiglipEncoderLayer(nn.Module):

497 self,

498 config: PretrainedConfig,

499 quant_config: QuantizationConfig | None = None,

500 - multimodal_config: MultiModalConfig | None = None,

500 prefix: str = "",

501 ):

502 super().__init__()

@@ -506,14 +505,12 @@ class KeyeSiglipEncoderLayer(nn.Module):

505 self.self_attn = KeyeSiglipAttention(

506 config,

507 quant_config=quant_config,

509 - multimodal_config=multimodal_config,

508 prefix=f"{prefix}.self_attn",

509 )

510 self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)

511 self.mlp = SiglipMLP(

512 config,

513 quant_config=quant_config,

516 - multimodal_config=multimodal_config,

514 prefix=f"{prefix}.mlp",

515 )

516

@@ -552,7 +549,6 @@ class KeyeSiglipEncoder(nn.Module):

549 self,

550 config: PretrainedConfig,

551 quant_config: QuantizationConfig | None = None,

555 - multimodal_config: MultiModalConfig | None = None,

552 prefix: str = "",

553 ):

554 super().__init__()

@@ -565,7 +561,6 @@ class KeyeSiglipEncoder(nn.Module):

561 KeyeSiglipEncoderLayer(

562 config,

563 quant_config=quant_config,

568 - multimodal_config=multimodal_config,

564 prefix=f"{prefix}.layers.{layer_idx}",

565 )

566 for layer_idx in range(config.num_hidden_layers)

@@ -647,7 +642,6 @@ class KeyeSiglipVisionTransformer(nn.Module):

642 self,

643 config: PretrainedConfig,

644 quant_config: QuantizationConfig | None = None,

650 - multimodal_config: MultiModalConfig | None = None,

645 prefix: str = "",

646 ):

647 super().__init__()

@@ -658,7 +652,6 @@ class KeyeSiglipVisionTransformer(nn.Module):

652 self.encoder = KeyeSiglipEncoder(

653 config,

654 quant_config=quant_config,

661 - multimodal_config=multimodal_config,

655 prefix=f"{prefix}.encoder",

656 )

657 self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)

@@ -730,7 +723,6 @@ class KeyeSiglipVisionModel(nn.Module):

723 self,

724 config: PretrainedConfig,

725 quant_config: QuantizationConfig | None = None,

733 - multimodal_config: MultiModalConfig | None = None,

726 prefix: str = "",

727 ):

728 super().__init__()

@@ -738,7 +730,6 @@ class KeyeSiglipVisionModel(nn.Module):

730 self.vision_model = KeyeSiglipVisionTransformer(

731 config,

732 quant_config=quant_config,

741 - multimodal_config=multimodal_config,

733 prefix=f"{prefix}.vision_model",

734 )

735 self.quant_config = quant_config

@@ -1275,16 +1266,13 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):

1266 super().__init__()

1267 config: PretrainedConfig = vllm_config.model_config.hf_config

1268 quant_config = vllm_config.quant_config

1278 - multimodal_config = vllm_config.model_config.multimodal_config

1269

1270 self.config = config

1281 - self.multimodal_config = multimodal_config

1271

1272 with self._mark_tower_model(vllm_config, {"image", "video"}):

1273 self.visual = KeyeSiglipVisionModel(

1274 config.vision_config,

1275 quant_config=quant_config,

1287 - multimodal_config=multimodal_config,

1276 prefix=maybe_prefix(prefix, "visual"),

1277 )

1278 self.mlp_AR = self._build_projector(

vllm/model_executor/models/kimi_vl.py

-1

@@ -317,7 +317,6 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

317 with self._mark_tower_model(vllm_config, "image"):

318 self.vision_tower = MoonVitPretrainedModel(

319 config.vision_config,

320 - multimodal_config=model_config.multimodal_config,

320 prefix=maybe_prefix(prefix, "vision_tower"),

321 )

322 self.multi_modal_projector = KimiVLMultiModalProjector(

vllm/model_executor/models/lfm2_siglip2.py

+3 -22

@@ -11,7 +11,6 @@ from torch.nn import functional as F

11 from transformers import Siglip2VisionConfig

12

13 from vllm.compilation.decorators import support_torch_compile

14 -from vllm.config import MultiModalConfig

14 from vllm.distributed import get_tensor_model_parallel_world_size

15 from vllm.model_executor.layers.activation import get_act_fn

16 from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention

@@ -23,7 +22,7 @@ from vllm.model_executor.layers.linear import (

22 from vllm.model_executor.layers.quantization import QuantizationConfig

23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

24

26 -from .vision import should_torch_compile_mm_vit

25 +from .vision import is_vit_use_data_parallel, should_torch_compile_mm_vit

26

27

28 class Siglip2VisionEmbeddings(nn.Module):

@@ -154,7 +153,6 @@ class Siglip2Attention(nn.Module):

153 self,

154 config: Siglip2VisionConfig,

155 quant_config: QuantizationConfig | None = None,

157 - multimodal_config: MultiModalConfig | None = None,

156 prefix: str = "",

157 ):

158 super().__init__()

@@ -171,10 +169,7 @@ class Siglip2Attention(nn.Module):

169 self.scale = self.head_dim**-0.5

170 self.dropout = config.attention_dropout

171

174 - use_data_parallel = (

175 - multimodal_config is not None

176 - and multimodal_config.mm_encoder_tp_mode == "data"

177 - )

172 + use_data_parallel = is_vit_use_data_parallel()

173 tp_size = 1 if use_data_parallel else get_tensor_model_parallel_world_size()

174 assert self.num_heads % tp_size == 0

175 self.num_heads_per_partition = self.num_heads // tp_size

@@ -199,7 +194,6 @@ class Siglip2Attention(nn.Module):

194 head_size=self.head_dim,

195 scale=self.scale,

196 prefix=f"{prefix}.attn",

202 - multimodal_config=multimodal_config,

197 )

198

199 def forward(

@@ -241,16 +235,12 @@ class Siglip2MLP(nn.Module):

235 self,

236 config: Siglip2VisionConfig,

237 quant_config: QuantizationConfig | None = None,

244 - multimodal_config: MultiModalConfig | None = None,

238 prefix: str = "",

239 ):

240 super().__init__()

241 self.config = config

242 self.activation_fn = get_act_fn(config.hidden_act)

250 - use_data_parallel = (

251 - multimodal_config is not None

252 - and multimodal_config.mm_encoder_tp_mode == "data"

253 - )

243 + use_data_parallel = is_vit_use_data_parallel()

244 self.fc1 = ColumnParallelLinear(

245 config.hidden_size,

246 config.intermediate_size,

@@ -282,7 +272,6 @@ class Siglip2EncoderLayer(nn.Module):

272 self,

273 config: Siglip2VisionConfig,

274 quant_config: QuantizationConfig | None = None,

285 - multimodal_config: MultiModalConfig | None = None,

275 prefix: str = "",

276 ):

277 super().__init__()

@@ -291,14 +280,12 @@ class Siglip2EncoderLayer(nn.Module):

280 self.self_attn = Siglip2Attention(

281 config,

282 quant_config=quant_config,

294 - multimodal_config=multimodal_config,

283 prefix=f"{prefix}.self_attn",

284 )

285 self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)

286 self.mlp = Siglip2MLP(

287 config,

288 quant_config=quant_config,

301 - multimodal_config=multimodal_config,

289 prefix=f"{prefix}.mlp",

290 )

291

@@ -344,7 +331,6 @@ class Siglip2Encoder(nn.Module):

331 self,

332 config: Siglip2VisionConfig,

333 quant_config: QuantizationConfig | None = None,

347 - multimodal_config: MultiModalConfig | None = None,

334 prefix: str = "",

335 ):

336 super().__init__()

@@ -354,7 +340,6 @@ class Siglip2Encoder(nn.Module):

340 Siglip2EncoderLayer(

341 config=config,

342 quant_config=quant_config,

357 - multimodal_config=multimodal_config,

343 prefix=f"{prefix}.layers.{idx}",

344 )

345 for idx in range(config.num_hidden_layers)

@@ -383,7 +368,6 @@ class Siglip2VisionTransformer(nn.Module):

368 self,

369 config: Siglip2VisionConfig,

370 quant_config: QuantizationConfig | None = None,

386 - multimodal_config: MultiModalConfig | None = None,

371 prefix: str = "",

372 ):

373 super().__init__()

@@ -397,7 +381,6 @@ class Siglip2VisionTransformer(nn.Module):

381 self.encoder = Siglip2Encoder(

382 config,

383 quant_config=quant_config,

400 - multimodal_config=multimodal_config,

384 prefix=f"{prefix}.encoder",

385 )

386 num_hidden_layers = config.num_hidden_layers

@@ -438,7 +421,6 @@ class Siglip2Model(torch.nn.Module):

421 self,

422 config: Siglip2VisionConfig,

423 quant_config: QuantizationConfig | None = None,

441 - multimodal_config: MultiModalConfig | None = None,

424 prefix: str = "",

425 ):

426 super().__init__()

@@ -446,7 +428,6 @@ class Siglip2Model(torch.nn.Module):

428 self.vision_model = Siglip2VisionTransformer(

429 config,

430 quant_config=quant_config,

449 - multimodal_config=multimodal_config,

431 prefix=f"{prefix}.vision_model",

432 )

433

vllm/model_executor/models/lfm2_vl.py

-1

@@ -600,7 +600,6 @@ class Lfm2VLForConditionalGeneration(

600 self.vision_tower = Siglip2Model(

601 config=vision_config,

602 quant_config=quant_config,

603 - multimodal_config=multimodal_config,

603 prefix=maybe_prefix(prefix, "vision_tower"),

604 )

605 else:

vllm/model_executor/models/lightonocr.py

-1

@@ -166,7 +166,6 @@ class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):

166 self.vision_tower = init_vision_tower_for_llava(

167 config,

168 quant_config=quant_config,

169 - multimodal_config=multimodal_config,

169 require_post_norm=False,

170 prefix=maybe_prefix(prefix, "vision_tower"),

171 )

vllm/model_executor/models/llava.py

+1 -6

@@ -19,7 +19,7 @@ from transformers.models.llava import LlavaProcessor

19 from transformers.models.pixtral import PixtralProcessor

20

21 from vllm.config import VllmConfig

22 -from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

22 +from vllm.config.multimodal import BaseDummyOptions

23 from vllm.model_executor.layers.activation import get_act_fn

24 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

25 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -456,7 +456,6 @@ def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:

456 def init_vision_tower_for_llava(

457 hf_config: LlavaLikeConfig,

458 quant_config: QuantizationConfig | None,

459 - multimodal_config: MultiModalConfig | None,

459 *,

460 require_post_norm: bool | None = None,

461 prefix: str = "",

@@ -470,7 +469,6 @@ def init_vision_tower_for_llava(

469 return CLIPVisionModel(

470 vision_config,

471 quant_config=quant_config,

473 - multimodal_config=multimodal_config,

472 num_hidden_layers_override=num_hidden_layers,

473 require_post_norm=require_post_norm,

474 prefix=prefix,

@@ -479,7 +477,6 @@ def init_vision_tower_for_llava(

477 return SiglipVisionModel(

478 vision_config,

479 quant_config=quant_config,

482 - multimodal_config=multimodal_config,

480 num_hidden_layers_override=num_hidden_layers,

481 require_post_norm=require_post_norm,

482 prefix=prefix,

@@ -488,7 +485,6 @@ def init_vision_tower_for_llava(

485 return PixtralHFVisionModel(

486 vision_config,

487 quant_config=quant_config,

491 - multimodal_config=multimodal_config,

488 num_hidden_layers_override=num_hidden_layers,

489 require_post_norm=require_post_norm,

490 prefix=prefix,

@@ -562,7 +558,6 @@ class LlavaForConditionalGeneration(

558 self.vision_tower = init_vision_tower_for_llava(

559 config,

560 quant_config=quant_config,

565 - multimodal_config=multimodal_config,

561 require_post_norm=False,

562 prefix=maybe_prefix(prefix, "vision_tower"),

563 )

vllm/model_executor/models/llava_next.py

-1

@@ -272,7 +272,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP

272 self.vision_tower = init_vision_tower_for_llava(

273 config,

274 quant_config=quant_config,

275 - multimodal_config=multimodal_config,

275 require_post_norm=False,

276 prefix=maybe_prefix(prefix, "vision_tower"),

277 )

vllm/model_executor/models/llava_next_video.py

-1

@@ -332,7 +332,6 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp

332 self.vision_tower = init_vision_tower_for_llava(

333 config,

334 quant_config=quant_config,

335 - multimodal_config=multimodal_config,

335 require_post_norm=False,

336 prefix=maybe_prefix(prefix, "vision_tower"),

337 )

vllm/model_executor/models/llava_onevision.py

-1

@@ -513,7 +513,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp

513 self.vision_tower = init_vision_tower_for_llava(

514 config,

515 quant_config=quant_config,

516 - multimodal_config=multimodal_config,

516 require_post_norm=False,

517 prefix=maybe_prefix(prefix, "vision_tower"),

518 )

vllm/model_executor/models/minimax_vl_01.py

-1

@@ -205,7 +205,6 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support

205 self.vision_tower = init_vision_tower_for_llava(

206 config,

207 quant_config=quant_config,

208 - multimodal_config=multimodal_config,

208 require_post_norm=False,

209 prefix=maybe_prefix(prefix, "vision_tower"),

210 )

vllm/model_executor/models/mistral3.py

+1 -4

@@ -16,7 +16,7 @@ from transformers import (

16 from transformers.models.pixtral import PixtralProcessor

17

18 from vllm.config import VllmConfig

19 -from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

19 +from vllm.config.multimodal import BaseDummyOptions

20 from vllm.model_executor.layers.activation import get_act_fn

21 from vllm.model_executor.layers.layernorm import RMSNorm

22 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

@@ -382,7 +382,6 @@ def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:

382 def init_vision_tower_for_llava(

383 hf_config: LlavaLikeConfig,

384 quant_config: QuantizationConfig | None,

385 - multimodal_config: MultiModalConfig | None,

385 *,

386 require_post_norm: bool | None = None,

387 prefix: str = "",

@@ -397,7 +396,6 @@ def init_vision_tower_for_llava(

396 return PixtralHFVisionModel(

397 vision_config,

398 quant_config=quant_config,

400 - multimodal_config=multimodal_config,

399 num_hidden_layers_override=num_hidden_layers,

400 require_post_norm=require_post_norm,

401 prefix=prefix,

@@ -461,7 +459,6 @@ class Mistral3ForConditionalGeneration(

459 self.vision_tower = init_vision_tower_for_llava(

460 config,

461 quant_config=quant_config,

464 - multimodal_config=multimodal_config,

462 require_post_norm=False,

463 prefix=maybe_prefix(prefix, "vision_tower"),

464 )

vllm/model_executor/models/moonvit.py

+3 -15

@@ -52,7 +52,6 @@ import torch.nn.functional as F

52 from transformers.activations import ACT2FN

53 from transformers.modeling_utils import PreTrainedModel

54

55 -from vllm.config import MultiModalConfig

55 from vllm.distributed import divide, get_tensor_model_parallel_world_size

56 from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention

57 from vllm.model_executor.layers.conv import Conv2dLayer

@@ -62,6 +61,7 @@ from vllm.model_executor.layers.linear import (

61 RowParallelLinear,

62 )

63 from vllm.model_executor.models.utils import maybe_prefix

64 +from vllm.model_executor.models.vision import is_vit_use_data_parallel

65 from vllm.platforms import current_platform

66 from vllm.transformers_utils.configs.moonvit import MoonViTConfig

67

@@ -308,11 +308,10 @@ class MLP2(nn.Module):

308 activation,

309 bias: bool = True,

310 prefix: str = "",

311 - use_data_parallel: bool = False,

311 ):

312 super().__init__()

313 assert len(dims) == 3

315 - self.use_data_parallel = use_data_parallel

314 + self.use_data_parallel = is_vit_use_data_parallel()

315 self.fc0 = ColumnParallelLinear(

316 dims[0],

317 dims[1],

@@ -343,17 +342,12 @@ class MoonVitEncoderLayer(nn.Module):

342 hidden_dim: int,

343 mlp_dim: int,

344 prefix: str = "",

346 - multimodal_config: MultiModalConfig | None = None,

345 *,

346 activation=F.gelu,

347 attn_bias: bool = False,

348 ):

349 super().__init__()

352 - self.use_data_parallel = (

353 - multimodal_config.mm_encoder_tp_mode == "data"

354 - if multimodal_config

355 - else False

356 - )

350 + self.use_data_parallel = is_vit_use_data_parallel()

351

352 self.num_heads = num_heads

353 self.hidden_dim = hidden_dim

@@ -369,7 +363,6 @@ class MoonVitEncoderLayer(nn.Module):

363 [hidden_dim, mlp_dim, hidden_dim],

364 activation,

365 prefix=f"{prefix}.mlp",

372 - use_data_parallel=self.use_data_parallel,

366 )

367 self.wqkv = QKVParallelLinear(

368 hidden_size=hidden_dim,

@@ -391,7 +384,6 @@ class MoonVitEncoderLayer(nn.Module):

384 num_heads=self.num_attention_heads_per_partition,

385 head_size=self.hidden_size_per_attention_head,

386 scale=self.hidden_size_per_attention_head**-0.5,

394 - multimodal_config=multimodal_config,

387 prefix=f"{prefix}.attn",

388 )

389

@@ -469,7 +461,6 @@ class MoonVitEncoder(nn.Module):

461 num_layers: int,

462 block_cfg: dict,

463 prefix: str = "",

472 - multimodal_config: MultiModalConfig | None = None,

464 ) -> None:

465 super().__init__()

466

@@ -479,7 +470,6 @@ class MoonVitEncoder(nn.Module):

470 self.blocks = nn.ModuleList(

471 [

472 MoonVitEncoderLayer(

482 - multimodal_config=multimodal_config,

473 prefix=f"{prefix}.blocks.{layer_idx}",

474 **block_cfg,

475 )

@@ -550,7 +540,6 @@ class MoonVitPretrainedModel(PreTrainedModel):

540 def __init__(

541 self,

542 config: MoonViTConfig,

553 - multimodal_config: MultiModalConfig | None = None,

543 prefix: str = "",

544 *inputs,

545 **kwargs,

@@ -579,7 +568,6 @@ class MoonVitPretrainedModel(PreTrainedModel):

568 "attn_bias": True,

569 },

570 prefix=f"{prefix}.encoder",

582 - multimodal_config=multimodal_config,

571 )

572

573 def forward(

vllm/model_executor/models/opencua.py

-1

@@ -244,7 +244,6 @@ class OpenCUAForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):

244 vision_config=config.vision_config,

245 norm_eps=getattr(config, "rms_norm_eps", 1e-6),

246 quant_config=self.quant_config,

247 - multimodal_config=self.multimodal_config,

247 prefix=maybe_prefix(prefix, "visual"),

248 )

249

vllm/model_executor/models/ovis2_5.py

+1 -7

@@ -10,7 +10,7 @@ import torch

10 import torch.nn as nn

11 from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig

12

13 -from vllm.config import MultiModalConfig, VllmConfig

13 +from vllm.config import VllmConfig

14 from vllm.config.multimodal import BaseDummyOptions

15 from vllm.model_executor.layers.linear import ReplicatedLinear

16 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -103,7 +103,6 @@ class VisualTokenizer(torch.nn.Module):

103 config: PretrainedConfig,

104 visual_vocab_size: int,

105 quant_config: QuantizationConfig | None = None,

106 - multimodal_config: MultiModalConfig | None = None,

106 prefix: str = "",

107 ):

108 super().__init__()

@@ -111,7 +110,6 @@ class VisualTokenizer(torch.nn.Module):

110 self.vit = self._init_backbone(

111 config=config,

112 quant_config=quant_config,

114 - multimodal_config=multimodal_config,

113 prefix=f"{prefix}.vit",

114 )

115 # reserved tokens for INDICATOR_IDS

@@ -130,7 +128,6 @@ class VisualTokenizer(torch.nn.Module):

128 self,

129 config: PretrainedConfig,

130 quant_config: QuantizationConfig | None = None,

133 - multimodal_config: QuantizationConfig | None = None,

131 prefix: str = "",

132 ):

133 model_type = config.model_type

@@ -138,7 +135,6 @@ class VisualTokenizer(torch.nn.Module):

135 return Siglip2NavitModel(

136 config=config,

137 quant_config=quant_config,

141 - multimodal_config=multimodal_config,

138 prefix=prefix,

139 )

140 raise ValueError(f"Unsupported visual tokenizer model_type: {model_type}")

@@ -464,7 +460,6 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):

460 super().__init__()

461 config = vllm_config.model_config.hf_config

462 quant_config = vllm_config.quant_config

467 - multimodal_config = vllm_config.model_config.multimodal_config

463

464 self.config: PretrainedConfig = config

465

@@ -478,7 +473,6 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):

473 self.visual_tokenizer = VisualTokenizer(

474 config=config.vit_config,

475 visual_vocab_size=config.visual_vocab_size,

481 - multimodal_config=multimodal_config,

476 quant_config=quant_config,

477 prefix=f"{prefix}.visual_tokenizer",

478 )

vllm/model_executor/models/paddleocr_vl.py

+1 -19

@@ -30,7 +30,7 @@ from transformers.modeling_outputs import (

30 )

31 from transformers.utils import torch_int

32

33 -from vllm.config import MultiModalConfig, VllmConfig

33 +from vllm.config import VllmConfig

34 from vllm.config.multimodal import BaseDummyOptions

35 from vllm.distributed import parallel_state

36 from vllm.distributed import utils as dist_utils

@@ -532,7 +532,6 @@ class SiglipAttention(nn.Module):

532 num_heads: int,

533 projection_size: int,

534 quant_config: QuantizationConfig | None = None,

535 - multimodal_config: MultiModalConfig | None = None,

535 prefix: str = "",

536 ) -> None:

537 super().__init__()

@@ -565,7 +564,6 @@ class SiglipAttention(nn.Module):

564 num_heads=self.num_attention_heads_per_partition,

565 head_size=self.hidden_size_per_attention_head,

566 scale=self.hidden_size_per_attention_head**-0.5,

568 - multimodal_config=multimodal_config,

567 prefix=f"{prefix}.attn",

568 )

569 self.apply_rotary_emb = ApplyRotaryEmb(

@@ -662,7 +660,6 @@ class SiglipEncoderLayer(nn.Module):

660 self,

661 config: PretrainedConfig,

662 quant_config: QuantizationConfig | None = None,

665 - multimodal_config: MultiModalConfig | None = None,

663 prefix: str = "",

664 ):

665 super().__init__()

@@ -673,14 +670,12 @@ class SiglipEncoderLayer(nn.Module):

670 num_heads=config.num_attention_heads,

671 projection_size=config.hidden_size,

672 quant_config=quant_config,

676 - multimodal_config=multimodal_config,

673 prefix=f"{prefix}.self_attn",

674 )

675 self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)

676 self.mlp = SiglipMLP(

677 config,

678 quant_config=quant_config,

683 - multimodal_config=multimodal_config,

679 prefix=f"{prefix}.mlp",

680 )

681

@@ -718,7 +713,6 @@ class SiglipEncoder(nn.Module):

713 self,

714 config: PretrainedConfig,

715 quant_config: QuantizationConfig | None = None,

721 - multimodal_config: MultiModalConfig | None = None,

716 prefix: str = "",

717 ):

718 super().__init__()

@@ -727,13 +721,9 @@ class SiglipEncoder(nn.Module):

721 num_heads = config.num_attention_heads

722 head_dim = embed_dim // num_heads

723

730 - attn_backend_override = (

731 - multimodal_config.mm_encoder_attn_backend if multimodal_config else None

732 - )

724 self.attn_backend = get_vit_attn_backend(

725 head_size=head_dim,

726 dtype=torch.get_default_dtype(),

736 - attn_backend_override=attn_backend_override,

727 )

728 if self.attn_backend not in {

729 AttentionBackendEnum.FLASH_ATTN,

@@ -748,7 +738,6 @@ class SiglipEncoder(nn.Module):

738 SiglipEncoderLayer(

739 config,

740 quant_config=quant_config,

751 - multimodal_config=multimodal_config,

741 prefix=f"{prefix}.layers.{layer_idx}",

742 )

743 for layer_idx in range(config.num_hidden_layers)

@@ -830,7 +819,6 @@ class SiglipVisionTransformer(nn.Module):

819 self,

820 config: PretrainedConfig,

821 quant_config: QuantizationConfig | None = None,

833 - multimodal_config: MultiModalConfig | None = None,

822 prefix: str = "",

823 ):

824 super().__init__()

@@ -841,7 +829,6 @@ class SiglipVisionTransformer(nn.Module):

829 self.encoder = SiglipEncoder(

830 config,

831 quant_config=quant_config,

844 - multimodal_config=multimodal_config,

832 prefix=f"{prefix}.encoder",

833 )

834 self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)

@@ -880,7 +867,6 @@ class SiglipVisionModel(nn.Module):

867 self,

868 config,

869 quant_config: QuantizationConfig | None = None,

883 - multimodal_config: MultiModalConfig | None = None,

870 prefix: str = "",

871 ):

872 super().__init__()

@@ -888,7 +874,6 @@ class SiglipVisionModel(nn.Module):

874 self.vision_model = SiglipVisionTransformer(

875 config,

876 quant_config=quant_config,

891 - multimodal_config=multimodal_config,

877 prefix=f"{prefix}.vision_model",

878 )

879 self.quant_config = quant_config

@@ -1010,16 +995,13 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support

995 super().__init__()

996 config = vllm_config.model_config.hf_config

997 quant_config = vllm_config.quant_config

1013 - multimodal_config = vllm_config.model_config.multimodal_config

998

999 self.config = config

1016 - self.multimodal_config = multimodal_config

1000

1001 with self._mark_tower_model(vllm_config, "image"):

1002 self.visual = SiglipVisionModel(

1003 config=config.vision_config,

1004 quant_config=quant_config,

1022 - multimodal_config=multimodal_config,

1005 prefix=maybe_prefix(prefix, "visual"),

1006 )

1007 self.mlp_AR = Projector(config, config.vision_config)

vllm/model_executor/models/phi3v.py

+1 -6

@@ -29,7 +29,7 @@ from transformers import (

29 )

30

31 from vllm.config import VllmConfig

32 -from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

32 +from vllm.config.multimodal import BaseDummyOptions

33 from vllm.logger import init_logger

34 from vllm.model_executor.layers.quantization import QuantizationConfig

35 from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding

@@ -96,7 +96,6 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(

96 def _init_img_processor(

97 hf_config: PretrainedConfig,

98 quant_config: QuantizationConfig | None,

99 - multimodal_config: MultiModalConfig | None,

99 prefix: str = "",

100 ) -> CLIPVisionModel:

101 clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG

@@ -111,7 +110,6 @@ def _init_img_processor(

110 img_processor = CLIPVisionModel(

111 clip_config,

112 quant_config=quant_config,

114 - multimodal_config=multimodal_config,

113 num_hidden_layers_override=num_hidden_layers,

114 prefix=prefix,

115 )

@@ -170,7 +168,6 @@ class Phi3HDImageEmbedding(nn.Module):

168 self,

169 config: PretrainedConfig,

170 quant_config: QuantizationConfig | None,

173 - multimodal_config: MultiModalConfig | None,

171 prefix: str = "",

172 ) -> None:

173 super().__init__()

@@ -181,7 +178,6 @@ class Phi3HDImageEmbedding(nn.Module):

178 self.img_processor = _init_img_processor(

179 config,

180 quant_config=quant_config,

184 - multimodal_config=multimodal_config,

181 prefix=f"{prefix}.img_processor",

182 )

183

@@ -596,7 +592,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant)

592 self.vision_embed_tokens = Phi3HDImageEmbedding(

593 config,

594 quant_config=quant_config,

599 - multimodal_config=multimodal_config,

595 prefix=maybe_prefix(prefix, "model.vision_embed_tokens"),

596 )

597

vllm/model_executor/models/pixtral.py

+4 -20

@@ -28,7 +28,7 @@ from transformers.models.pixtral.modeling_pixtral import (

28 from transformers.tokenization_utils_base import TextInput

29

30 from vllm.config import VllmConfig

31 -from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

31 +from vllm.config.multimodal import BaseDummyOptions

32 from vllm.distributed import divide, get_tensor_model_parallel_world_size

33 from vllm.model_executor.layers.activation import get_act_and_mul_fn

34 from vllm.model_executor.layers.conv import Conv2dLayer

@@ -74,6 +74,7 @@ from .utils import init_vllm_registered_model, maybe_prefix

74 from .vision import (

75 VisionEncoderInfo,

76 VisionFeatureSelectStrategy,

77 + is_vit_use_data_parallel,

78 resolve_visual_encoder_outputs,

79 )

80

@@ -1065,17 +1066,12 @@ class PixtralHFMLP(nn.Module):

1066 self,

1067 config: PixtralVisionConfig,

1068 quant_config: QuantizationConfig | None = None,

1068 - multimodal_config: MultiModalConfig | None = None,

1069 *,

1070 prefix: str = "",

1071 ) -> None:

1072 super().__init__()

1073

1074 - use_data_parallel = (

1075 - multimodal_config.mm_encoder_tp_mode == "data"

1076 - if multimodal_config

1077 - else False

1078 - )

1074 + use_data_parallel = is_vit_use_data_parallel()

1075

1076 assert config.intermediate_size is not None

1077 self.gate_up_proj = MergedColumnParallelLinear(

@@ -1108,7 +1104,6 @@ class PixtralHFAttention(nn.Module):

1104 self,

1105 config: PixtralVisionConfig,

1106 quant_config: QuantizationConfig | None = None,

1111 - multimodal_config: MultiModalConfig | None = None,

1107 *,

1108 prefix: str = "",

1109 ) -> None:

@@ -1120,11 +1115,7 @@ class PixtralHFAttention(nn.Module):

1115 self.head_dim = config.hidden_size // config.num_attention_heads

1116 assert self.total_num_heads * self.head_dim == config.hidden_size

1117

1123 - use_data_parallel = (

1124 - multimodal_config.mm_encoder_tp_mode == "data"

1125 - if multimodal_config

1126 - else False

1127 - )

1118 + use_data_parallel = is_vit_use_data_parallel()

1119 self.qkv_proj = QKVParallelLinear(

1120 hidden_size=config.hidden_size,

1121 head_size=self.head_dim,

@@ -1189,7 +1180,6 @@ class PixtralHFTransformerBlock(nn.Module):

1180 self,

1181 config: PixtralVisionConfig,

1182 quant_config: QuantizationConfig | None = None,

1192 - multimodal_config: MultiModalConfig | None = None,

1183 *,

1184 prefix: str = "",

1185 ) -> None:

@@ -1199,13 +1189,11 @@ class PixtralHFTransformerBlock(nn.Module):

1189 self.attention = PixtralHFAttention(

1190 config,

1191 quant_config=quant_config,

1202 - multimodal_config=multimodal_config,

1192 prefix=f"{prefix}.attention",

1193 )

1194 self.feed_forward = PixtralHFMLP(

1195 config,

1196 quant_config=quant_config,

1208 - multimodal_config=multimodal_config,

1197 prefix=f"{prefix}.feed_forward",

1198 )

1199 self.ffn_norm = RMSNorm(config.hidden_size, eps=1e-5)

@@ -1232,7 +1220,6 @@ class PixtralHFTransformer(nn.Module):

1220 self,

1221 config: PixtralVisionConfig,

1222 quant_config: QuantizationConfig | None = None,

1235 - multimodal_config: MultiModalConfig | None = None,

1223 *,

1224 num_hidden_layers_override: int | None = None,

1225 prefix: str = "",

@@ -1249,7 +1236,6 @@ class PixtralHFTransformer(nn.Module):

1236 PixtralHFTransformerBlock(

1237 config=config,

1238 quant_config=quant_config,

1252 - multimodal_config=multimodal_config,

1239 prefix=f"{prefix}.layers.{layer_idx}",

1240 )

1241 for layer_idx in range(num_hidden_layers)

@@ -1281,7 +1267,6 @@ class PixtralHFVisionModel(nn.Module):

1267 self,

1268 config: PixtralVisionConfig,

1269 quant_config: QuantizationConfig | None = None,

1284 - multimodal_config: MultiModalConfig | None = None,

1270 *,

1271 num_hidden_layers_override: int | None = None,

1272 require_post_norm: bool | None = None,

@@ -1302,7 +1287,6 @@ class PixtralHFVisionModel(nn.Module):

1287 self.transformer = PixtralHFTransformer(

1288 config,

1289 quant_config=quant_config,

1305 - multimodal_config=multimodal_config,

1290 num_hidden_layers_override=num_hidden_layers_override,

1291 prefix=f"{prefix}.transformer",

1292 )

vllm/model_executor/models/qwen2_5_omni_thinker.py

-1

@@ -846,7 +846,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

846 norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),

847 quant_config=quant_config,

848 prefix=maybe_prefix(prefix, "visual"),

849 - multimodal_config=multimodal_config,

849 )

850

851 with self._mark_language_model(vllm_config):

vllm/model_executor/models/qwen2_5_vl.py

+5 -33

@@ -43,7 +43,7 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (

43 )

44

45 from vllm.compilation.decorators import support_torch_compile

46 -from vllm.config import MultiModalConfig, VllmConfig

46 +from vllm.config import VllmConfig

47 from vllm.distributed import parallel_state

48 from vllm.distributed import utils as dist_utils

49 from vllm.forward_context import set_forward_context

@@ -109,6 +109,7 @@ from .utils import (

109 )

110 from .vision import (

111 get_vit_attn_backend,

112 + is_vit_use_data_parallel,

113 run_dp_sharded_mrope_vision_model,

114 )

115

@@ -266,15 +267,10 @@ class Qwen2_5_VisionMLP(nn.Module):

267 bias: bool = False,

268 act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,

269 quant_config: QuantizationConfig | None = None,

269 - multimodal_config: MultiModalConfig | None = None,

270 prefix: str = "",

271 ):

272 super().__init__()

273 - use_data_parallel = (

274 - multimodal_config.mm_encoder_tp_mode == "data"

275 - if multimodal_config

276 - else False

277 - )

273 + use_data_parallel = is_vit_use_data_parallel()

274 self.gate_up_proj = MergedColumnParallelLinear(

275 input_size=in_features,

276 output_sizes=[hidden_features] * 2, # [gate_proj, up_proj]

@@ -308,16 +304,11 @@ class Qwen2_5_VisionAttention(nn.Module):

304 num_heads: int,

305 projection_size: int,

306 quant_config: QuantizationConfig | None = None,

311 - multimodal_config: MultiModalConfig | None = None,

307 prefix: str = "",

308 ) -> None:

309 super().__init__()

310 # Per attention head and per partition values.

316 - use_data_parallel = (

317 - multimodal_config.mm_encoder_tp_mode == "data"

318 - if multimodal_config

319 - else False

320 - )

311 + use_data_parallel = is_vit_use_data_parallel()

312 self.tp_size = (

313 1

314 if use_data_parallel

@@ -354,7 +345,6 @@ class Qwen2_5_VisionAttention(nn.Module):

345 num_heads=self.num_attention_heads_per_partition,

346 head_size=self.hidden_size_per_attention_head,

347 scale=self.hidden_size_per_attention_head**-0.5,

357 - multimodal_config=multimodal_config,

348 )

349

350 self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)

@@ -435,7 +425,6 @@ class Qwen2_5_VisionBlock(nn.Module):

425 act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,

426 norm_layer: Callable[[int], nn.Module] | None = None,

427 quant_config: QuantizationConfig | None = None,

438 - multimodal_config: MultiModalConfig | None = None,

428 prefix: str = "",

429 ) -> None:

430 super().__init__()

@@ -448,7 +437,6 @@ class Qwen2_5_VisionBlock(nn.Module):

437 num_heads=num_heads,

438 projection_size=dim,

439 quant_config=quant_config,

451 - multimodal_config=multimodal_config,

440 prefix=f"{prefix}.attn",

441 )

442 self.mlp = Qwen2_5_VisionMLP(

@@ -457,7 +445,6 @@ class Qwen2_5_VisionBlock(nn.Module):

445 act_fn=act_fn,

446 bias=True,

447 quant_config=quant_config,

460 - multimodal_config=multimodal_config,

448 prefix=f"{prefix}.mlp",

449 )

450

@@ -530,15 +517,10 @@ class Qwen2_5_VisionPatchMerger(nn.Module):

517 norm_layer: Callable[[int], nn.Module] | None = None,

518 spatial_merge_size: int = 2,

519 quant_config: QuantizationConfig | None = None,

533 - multimodal_config: MultiModalConfig | None = None,

520 prefix: str = "",

521 ) -> None:

522 super().__init__()

537 - use_data_parallel = (

538 - multimodal_config.mm_encoder_tp_mode == "data"

539 - if multimodal_config

540 - else False

541 - )

523 + use_data_parallel = is_vit_use_data_parallel()

524 self.hidden_size = context_dim * (spatial_merge_size**2)

525 if norm_layer is None:

526 norm_layer = partial(nn.LayerNorm, eps=1e-6)

@@ -579,7 +561,6 @@ class Qwen2_5_VisionTransformer(nn.Module):

561 vision_config: Qwen2_5_VLVisionConfig,

562 norm_eps: float = 1e-6,

563 quant_config: QuantizationConfig | None = None,

582 - multimodal_config: MultiModalConfig | None = None,

564 prefix: str = "",

565 ) -> None:

566 super().__init__()

@@ -620,15 +601,9 @@ class Qwen2_5_VisionTransformer(nn.Module):

601 rope_parameters={"partial_rotary_factor": 0.5},

602 )

603

623 - attn_backend_override = (

624 - multimodal_config.mm_encoder_attn_backend

625 - if multimodal_config is not None

626 - else None

627 - )

604 self.attn_backend = get_vit_attn_backend(

605 head_size=head_dim,

606 dtype=torch.get_default_dtype(),

631 - attn_backend_override=attn_backend_override,

607 )

608

609 if self.attn_backend not in {

@@ -650,7 +625,6 @@ class Qwen2_5_VisionTransformer(nn.Module):

625 act_fn=get_act_and_mul_fn(vision_config.hidden_act),

626 norm_layer=norm_layer,

627 quant_config=quant_config,

653 - multimodal_config=multimodal_config,

628 prefix=f"{prefix}.blocks.{layer_idx}",

629 )

630 for layer_idx in range(depth)

@@ -664,7 +638,6 @@ class Qwen2_5_VisionTransformer(nn.Module):

638 norm_layer=norm_layer,

639 spatial_merge_size=self.spatial_merge_size,

640 quant_config=quant_config,

667 - multimodal_config=multimodal_config,

641 prefix=f"{prefix}.merger",

642 )

643

@@ -1152,7 +1125,6 @@ class Qwen2_5_VLForConditionalGeneration(

1125 norm_eps=getattr(config, "rms_norm_eps", 1e-6),

1126 quant_config=self.quant_config,

1127 prefix=maybe_prefix(prefix, "visual"),

1155 - multimodal_config=multimodal_config,

1128 )

1129

1130 with self._mark_language_model(vllm_config):

vllm/model_executor/models/qwen2_vl.py

+6 -36

@@ -43,7 +43,7 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import (

43 from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize

44 from transformers.models.qwen2_vl.video_processing_qwen2_vl import Qwen2VLVideoProcessor

45

46 -from vllm.config import MultiModalConfig, VllmConfig

46 +from vllm.config import VllmConfig

47 from vllm.config.multimodal import BaseDummyOptions

48 from vllm.distributed import parallel_state, tensor_model_parallel_all_gather

49 from vllm.distributed import utils as dist_utils

@@ -106,6 +106,7 @@ from .utils import (

106 )

107 from .vision import (

108 get_vit_attn_backend,

109 + is_vit_use_data_parallel,

110 run_dp_sharded_mrope_vision_model,

111 )

112

@@ -247,15 +248,10 @@ class Qwen2VisionMLP(nn.Module):

248 hidden_features: int,

249 act_layer: type[nn.Module] = QuickGELU,

250 quant_config: QuantizationConfig | None = None,

250 - multimodal_config: MultiModalConfig | None = None,

251 prefix: str = "",

252 ):

253 super().__init__()

254 - use_data_parallel = (

255 - multimodal_config.mm_encoder_tp_mode == "data"

256 - if multimodal_config

257 - else False

258 - )

254 + use_data_parallel = is_vit_use_data_parallel()

255 self.fc1 = ColumnParallelLinear(

256 in_features,

257 hidden_features,

@@ -286,16 +282,11 @@ class Qwen2VisionAttention(nn.Module):

282 num_heads: int,

283 projection_size: int,

284 quant_config: QuantizationConfig | None = None,

289 - multimodal_config: MultiModalConfig | None = None,

285 prefix: str = "",

286 ) -> None:

287 super().__init__()

288 # Per attention head and per partition values.

294 - use_data_parallel = (

295 - multimodal_config.mm_encoder_tp_mode == "data"

296 - if multimodal_config

297 - else False

298 - )

289 + use_data_parallel = is_vit_use_data_parallel()

290 self.tp_size = (

291 1

292 if use_data_parallel

@@ -328,7 +319,6 @@ class Qwen2VisionAttention(nn.Module):

319 num_heads=self.num_attention_heads_per_partition,

320 head_size=self.hidden_size_per_attention_head,

321 scale=self.hidden_size_per_attention_head**-0.5,

331 - multimodal_config=multimodal_config,

322 )

323

324 self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)

@@ -409,7 +399,6 @@ class Qwen2VisionBlock(nn.Module):

399 act_layer: type[nn.Module] = QuickGELU,

400 norm_layer: Callable[[int], nn.Module] | None = None,

401 quant_config: QuantizationConfig | None = None,

412 - multimodal_config: MultiModalConfig | None = None,

402 prefix: str = "",

403 ) -> None:

404 super().__init__()

@@ -424,7 +413,6 @@ class Qwen2VisionBlock(nn.Module):

413 num_heads=num_heads,

414 projection_size=dim,

415 quant_config=quant_config,

427 - multimodal_config=multimodal_config,

416 prefix=f"{prefix}.attn",

417 )

418 self.mlp = Qwen2VisionMLP(

@@ -432,7 +420,6 @@ class Qwen2VisionBlock(nn.Module):

420 mlp_hidden_dim,

421 act_layer=act_layer,

422 quant_config=quant_config,

435 - multimodal_config=multimodal_config,

423 prefix=f"{prefix}.mlp",

424 )

425

@@ -493,15 +480,10 @@ class Qwen2VisionPatchMerger(nn.Module):

480 norm_layer: Callable[[int], nn.Module] | None = None,

481 spatial_merge_size: int = 2,

482 quant_config: QuantizationConfig | None = None,

496 - multimodal_config: MultiModalConfig | None = None,

483 prefix: str = "",

484 ) -> None:

485 super().__init__()

500 - use_data_parallel = (

501 - multimodal_config.mm_encoder_tp_mode == "data"

502 - if multimodal_config

503 - else False

504 - )

486 + use_data_parallel = is_vit_use_data_parallel()

487 self.hidden_size = context_dim * (spatial_merge_size**2)

488 if norm_layer is None:

489 norm_layer = partial(nn.LayerNorm, eps=1e-6)

@@ -545,7 +527,6 @@ class Qwen2VisionTransformer(nn.Module):

527 vision_config: Qwen2VLVisionConfig,

528 norm_eps: float = 1e-6,

529 quant_config: QuantizationConfig | None = None,

548 - multimodal_config: MultiModalConfig | None = None,

530 prefix: str = "",

531 ) -> None:

532 super().__init__()

@@ -560,11 +541,7 @@ class Qwen2VisionTransformer(nn.Module):

541 num_heads = vision_config.num_heads

542 mlp_ratio = vision_config.mlp_ratio

543

563 - self.use_data_parallel = (

564 - multimodal_config.mm_encoder_tp_mode == "data"

565 - if multimodal_config

566 - else False

567 - )

544 + self.use_data_parallel = is_vit_use_data_parallel()

545 self.out_hidden_size = vision_config.hidden_size

546

547 self.spatial_merge_size = spatial_merge_size

@@ -596,7 +573,6 @@ class Qwen2VisionTransformer(nn.Module):

573 norm_layer=norm_layer,

574 quant_config=quant_config,

575 prefix=f"{prefix}.blocks.{layer_idx}",

599 - multimodal_config=multimodal_config,

576 )

577 for layer_idx in range(depth)

578 ]

@@ -607,15 +583,10 @@ class Qwen2VisionTransformer(nn.Module):

583 norm_layer=norm_layer,

584 quant_config=quant_config,

585 prefix=f"{prefix}.merger",

610 - multimodal_config=multimodal_config,

611 - )

612 - attn_backend_override = (

613 - multimodal_config.mm_encoder_attn_backend if multimodal_config else None

586 )

587 self.attn_backend = get_vit_attn_backend(

588 head_size=head_dim,

589 dtype=torch.get_default_dtype(),

618 - attn_backend_override=attn_backend_override,

590 )

591

592 @property

@@ -1238,7 +1209,6 @@ class Qwen2VLForConditionalGeneration(

1209 config.vision_config,

1210 norm_eps=getattr(config, "rms_norm_eps", 1e-6),

1211 quant_config=quant_config,

1241 - multimodal_config=multimodal_config,

1212 prefix=maybe_prefix(prefix, "visual"),

1213 )

1214

vllm/model_executor/models/qwen3_omni_moe_thinker.py

+2 -26

@@ -48,7 +48,7 @@ from transformers import __version__ as TRANSFORMERS_VERSION

48 # isort: on

49

50 from vllm.compilation.decorators import support_torch_compile

51 -from vllm.config import MultiModalConfig, VllmConfig

51 +from vllm.config import VllmConfig

52 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size

53 from vllm.logger import init_logger

54 from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY

@@ -160,7 +160,6 @@ class Qwen3OmniMoeAudioAttention(nn.Module):

160 def __init__(

161 self,

162 config: Qwen3OmniMoeAudioEncoderConfig,

163 - multimodal_config: MultiModalConfig | None = None,

163 prefix: str = "",

164 ):

165 super().__init__()

@@ -198,7 +197,6 @@ class Qwen3OmniMoeAudioAttention(nn.Module):

197 num_heads=self.num_local_heads,

198 head_size=self.head_dim,

199 scale=self.scaling,

201 - multimodal_config=multimodal_config,

200 )

201

202 def forward(

@@ -233,13 +231,12 @@ class Qwen3OmniMoeAudioEncoderLayer(nn.Module):

231 def __init__(

232 self,

233 config: Qwen3OmniMoeAudioEncoderConfig,

236 - multimodal_config: MultiModalConfig | None = None,

234 prefix: str = "",

235 ):

236 super().__init__()

237 self.embed_dim = config.d_model

238 self.self_attn = Qwen3OmniMoeAudioAttention(

242 - config, multimodal_config=multimodal_config, prefix=f"{prefix}.self_attn"

239 + config, prefix=f"{prefix}.self_attn"

240 )

241 self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)

242 self.activation_fn = _ACTIVATION_REGISTRY[config.activation_function]

@@ -301,7 +298,6 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):

298 def __init__(

299 self,

300 config: Qwen3OmniMoeAudioEncoderConfig,

304 - multimodal_config: MultiModalConfig | None = None,

301 prefix: str = "",

302 ):

303 super().__init__()

@@ -345,7 +341,6 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):

341 [

342 Qwen3OmniMoeAudioEncoderLayer(

343 config,

348 - multimodal_config=multimodal_config,

344 prefix=f"{prefix}.layers.{i}",

345 )

346 for i in range(config.encoder_layers)

@@ -359,15 +354,9 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):

354 self.proj2 = nn.Linear(config.d_model, config.output_dim)

355

356 # Get attention backend

362 - attn_backend_override = (

363 - multimodal_config.mm_encoder_attn_backend

364 - if multimodal_config is not None

365 - else None

366 - )

357 self.attn_backend = get_vit_attn_backend(

358 head_size=config.d_model // config.encoder_attention_heads,

359 dtype=torch.get_default_dtype(),

370 - attn_backend_override=attn_backend_override,

360 )

361

362 def compute_attn_mask_seqlen(self, cu_seqlens: torch.Tensor) -> torch.Tensor | None:

@@ -601,7 +590,6 @@ class Qwen3_VisionBlock(nn.Module):

590 mlp_hidden_dim: int,

591 act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,

592 norm_layer: Callable[[int], nn.Module] | None = None,

604 - multimodal_config: MultiModalConfig | None = None,

593 quant_config: QuantizationConfig | None = None,

594 prefix: str = "",

595 ) -> None:

@@ -615,7 +603,6 @@ class Qwen3_VisionBlock(nn.Module):

603 num_heads=num_heads,

604 projection_size=dim,

605 quant_config=quant_config,

618 - multimodal_config=multimodal_config,

606 prefix=f"{prefix}.attn",

607 )

608 self.mlp = Qwen3_VisionMLP(

@@ -710,7 +697,6 @@ class Qwen3Omni_VisionTransformer(nn.Module):

697 vision_config,

698 norm_eps: float = 1e-6,

699 quant_config: QuantizationConfig | None = None,

713 - multimodal_config: MultiModalConfig | None = None,

700 prefix: str = "",

701 ) -> None:

702 super().__init__()

@@ -758,7 +744,6 @@ class Qwen3Omni_VisionTransformer(nn.Module):

744 act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],

745 norm_layer=norm_layer,

746 quant_config=quant_config,

761 - multimodal_config=multimodal_config,

747 prefix=f"{prefix}.blocks.{layer_idx}",

748 )

749 for layer_idx in range(vision_config.depth)

@@ -788,16 +773,9 @@ class Qwen3Omni_VisionTransformer(nn.Module):

773 ]

774 )

775

791 - attn_backend_override = (

792 - multimodal_config.mm_encoder_attn_backend

793 - if multimodal_config is not None

794 - else None

795 - )

796 -

776 self.attn_backend = get_vit_attn_backend(

777 head_size=head_dim,

778 dtype=torch.get_default_dtype(),

800 - attn_backend_override=attn_backend_override,

779 )

780

781 @property

@@ -1617,7 +1595,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1595 with self._mark_tower_model(vllm_config, "audio"):

1596 self.audio_tower = Qwen3OmniMoeAudioEncoder(

1597 thinker_config.audio_config,

1620 - multimodal_config=multimodal_config,

1598 prefix=maybe_prefix(prefix, "audio_tower"),

1599 )

1600

@@ -1638,7 +1615,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1615 norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),

1616 quant_config=quant_config,

1617 prefix=maybe_prefix(prefix, "visual"),

1641 - multimodal_config=multimodal_config,

1618 )

1619

1620 # register buffer for deepstack

vllm/model_executor/models/qwen3_vl.py

+4 -25

@@ -49,7 +49,7 @@ from transformers.models.qwen3_vl.video_processing_qwen3_vl import (

49 from transformers.video_utils import VideoMetadata

50

51 from vllm.compilation.decorators import support_torch_compile

52 -from vllm.config import MultiModalConfig, VllmConfig

52 +from vllm.config import VllmConfig

53 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

54 from vllm.distributed import get_pp_group

55 from vllm.logger import init_logger

@@ -123,6 +123,7 @@ from .utils import (

123 )

124 from .vision import (

125 get_vit_attn_backend,

126 + is_vit_use_data_parallel,

127 run_dp_sharded_mrope_vision_model,

128 )

129

@@ -169,15 +170,10 @@ class Qwen3_VisionMLP(nn.Module):

170 bias: bool = False,

171 act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,

172 quant_config: QuantizationConfig | None = None,

172 - multimodal_config: MultiModalConfig | None = None,

173 prefix: str = "",

174 ):

175 super().__init__()

176 - use_data_parallel = (

177 - multimodal_config.mm_encoder_tp_mode == "data"

178 - if multimodal_config

179 - else False

180 - )

176 + use_data_parallel = is_vit_use_data_parallel()

177 self.linear_fc1 = ColumnParallelLinear(

178 in_features,

179 hidden_features,

@@ -211,7 +207,6 @@ class Qwen3_VisionBlock(nn.Module):

207 mlp_hidden_dim: int,

208 act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,

209 norm_layer: Callable[[int], nn.Module] | None = None,

214 - multimodal_config: MultiModalConfig | None = None,

210 quant_config: QuantizationConfig | None = None,

211 prefix: str = "",

212 ) -> None:

@@ -225,7 +220,6 @@ class Qwen3_VisionBlock(nn.Module):

220 num_heads=num_heads,

221 projection_size=dim,

222 quant_config=quant_config,

228 - multimodal_config=multimodal_config,

223 prefix=f"{prefix}.attn",

224 )

225 self.mlp = Qwen3_VisionMLP(

@@ -234,7 +228,6 @@ class Qwen3_VisionBlock(nn.Module):

228 act_fn=act_fn,

229 bias=True,

230 quant_config=quant_config,

237 - multimodal_config=multimodal_config,

231 prefix=f"{prefix}.mlp",

232 )

233

@@ -267,15 +260,10 @@ class Qwen3_VisionPatchMerger(nn.Module):

260 spatial_merge_size: int = 2,

261 use_postshuffle_norm: bool = False,

262 quant_config: QuantizationConfig | None = None,

270 - multimodal_config: MultiModalConfig | None = None,

263 prefix: str = "",

264 ) -> None:

265 super().__init__()

274 - use_data_parallel = (

275 - multimodal_config.mm_encoder_tp_mode == "data"

276 - if multimodal_config

277 - else False

278 - )

266 + use_data_parallel = is_vit_use_data_parallel()

267 self.hidden_size = context_dim * (spatial_merge_size**2)

268

269 self.use_postshuffle_norm = use_postshuffle_norm

@@ -321,7 +309,6 @@ class Qwen3_VisionTransformer(nn.Module):

309 vision_config: Qwen3VLVisionConfig,

310 norm_eps: float = 1e-6,

311 quant_config: QuantizationConfig | None = None,

324 - multimodal_config: MultiModalConfig | None = None,

312 prefix: str = "",

313 ) -> None:

314 super().__init__()

@@ -365,7 +352,6 @@ class Qwen3_VisionTransformer(nn.Module):

352 norm_layer=norm_layer,

353 spatial_merge_size=self.spatial_merge_size,

354 quant_config=quant_config,

368 - multimodal_config=multimodal_config,

355 prefix=f"{prefix}.merger",

356 )

357

@@ -378,20 +364,15 @@ class Qwen3_VisionTransformer(nn.Module):

364 use_postshuffle_norm=True,

365 norm_layer=norm_layer,

366 quant_config=quant_config,

381 - multimodal_config=multimodal_config,

367 prefix=f"{prefix}.deepstack_merger_list.{layer_idx}",

368 )

369 for layer_idx in range(len(self.deepstack_visual_indexes))

370 ]

371 )

372

388 - attn_backend_override = (

389 - multimodal_config.mm_encoder_attn_backend if multimodal_config else None

390 - )

373 self.attn_backend = get_vit_attn_backend(

374 head_size=head_dim,

375 dtype=torch.get_default_dtype(),

394 - attn_backend_override=attn_backend_override,

376 )

377

378 if self.attn_backend not in {

@@ -411,7 +392,6 @@ class Qwen3_VisionTransformer(nn.Module):

392 act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],

393 norm_layer=norm_layer,

394 quant_config=quant_config,

414 - multimodal_config=multimodal_config,

395 prefix=f"{prefix}.blocks.{layer_idx}",

396 )

397 for layer_idx in range(vision_config.depth)

@@ -1291,7 +1271,6 @@ class Qwen3VLForConditionalGeneration(

1271 config.vision_config,

1272 norm_eps=getattr(config, "rms_norm_eps", 1e-6),

1273 quant_config=quant_config,

1294 - multimodal_config=multimodal_config,

1274 prefix=maybe_prefix(prefix, "visual"),

1275 )

1276

vllm/model_executor/models/qwen3_vl_moe.py

-1

@@ -446,7 +446,6 @@ class Qwen3VLMoeForConditionalGeneration(

446 config.vision_config,

447 norm_eps=getattr(config, "rms_norm_eps", 1e-6),

448 quant_config=quant_config,

449 - multimodal_config=multimodal_config,

449 prefix=maybe_prefix(prefix, "visual"),

450 )

451

vllm/model_executor/models/siglip.py

+4 -29

@@ -16,7 +16,7 @@ from transformers import (

16 )

17

18 from vllm.config import VllmConfig

19 -from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

19 +from vllm.config.multimodal import BaseDummyOptions

20 from vllm.distributed import divide, get_tensor_model_parallel_world_size

21 from vllm.model_executor.layers.activation import get_act_fn

22 from vllm.model_executor.layers.attention.encoder_only_attention import (

@@ -64,6 +64,7 @@ from .vision import (

64 VisionFeatureSelectStrategy,

65 VisionFeatureSelectStrategyStr,

66 get_num_selected_vision_tokens,

67 + is_vit_use_data_parallel,

68 resolve_visual_encoder_outputs,

69 )

70

@@ -356,7 +357,6 @@ class SiglipAttention(nn.Module):

357 self,

358 config: SiglipVisionConfig | SiglipTextConfig,

359 quant_config: QuantizationConfig | None = None,

359 - multimodal_config: MultiModalConfig | None = None,

360 *,

361 prefix: str = "",

362 attn_cls: type[EncoderOnlyAttention] | type[MMEncoderAttention],

@@ -376,11 +376,7 @@ class SiglipAttention(nn.Module):

376

377 self.scale = self.head_dim**-0.5

378

379 - use_data_parallel = (

380 - multimodal_config.mm_encoder_tp_mode == "data"

381 - if multimodal_config

382 - else False

383 - )

379 + use_data_parallel = is_vit_use_data_parallel()

380 self.qkv_proj = QKVParallelLinear(

381 hidden_size=self.embed_dim,

382 head_size=self.head_dim,

@@ -409,7 +405,6 @@ class SiglipAttention(nn.Module):

405 self.head_dim,

406 self.scale,

407 prefix=f"{prefix}.attn",

412 - multimodal_config=multimodal_config,

408 )

409 else:

410 self.attn = attn_cls(

@@ -437,17 +432,12 @@ class SiglipMLP(nn.Module):

432 self,

433 config: SiglipVisionConfig | SiglipTextConfig,

434 quant_config: QuantizationConfig | None = None,

440 - multimodal_config: MultiModalConfig | None = None,

435 prefix: str = "",

436 ) -> None:

437 super().__init__()

438

439 self.config = config

446 - use_data_parallel = (

447 - multimodal_config.mm_encoder_tp_mode == "data"

448 - if multimodal_config

449 - else False

450 - )

440 + use_data_parallel = is_vit_use_data_parallel()

441 self.activation_fn = get_act_fn(config.hidden_act)

442

443 # Special handling for BNB and torchao quantization

@@ -487,7 +477,6 @@ class SiglipEncoderLayer(nn.Module):

477 self,

478 config: SiglipVisionConfig | SiglipTextConfig,

479 quant_config: QuantizationConfig | None = None,

490 - multimodal_config: MultiModalConfig | None = None,

480 *,

481 prefix: str = "",

482 attn_cls: type[EncoderOnlyAttention] | type[MMEncoderAttention],

@@ -499,7 +488,6 @@ class SiglipEncoderLayer(nn.Module):

488 self.self_attn = SiglipAttention(

489 config,

490 quant_config=quant_config,

502 - multimodal_config=multimodal_config,

491 prefix=f"{prefix}.self_attn",

492 attn_cls=attn_cls,

493 )

@@ -507,7 +495,6 @@ class SiglipEncoderLayer(nn.Module):

495 self.mlp = SiglipMLP(

496 config,

497 quant_config=quant_config,

510 - multimodal_config=multimodal_config,

498 prefix=f"{prefix}.mlp",

499 )

500 self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)

@@ -535,7 +522,6 @@ class SiglipEncoder(nn.Module):

522 self,

523 config: SiglipVisionConfig | SiglipTextConfig,

524 quant_config: QuantizationConfig | None = None,

538 - multimodal_config: MultiModalConfig | None = None,

525 num_hidden_layers_override: int | None = None,

526 *,

527 prefix: str = "",

@@ -555,7 +541,6 @@ class SiglipEncoder(nn.Module):

541 SiglipEncoderLayer(

542 config,

543 quant_config=quant_config,

558 - multimodal_config=multimodal_config,

544 prefix=f"{prefix}.layers.{layer_idx}",

545 attn_cls=attn_cls,

546 )

@@ -660,7 +645,6 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):

645 self,

646 config: SiglipVisionConfig,

647 quant_config: QuantizationConfig | None = None,

663 - multimodal_config: MultiModalConfig | None = None,

648 prefix: str = "",

649 ) -> None:

650 super().__init__()

@@ -674,7 +658,6 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):

658 self.mlp = SiglipMLP(

659 config=config,

660 quant_config=quant_config,

677 - multimodal_config=multimodal_config,

661 prefix=f"{prefix}.mlp",

662 )

663

@@ -700,7 +683,6 @@ class SiglipVisionTransformer(nn.Module):

683 self,

684 config: SiglipVisionConfig,

685 quant_config: QuantizationConfig | None = None,

703 - multimodal_config: MultiModalConfig | None = None,

686 *,

687 num_hidden_layers_override: int | None = None,

688 require_post_norm: bool | None = None,

@@ -717,7 +699,6 @@ class SiglipVisionTransformer(nn.Module):

699 self.encoder = SiglipEncoder(

700 config,

701 quant_config=quant_config,

720 - multimodal_config=multimodal_config,

702 num_hidden_layers_override=num_hidden_layers_override,

703 prefix=f"{prefix}.encoder",

704 attn_cls=MMEncoderAttention,

@@ -756,7 +737,6 @@ class SiglipVisionTransformer(nn.Module):

737 SiglipMultiheadAttentionPoolingHead(

738 config=config,

739 quant_config=quant_config,

759 - multimodal_config=multimodal_config,

740 prefix=f"{prefix}.head",

741 )

742 if self.use_head

@@ -870,7 +850,6 @@ class SiglipVisionModel(nn.Module):

850 self,

851 config: SiglipVisionConfig,

852 quant_config: QuantizationConfig | None = None,

873 - multimodal_config: MultiModalConfig | None = None,

853 *,

854 num_hidden_layers_override: int | None = None,

855 require_post_norm: bool | None = None,

@@ -883,7 +862,6 @@ class SiglipVisionModel(nn.Module):

862 self.vision_model = SiglipVisionTransformer(

863 config,

864 quant_config=quant_config,

886 - multimodal_config=multimodal_config,

865 num_hidden_layers_override=num_hidden_layers_override,

866 require_post_norm=require_post_norm,

867 prefix=f"{prefix}.vision_model",

@@ -1062,9 +1040,7 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1040

1041 config: SiglipConfig = vllm_config.model_config.hf_config

1042 quant_config = vllm_config.quant_config

1065 - multimodal_config = vllm_config.model_config.multimodal_config

1043 self.config = config

1067 - self.multimodal_config = multimodal_config

1044

1045 if hasattr(config, "num_labels"):

1046 config.num_labels = 0

@@ -1087,7 +1063,6 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1063 self.vision_model = SiglipVisionTransformer(

1064 vision_config,

1065 quant_config=quant_config,

1090 - multimodal_config=multimodal_config,

1066 prefix=maybe_prefix(prefix, "vision_model"),

1067 use_head=None, # Allows potential pooling head

1068 )

vllm/model_executor/models/siglip2navit.py

+4 -24

@@ -11,7 +11,6 @@ from torch.nn import functional as F

11 from transformers import Siglip2VisionConfig

12 from transformers.configuration_utils import PretrainedConfig

13

14 -from vllm.config import MultiModalConfig

14 from vllm.distributed import divide, get_tensor_model_parallel_world_size

15 from vllm.model_executor.layers.activation import get_act_fn

16 from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention

@@ -30,6 +29,8 @@ from vllm.model_executor.layers.rotary_embedding.common import (

29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

30 from vllm.platforms import current_platform

31

32 +from .vision import is_vit_use_data_parallel

33 +

34

35 class VisionRotaryEmbedding(nn.Module):

36 def __init__(self, dim: int, theta: float = 10000.0) -> None:

@@ -178,9 +179,7 @@ class Siglip2Attention(nn.Module):

179 self,

180 config: Siglip2VisionConfig,

181 quant_config: QuantizationConfig | None = None,

181 - multimodal_config: MultiModalConfig | None = None,

182 prefix: str = "",

183 - use_data_parallel: bool = False,

183 ):

184 super().__init__()

185 self.config = config

@@ -196,11 +195,7 @@ class Siglip2Attention(nn.Module):

195 self.scale = self.head_dim**-0.5

196 self.dropout = config.attention_dropout

197

199 - use_data_parallel = (

200 - multimodal_config.mm_encoder_tp_mode == "data"

201 - if multimodal_config

202 - else False

203 - )

198 + use_data_parallel = is_vit_use_data_parallel()

199 self.qkv_proj = QKVParallelLinear(

200 hidden_size=self.embed_dim,

201 head_size=self.head_dim,

@@ -228,7 +223,6 @@ class Siglip2Attention(nn.Module):

223 head_size=self.head_dim,

224 scale=self.scale,

225 prefix=f"{prefix}.attn",

231 - multimodal_config=multimodal_config,

226 )

227

228 self.apply_rotary_emb = ApplyRotaryEmb(

@@ -287,16 +281,11 @@ class Siglip2MLP(nn.Module):

281 self,

282 config: Siglip2VisionConfig,

283 quant_config: QuantizationConfig | None = None,

290 - multimodal_config: MultiModalConfig | None = None,

284 prefix: str = "",

285 ):

286 super().__init__()

287 self.config = config

295 - use_data_parallel = (

296 - multimodal_config.mm_encoder_tp_mode == "data"

297 - if multimodal_config

298 - else False

299 - )

288 + use_data_parallel = is_vit_use_data_parallel()

289 self.activation_fn = get_act_fn(config.hidden_act)

290 self.fc1 = ColumnParallelLinear(

291 config.hidden_size,

@@ -325,7 +314,6 @@ class Siglip2EncoderLayer(nn.Module):

314 self,

315 config: Siglip2VisionConfig,

316 quant_config: QuantizationConfig | None = None,

328 - multimodal_config: MultiModalConfig | None = None,

317 prefix: str = "",

318 ):

319 super().__init__()

@@ -334,14 +322,12 @@ class Siglip2EncoderLayer(nn.Module):

322 self.self_attn = Siglip2Attention(

323 config,

324 quant_config=quant_config,

337 - multimodal_config=multimodal_config,

325 prefix=f"{prefix}.self_attn",

326 )

327 self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)

328 self.mlp = Siglip2MLP(

329 config,

330 quant_config=quant_config,

344 - multimodal_config=multimodal_config,

331 prefix=f"{prefix}.mlp",

332 )

333

@@ -387,7 +373,6 @@ class Siglip2Encoder(nn.Module):

373 self,

374 config: Siglip2VisionConfig,

375 quant_config: QuantizationConfig | None = None,

390 - multimodal_config: MultiModalConfig | None = None,

376 prefix: str = "",

377 ):

378 super().__init__()

@@ -397,7 +382,6 @@ class Siglip2Encoder(nn.Module):

382 Siglip2EncoderLayer(

383 config,

384 quant_config=quant_config,

400 - multimodal_config=multimodal_config,

385 prefix=f"{prefix}.layers.{idx}",

386 )

387 for idx in range(config.num_hidden_layers)

@@ -571,7 +555,6 @@ class Siglip2VisionTransformer(nn.Module):

555 self,

556 config: Siglip2VisionConfig,

557 quant_config: QuantizationConfig | None = None,

574 - multimodal_config: MultiModalConfig | None = None,

558 prefix: str = "",

559 ):

560 super().__init__()

@@ -582,7 +565,6 @@ class Siglip2VisionTransformer(nn.Module):

565 self.encoder = Siglip2Encoder(

566 config,

567 quant_config=quant_config,

585 - multimodal_config=multimodal_config,

568 prefix=f"{prefix}.encoder",

569 )

570 self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)

@@ -610,7 +592,6 @@ class Siglip2NavitModel(torch.nn.Module):

592 self,

593 config: Siglip2VisionConfig,

594 quant_config: QuantizationConfig | None = None,

613 - multimodal_config: MultiModalConfig | None = None,

595 prefix: str = "",

596 ):

597 super().__init__()

@@ -618,7 +599,6 @@ class Siglip2NavitModel(torch.nn.Module):

599 self.vision_model = Siglip2VisionTransformer(

600 config,

601 quant_config=quant_config,

621 - multimodal_config=multimodal_config,

602 prefix=f"{prefix}.vision_model",

603 )

604

vllm/model_executor/models/tarsier.py

+1 -6

@@ -19,7 +19,7 @@ from transformers.models.llava import LlavaProcessor

19 from transformers.processing_utils import ProcessingKwargs, Unpack

20 from transformers.tokenization_utils_base import PreTokenizedInput, TextInput

21

22 -from vllm.config import MultiModalConfig, VllmConfig

22 +from vllm.config import VllmConfig

23 from vllm.model_executor.layers.activation import get_act_fn

24 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

25 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -351,7 +351,6 @@ def _build_tarsier_hf_processor(

351 def init_vision_tower_for_tarsier(

352 hf_config: TarsierHfConfig, # Use the Tarsier specific config protocol

353 quant_config: QuantizationConfig | None,

354 - multimodal_config: MultiModalConfig | None,

354 *,

355 require_post_norm: bool | None = None,

356 prefix: str = "",

@@ -378,7 +377,6 @@ def init_vision_tower_for_tarsier(

377 return CLIPVisionModel(

378 vision_config,

379 quant_config=quant_config,

381 - multimodal_config=multimodal_config,

380 num_hidden_layers_override=num_hidden_layers_to_init,

381 require_post_norm=require_post_norm,

382 prefix=prefix,

@@ -387,7 +385,6 @@ def init_vision_tower_for_tarsier(

385 return SiglipVisionModel(

386 vision_config,

387 quant_config=quant_config,

390 - multimodal_config=multimodal_config,

388 num_hidden_layers_override=num_hidden_layers_to_init,

389 require_post_norm=require_post_norm,

390 prefix=prefix,

@@ -420,7 +417,6 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

417

418 config: TarsierHfConfig = vllm_config.model_config.hf_config

419 quant_config = vllm_config.quant_config

423 - multimodal_config = vllm_config.model_config.multimodal_config

420

421 self.config = config # Storing the Tarsier-specific HF config

422

@@ -428,7 +424,6 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

424 self.vision_tower = init_vision_tower_for_tarsier(

425 config,

426 quant_config=quant_config,

431 - multimodal_config=multimodal_config,

427 require_post_norm=False,

428 prefix=maybe_prefix(prefix, "vision_tower"),

429 )

vllm/model_executor/models/vision.py

+48 -2

@@ -10,7 +10,7 @@ from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar

10 import torch

11 from transformers import PretrainedConfig

12

13 -from vllm.config import VllmConfig

13 +from vllm.config import MultiModalConfig, VllmConfig, get_current_vllm_config

14 from vllm.distributed import (

15 get_tensor_model_parallel_rank,

16 get_tensor_model_parallel_world_size,

@@ -79,7 +79,7 @@ def get_vision_encoder_info(hf_config: VisionLanguageConfig) -> VisionEncoderInf

79 raise NotImplementedError(msg)

80

81

82 -def get_vit_attn_backend(

82 +def _get_vit_attn_backend(

83 head_size: int,

84 dtype: torch.dtype,

85 *,

@@ -95,6 +95,52 @@ def get_vit_attn_backend(

95 )

96

97

98 +def get_vit_attn_backend(

99 + head_size: int,

100 + dtype: torch.dtype,

101 +) -> AttentionBackendEnum:

102 + """

103 + Get the attention backend for Vision Transformer.

104 + """

105 + try:

106 + vllm_config: VllmConfig = get_current_vllm_config()

107 + multimodal_config: MultiModalConfig | None = (

108 + vllm_config.model_config.multimodal_config

109 + )

110 + except AssertionError:

111 + multimodal_config = None

112 +

113 + attn_backend_override = (

114 + multimodal_config.mm_encoder_attn_backend

115 + if multimodal_config is not None

116 + else None

117 + )

118 + attn_backend = _get_vit_attn_backend(

119 + head_size,

120 + dtype,

121 + attn_backend_override=attn_backend_override,

122 + )

123 + return attn_backend

124 +

125 +

126 +def is_vit_use_data_parallel():

127 + """

128 + Get the tensor parallel type for Vision Transformer.

129 + """

130 + try:

131 + vllm_config: VllmConfig = get_current_vllm_config()

132 + multimodal_config: MultiModalConfig | None = (

133 + vllm_config.model_config.multimodal_config

134 + )

135 + except AssertionError:

136 + multimodal_config = None

137 +

138 + mm_encoder_tp_mode = (

139 + multimodal_config.mm_encoder_tp_mode if multimodal_config is not None else None

140 + )

141 + return mm_encoder_tp_mode == "data"

142 +

143 +

144 def should_torch_compile_mm_vit(vllm_config: VllmConfig) -> bool:

145 """Callable to be passed to `@support_torch_compile`'s `enable_if` argument."""

146 return vllm_config.compilation_config.compile_mm_encoder

[Models]: Make Multimodal config implicit in ViT implementation (#31972)