@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5576227bc1

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 12:45 UTC

[Model] Standardize common vision encoders (#31947)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 8, 2026 at 18:33 UTC 5576227bc132996d9e4ca279a8a8cf9e7b8a8ca4

19 files changed +254 -174

vllm/model_executor/models/clip.py

+57 -12

@@ -17,7 +17,7 @@ from transformers import (

17 from vllm.attention.layer import Attention

18 from vllm.attention.layers.mm_encoder_attention import MMEncoderAttention

19 from vllm.config import VllmConfig

20 -from vllm.config.multimodal import BaseDummyOptions

20 +from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

21 from vllm.distributed import divide, get_tensor_model_parallel_world_size

22 from vllm.model_executor.layers.activation import get_act_fn

23 from vllm.model_executor.layers.conv import Conv2dLayer

@@ -353,6 +353,7 @@ class CLIPAttention(nn.Module):

353 self,

354 config: CLIPTextConfig | CLIPVisionConfig,

355 quant_config: QuantizationConfig | None = None,

356 + multimodal_config: MultiModalConfig | None = None,

357 *,

358 prefix: str = "",

359 attn_cls: type[Attention] | type[MMEncoderAttention],

@@ -365,18 +366,24 @@ class CLIPAttention(nn.Module):

366 self.head_dim = self.embed_dim // self.num_heads

367 if self.head_dim * self.num_heads != self.embed_dim:

368 raise ValueError(

368 - "embed_dim must be divisible by num_heads "

369 - f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"

370 - f" {self.num_heads})."

369 + f"embed_dim must be divisible by num_heads "

370 + f"(got `embed_dim`: {self.embed_dim} and "

371 + f"`num_heads`: {self.num_heads})."

372 )

373 self.scale = self.head_dim**-0.5

374

375 + use_data_parallel = (

376 + multimodal_config.mm_encoder_tp_mode == "data"

377 + if multimodal_config

378 + else False

379 + )

380 self.qkv_proj = QKVParallelLinear(

381 hidden_size=self.embed_dim,

382 head_size=self.head_dim,

383 total_num_heads=self.num_heads,

384 quant_config=quant_config,

385 prefix=f"{prefix}.qkv_proj",

386 + disable_tp=use_data_parallel,

387 )

388

389 self.out_proj = RowParallelLinear(

@@ -384,17 +391,29 @@ class CLIPAttention(nn.Module):

391 output_size=self.embed_dim,

392 quant_config=quant_config,

393 prefix=f"{prefix}.out_proj",

394 + disable_tp=use_data_parallel,

395 )

396

389 - self.tp_size = get_tensor_model_parallel_world_size()

397 + self.tp_size = (

398 + 1 if use_data_parallel else get_tensor_model_parallel_world_size()

399 + )

400 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)

401

392 - self.attn = attn_cls(

393 - self.num_heads_per_partition,

394 - self.head_dim,

395 - self.scale,

396 - prefix=f"{prefix}.attn",

397 - )

402 + if attn_cls == MMEncoderAttention:

403 + self.attn = attn_cls(

404 + self.num_heads_per_partition,

405 + self.head_dim,

406 + self.scale,

407 + prefix=f"{prefix}.attn",

408 + multimodal_config=multimodal_config,

409 + )

410 + else:

411 + self.attn = attn_cls(

412 + self.num_heads_per_partition,

413 + self.head_dim,

414 + self.scale,

415 + prefix=f"{prefix}.attn",

416 + )

417

418 def forward(

419 self,

@@ -415,17 +434,26 @@ class CLIPMLP(nn.Module):

434 self,

435 config: CLIPTextConfig | CLIPVisionConfig,

436 quant_config: QuantizationConfig | None = None,

437 + multimodal_config: MultiModalConfig | None = None,

438 prefix: str = "",

439 ) -> None:

440 super().__init__()

441 +

442 self.config = config

443 + use_data_parallel = (

444 + multimodal_config.mm_encoder_tp_mode == "data"

445 + if multimodal_config

446 + else False

447 + )

448 self.activation_fn = get_act_fn(config.hidden_act)

449 +

450 self.fc1 = ColumnParallelLinear(

451 config.hidden_size,

452 config.intermediate_size,

453 bias=True,

454 quant_config=quant_config,

455 prefix=f"{prefix}.fc1",

456 + disable_tp=use_data_parallel,

457 )

458 self.fc2 = RowParallelLinear(

459 config.intermediate_size,

@@ -433,6 +461,7 @@ class CLIPMLP(nn.Module):

461 bias=True,

462 quant_config=quant_config,

463 prefix=f"{prefix}.fc2",

464 + disable_tp=use_data_parallel,

465 )

466

467 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:

@@ -448,19 +477,27 @@ class CLIPEncoderLayer(nn.Module):

477 self,

478 config: CLIPTextConfig | CLIPVisionConfig,

479 quant_config: QuantizationConfig | None = None,

480 + multimodal_config: MultiModalConfig | None = None,

481 *,

482 prefix: str = "",

483 attn_cls: type[Attention] | type[MMEncoderAttention],

484 ) -> None:

485 super().__init__()

486 +

487 self.self_attn = CLIPAttention(

488 config,

489 quant_config=quant_config,

490 + multimodal_config=multimodal_config,

491 prefix=f"{prefix}.self_attn",

492 attn_cls=attn_cls,

493 )

494 self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)

463 - self.mlp = CLIPMLP(config, quant_config=quant_config, prefix=f"{prefix}.mlp")

495 + self.mlp = CLIPMLP(

496 + config,

497 + quant_config=quant_config,

498 + multimodal_config=multimodal_config,

499 + prefix=f"{prefix}.mlp",

500 + )

501 self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)

502

503 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:

@@ -491,6 +528,7 @@ class CLIPEncoder(nn.Module):

528 self,

529 config: CLIPTextConfig | CLIPVisionConfig,

530 quant_config: QuantizationConfig | None = None,

531 + multimodal_config: MultiModalConfig | None = None,

532 num_hidden_layers_override: int | None = None,

533 *,

534 prefix: str = "",

@@ -504,11 +542,13 @@ class CLIPEncoder(nn.Module):

542 num_hidden_layers = config.num_hidden_layers

543 else:

544 num_hidden_layers = num_hidden_layers_override

545 +

546 self.layers = nn.ModuleList(

547 [

548 CLIPEncoderLayer(

549 config=config,

550 quant_config=quant_config,

551 + multimodal_config=multimodal_config,

552 prefix=f"{prefix}.layers.{layer_idx}",

553 attn_cls=attn_cls,

554 )

@@ -618,6 +658,7 @@ class CLIPVisionTransformer(nn.Module):

658 self,

659 config: CLIPVisionConfig,

660 quant_config: QuantizationConfig | None = None,

661 + multimodal_config: MultiModalConfig | None = None,

662 *,

663 num_hidden_layers_override: int | None = None,

664 require_post_norm: bool | None = None,

@@ -637,6 +678,7 @@ class CLIPVisionTransformer(nn.Module):

678 self.encoder = CLIPEncoder(

679 config=config,

680 quant_config=quant_config,

681 + multimodal_config=multimodal_config,

682 num_hidden_layers_override=num_hidden_layers_override,

683 prefix=f"{prefix}.encoder",

684 attn_cls=MMEncoderAttention,

@@ -738,6 +780,7 @@ class CLIPVisionModel(nn.Module):

780 self,

781 config: CLIPVisionConfig,

782 quant_config: QuantizationConfig | None = None,

783 + multimodal_config: MultiModalConfig | None = None,

784 *,

785 num_hidden_layers_override: int | None = None,

786 require_post_norm: bool | None = None,

@@ -748,6 +791,7 @@ class CLIPVisionModel(nn.Module):

791 self.vision_model = CLIPVisionTransformer(

792 config=config,

793 quant_config=quant_config,

794 + multimodal_config=multimodal_config,

795 num_hidden_layers_override=num_hidden_layers_override,

796 require_post_norm=require_post_norm,

797 prefix=f"{prefix}.vision_model",

@@ -817,6 +861,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

861 self.vision_model = CLIPVisionTransformer(

862 vision_config,

863 quant_config=quant_config,

864 + multimodal_config=multimodal_config,

865 prefix=maybe_prefix(prefix, "vision_model"),

866 )

867

vllm/model_executor/models/deepencoder.py

@@ -19,6 +19,7 @@ import torch.nn.functional as F

19 from transformers import CLIPVisionConfig

21 from vllm.attention.layers.mm_encoder_attention import MMEncoderAttention

22 +from vllm.config import MultiModalConfig

23 from vllm.model_executor.layers.conv import Conv2dLayer

24 from vllm.model_executor.layers.quantization import QuantizationConfig

25 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -608,6 +609,7 @@ class DeepCLIPVisionTransformer(nn.Module):

609 self,

610 config: CLIPVisionConfig,

611 quant_config: QuantizationConfig | None = None,

612 + multimodal_config: MultiModalConfig | None = None,

613 *,

614 num_hidden_layers_override: int | None = None,

615 prefix: str = "",

@@ -626,6 +628,7 @@ class DeepCLIPVisionTransformer(nn.Module):

628 self.transformer = CLIPEncoder(

629 config=config,

630 quant_config=quant_config,

631 + multimodal_config=multimodal_config,

632 num_hidden_layers_override=num_hidden_layers_override,

633 prefix=f"{prefix}.encoder",

634 attn_cls=MMEncoderAttention,

vllm/model_executor/models/deepseek_ocr.py

@@ -397,6 +397,7 @@ class DeepseekOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports

397 self.vision_model = DeepCLIPVisionTransformer(

398 config=clip_vision_config,

399 quant_config=quant_config,

400 + multimodal_config=multimodal_config,

401 prefix=maybe_prefix(prefix, "vision_model"),

402 )

403

vllm/model_executor/models/hyperclovax_vision.py

+9 -10

@@ -6,7 +6,7 @@ from collections import defaultdict

6 from collections.abc import Iterable, Mapping, Sequence

7 from functools import partial

8 from itertools import accumulate

9 -from typing import Annotated, Any, Literal

9 +from typing import Annotated, Literal

11 import numpy as np

12 import torch

@@ -18,7 +18,7 @@ from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig

18 from transformers.modeling_utils import no_init_weights

20 from vllm.config import VllmConfig

21 -from vllm.config.multimodal import BaseDummyOptions

21 +from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

22 from vllm.model_executor.layers.quantization import QuantizationConfig

23 from vllm.multimodal import MULTIMODAL_REGISTRY

24 from vllm.multimodal.cache import BaseMultiModalProcessorCache

@@ -361,6 +361,7 @@ def _build_hcxvision_hf_processor(

361 def init_vision_tower_for_hcxvision(

362 vision_config,

363 quant_config: QuantizationConfig | None,

364 + multimodal_config: MultiModalConfig | None,

365 *,

366 use_nth_layer: int | None = None,

367 require_post_norm: bool | None = None,

@@ -378,6 +379,7 @@ def init_vision_tower_for_hcxvision(

379 return CLIPVisionModel(

380 vision_config,

381 quant_config=quant_config,

382 + multimodal_config=multimodal_config,

383 num_hidden_layers_override=num_hidden_layers,

384 require_post_norm=require_post_norm,

385 prefix=prefix,

@@ -386,6 +388,7 @@ def init_vision_tower_for_hcxvision(

388 return SiglipVisionModel(

389 vision_config,

390 quant_config=quant_config,

391 + multimodal_config=multimodal_config,

392 num_hidden_layers_override=num_hidden_layers,

393 require_post_norm=require_post_norm,

394 prefix=prefix,

@@ -597,18 +600,13 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

600 "gate_up_proj": ["gate_proj", "up_proj"],

601 }

602

600 - def __init__(

601 - self,

602 - *,

603 - vllm_config: VllmConfig,

604 - prefix: str = "",

605 - **kwargs: Any | None,

606 - ) -> None:

603 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

604 super().__init__()

605

606 # init configs

607 config = vllm_config.model_config.hf_config

608 quant_config = vllm_config.quant_config

609 + multimodal_config = vllm_config.model_config.multimodal_config

610 # text_config

611 text_config = config.text_config

612 if text_config.model_type in ["gpt2", "hyperclovax", "llama"]:

@@ -631,7 +629,8 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

629 with no_init_weights(): # weight will be loaded in from_pretrained

630 self.vision_model = init_vision_tower_for_hcxvision(

631 vision_config,

634 - quant_config,

632 + quant_config=quant_config,

633 + multimodal_config=multimodal_config,

634 use_nth_layer=getattr(config, "use_nth_layer", -1),

635 require_post_norm=False,

636 prefix=maybe_prefix(prefix, "vision_model"),

vllm/model_executor/models/isaac.py

+1 -1

@@ -1226,8 +1226,8 @@ class IsaacVisionEmbedding(nn.Module):

1226 self.transformer = Siglip2VisionTransformer(

1227 vision_cfg,

1228 quant_config=quant_config,

1229 - prefix=maybe_prefix(prefix, "0"),

1229 multimodal_config=multimodal_config,

1230 + prefix=maybe_prefix(prefix, "0"),

1231 )

1232 self.linear_fc1 = ColumnParallelLinear(

1233 hidden_dim,

vllm/model_executor/models/keye.py

@@ -404,6 +404,7 @@ class KeyeSiglipAttention(nn.Module):

404 self.attn = MMEncoderAttention(

405 num_heads=self.num_heads,

406 head_size=self.head_dim,

407 + scale=self.scale,

408 num_kv_heads=self.num_kv_heads,

409 prefix=f"{prefix}.attn",

410 multimodal_config=multimodal_config,

@@ -511,6 +512,7 @@ class KeyeSiglipEncoderLayer(nn.Module):

512 self.mlp = SiglipMLP(

513 config,

514 quant_config=quant_config,

515 + multimodal_config=multimodal_config,

516 prefix=f"{prefix}.mlp",

517 )

518

vllm/model_executor/models/lightonocr.py

+3 -1

@@ -155,6 +155,7 @@ class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):

155

156 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

157 nn.Module.__init__(self)

158 +

159 config = vllm_config.model_config.hf_config

160 quant_config = vllm_config.quant_config

161 multimodal_config = vllm_config.model_config.multimodal_config

@@ -164,7 +165,8 @@ class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):

165

166 self.vision_tower = init_vision_tower_for_llava(

167 config,

167 - quant_config,

168 + quant_config=quant_config,

169 + multimodal_config=multimodal_config,

170 require_post_norm=False,

171 prefix=maybe_prefix(prefix, "vision_tower"),

172 )

vllm/model_executor/models/llava.py

+7 -2

@@ -19,7 +19,7 @@ from transformers.models.llava import LlavaProcessor

19 from transformers.models.pixtral import PixtralProcessor

21 from vllm.config import VllmConfig

22 -from vllm.config.multimodal import BaseDummyOptions

22 +from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

23 from vllm.model_executor.layers.activation import get_act_fn

24 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

25 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -468,6 +468,7 @@ def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:

468 def init_vision_tower_for_llava(

469 hf_config: LlavaLikeConfig,

470 quant_config: QuantizationConfig | None,

471 + multimodal_config: MultiModalConfig | None,

472 *,

473 require_post_norm: bool | None = None,

474 prefix: str = "",

@@ -481,6 +482,7 @@ def init_vision_tower_for_llava(

482 return CLIPVisionModel(

483 vision_config,

484 quant_config=quant_config,

485 + multimodal_config=multimodal_config,

486 num_hidden_layers_override=num_hidden_layers,

487 require_post_norm=require_post_norm,

488 prefix=prefix,

@@ -489,6 +491,7 @@ def init_vision_tower_for_llava(

491 return SiglipVisionModel(

492 vision_config,

493 quant_config=quant_config,

494 + multimodal_config=multimodal_config,

495 num_hidden_layers_override=num_hidden_layers,

496 require_post_norm=require_post_norm,

497 prefix=prefix,

@@ -497,6 +500,7 @@ def init_vision_tower_for_llava(

500 return PixtralHFVisionModel(

501 vision_config,

502 quant_config=quant_config,

503 + multimodal_config=multimodal_config,

504 num_hidden_layers_override=num_hidden_layers,

505 require_post_norm=require_post_norm,

506 prefix=prefix,

@@ -563,7 +567,8 @@ class LlavaForConditionalGeneration(

567 if multimodal_config.get_limit_per_prompt("image"):

568 self.vision_tower = init_vision_tower_for_llava(

569 config,

566 - quant_config,

570 + quant_config=quant_config,

571 + multimodal_config=multimodal_config,

572 require_post_norm=False,

573 prefix=maybe_prefix(prefix, "vision_tower"),

574 )

vllm/model_executor/models/llava_next.py

+3 -1

@@ -243,6 +243,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP

243

244 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

245 super().__init__()

246 +

247 config = vllm_config.model_config.hf_config

248 quant_config = vllm_config.quant_config

249 multimodal_config = vllm_config.model_config.multimodal_config

@@ -270,7 +271,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP

271 # TODO: Optionally initializes this for supporting embeddings.

272 self.vision_tower = init_vision_tower_for_llava(

273 config,

273 - quant_config,

274 + quant_config=quant_config,

275 + multimodal_config=multimodal_config,

276 require_post_norm=False,

277 prefix=maybe_prefix(prefix, "vision_tower"),

278 )

vllm/model_executor/models/llava_next_video.py

+3 -1

@@ -321,6 +321,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp

321

322 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

323 super().__init__()

324 +

325 config = vllm_config.model_config.hf_config

326 quant_config = vllm_config.quant_config

327 multimodal_config = vllm_config.model_config.multimodal_config

@@ -331,7 +332,8 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp

332 # Initialize the vision tower only up to the required feature layer

333 self.vision_tower = init_vision_tower_for_llava(

334 config,

334 - quant_config,

335 + quant_config=quant_config,

336 + multimodal_config=multimodal_config,

337 require_post_norm=False,

338 prefix=maybe_prefix(prefix, "vision_tower"),

339 )

vllm/model_executor/models/llava_onevision.py

+2 -1

@@ -511,7 +511,8 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp

511 # Initialize the vision tower only up to the required feature layer

512 self.vision_tower = init_vision_tower_for_llava(

513 config,

514 - quant_config,

514 + quant_config=quant_config,

515 + multimodal_config=multimodal_config,

516 require_post_norm=False,

517 prefix=maybe_prefix(prefix, "vision_tower"),

518 )

vllm/model_executor/models/minimax_vl_01.py

+2 -1

@@ -204,7 +204,8 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support

204 # TODO: Optionally initializes this for supporting embeddings.

205 self.vision_tower = init_vision_tower_for_llava(

206 config,

207 - quant_config,

207 + quant_config=quant_config,

208 + multimodal_config=multimodal_config,

209 require_post_norm=False,

210 prefix=maybe_prefix(prefix, "vision_tower"),

211 )

vllm/model_executor/models/mistral3.py

+5 -2

@@ -16,7 +16,7 @@ from transformers import (

16 from transformers.models.pixtral import PixtralProcessor

18 from vllm.config import VllmConfig

19 -from vllm.config.multimodal import BaseDummyOptions

19 +from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

20 from vllm.model_executor.layers.activation import get_act_fn

21 from vllm.model_executor.layers.layernorm import RMSNorm

22 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

@@ -395,6 +395,7 @@ def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:

395 def init_vision_tower_for_llava(

396 hf_config: LlavaLikeConfig,

397 quant_config: QuantizationConfig | None,

398 + multimodal_config: MultiModalConfig | None,

399 *,

400 require_post_norm: bool | None = None,

401 prefix: str = "",

@@ -409,6 +410,7 @@ def init_vision_tower_for_llava(

410 return PixtralHFVisionModel(

411 vision_config,

412 quant_config=quant_config,

413 + multimodal_config=multimodal_config,

414 num_hidden_layers_override=num_hidden_layers,

415 require_post_norm=require_post_norm,

416 prefix=prefix,

@@ -472,7 +474,8 @@ class Mistral3ForConditionalGeneration(

474 if multimodal_config.get_limit_per_prompt("image"):

475 self.vision_tower = init_vision_tower_for_llava(

476 config,

475 - quant_config,

477 + quant_config=quant_config,

478 + multimodal_config=multimodal_config,

479 require_post_norm=False,

480 prefix=maybe_prefix(prefix, "vision_tower"),

481 )

vllm/model_executor/models/paddleocr_vl.py

+2 -42

@@ -38,10 +38,8 @@ from vllm.config import MultiModalConfig, VllmConfig

38 from vllm.config.multimodal import BaseDummyOptions

39 from vllm.distributed import parallel_state

40 from vllm.distributed import utils as dist_utils

41 -from vllm.model_executor.layers.activation import get_act_fn

41 from vllm.model_executor.layers.conv import Conv2dLayer

42 from vllm.model_executor.layers.linear import (

44 - ColumnParallelLinear,

43 QKVParallelLinear,

44 RowParallelLinear,

45 )

@@ -77,6 +75,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape

76 from .ernie45 import Ernie4_5ForCausalLM

77 from .interfaces import MultiModalEmbeddings, SupportsMRoPE, SupportsMultiModal

78 +from .siglip import SiglipMLP

79 from .utils import (

80 AutoWeightsLoader,

81 PPMissingLayer,

@@ -657,46 +656,6 @@ class SigLIPRotaryEmbedding(nn.Module):

656 return freqs

657

658

660 -class SiglipMLP(nn.Module):

661 - def __init__(

662 - self,

663 - config: PretrainedConfig,

664 - quant_config: QuantizationConfig | None = None,

665 - prefix: str = "",

666 - ) -> None:

667 - super().__init__()

668 -

669 - self.config = config

670 - self.activation_fn = get_act_fn(config.hidden_act)

671 - # Special handling for BNB and torchao quantization

672 - if quant_config and quant_config.get_name() in ["bitsandbytes", "torchao"]:

673 - quantizable = True

674 - else:

675 - # For other quantization, we require the hidden size to be a

676 - # multiple of 64

677 - quantizable = (

678 - config.hidden_size % 64 == 0 and config.intermediate_size % 64 == 0

679 - )

680 - self.fc1 = ColumnParallelLinear(

681 - config.hidden_size,

682 - config.intermediate_size,

683 - quant_config=quant_config if quantizable else None,

684 - prefix=f"{prefix}.fc1",

685 - )

686 - self.fc2 = RowParallelLinear(

687 - config.intermediate_size,

688 - config.hidden_size,

689 - quant_config=quant_config if quantizable else None,

690 - prefix=f"{prefix}.fc2",

691 - )

692 -

693 - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:

694 - hidden_states, _ = self.fc1(hidden_states)

695 - hidden_states = self.activation_fn(hidden_states)

696 - hidden_states, _ = self.fc2(hidden_states)

697 - return hidden_states

698 -

699 -

659 class SiglipEncoderLayer(nn.Module):

660 def __init__(

661 self,

@@ -720,6 +679,7 @@ class SiglipEncoderLayer(nn.Module):

679 self.mlp = SiglipMLP(

680 config,

681 quant_config=quant_config,

682 + multimodal_config=multimodal_config,

683 prefix=f"{prefix}.mlp",

684 )

685

vllm/model_executor/models/phi3v.py

+31 -31

@@ -29,7 +29,7 @@ from transformers import (

29 )

31 from vllm.config import VllmConfig

32 -from vllm.config.multimodal import BaseDummyOptions

32 +from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

33 from vllm.logger import init_logger

34 from vllm.model_executor.layers.quantization import QuantizationConfig

35 from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding

@@ -96,6 +96,7 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(

96 def _init_img_processor(

97 hf_config: PretrainedConfig,

98 quant_config: QuantizationConfig | None,

99 + multimodal_config: MultiModalConfig | None,

100 prefix: str = "",

101 ) -> CLIPVisionModel:

102 clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG

@@ -109,7 +110,8 @@ def _init_img_processor(

110

111 img_processor = CLIPVisionModel(

112 clip_config,

112 - quant_config,

113 + quant_config=quant_config,

114 + multimodal_config=multimodal_config,

115 num_hidden_layers_override=num_hidden_layers,

116 prefix=prefix,

117 )

@@ -160,38 +162,15 @@ class Phi3VImageEmbeddingInputs(TensorSchema):

162 Phi3VImageInputs: TypeAlias = Phi3VImagePixelInputs | Phi3VImageEmbeddingInputs

163

164

163 -class Phi3ImageEmbeddingBase(nn.Module):

164 - def __init__(self) -> None:

165 - super().__init__()

166 - self.layer_idx: int

167 - self.type_feature: str

168 - self.img_processor: CLIPVisionModel

169 -

170 - def get_img_features(self, img_embeds: torch.FloatTensor) -> torch.FloatTensor:

171 - TYPE_FEATURE = self.type_feature

172 -

173 - # NOTE: we skip the step to select the vision feature layer since

174 - # this is already done inside the img_processor

175 - img_feature = self.img_processor(img_embeds)

176 -

177 - if TYPE_FEATURE == "patch":

178 - patch_feature = img_feature[:, 1:]

179 - return patch_feature

180 -

181 - if TYPE_FEATURE == "cls_patch":

182 - return img_feature

183 -

184 - raise NotImplementedError

185 -

186 -

165 # adapted from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_embedding_phi3_v.py

188 -class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):

166 +class Phi3HDImageEmbedding(nn.Module):

167 """Phi3 Image embedding with HD transform."""

168

169 def __init__(

170 self,

171 config: PretrainedConfig,

172 quant_config: QuantizationConfig | None,

173 + multimodal_config: MultiModalConfig | None,

174 prefix: str = "",

175 ) -> None:

176 super().__init__()

@@ -200,7 +179,10 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):

179 hidden_size = config.n_embd if hasattr(config, "n_embd") else config.hidden_size

180

181 self.img_processor = _init_img_processor(

203 - config, quant_config, prefix=f"{prefix}.img_processor"

182 + config,

183 + quant_config=quant_config,

184 + multimodal_config=multimodal_config,

185 + prefix=f"{prefix}.img_processor",

186 )

187

188 image_dim_out = config.img_processor["image_dim_out"]

@@ -223,13 +205,29 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):

205

206 dim_projection = hidden_size

207 depth = 2

226 - layers = [nn.Linear(image_dim_out * 4, dim_projection)]

208 + layers: list[nn.Module] = [nn.Linear(image_dim_out * 4, dim_projection)]

209 for _ in range(1, depth):

210 layers.extend([nn.GELU(), nn.Linear(dim_projection, dim_projection)])

211 self.img_projection = nn.Sequential(*layers)

212

213 self.type_feature = config.img_processor.get("type_feature", "patch")

214

215 + def get_img_features(self, img_embeds: torch.FloatTensor) -> torch.FloatTensor:

216 + type_feature = self.type_feature

217 +

218 + # NOTE: we skip the step to select the vision feature layer since

219 + # this is already done inside the img_processor

220 + img_feature = self.img_processor(img_embeds)

221 +

222 + if type_feature == "patch":

223 + patch_feature = img_feature[:, 1:]

224 + return patch_feature

225 +

226 + if type_feature == "cls_patch":

227 + return img_feature

228 +

229 + raise NotImplementedError(type_feature)

230 +

231 def forward(

232 self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor

233 ) -> torch.FloatTensor:

@@ -582,6 +580,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant)

580 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

581 super().__init__()

582 config = vllm_config.model_config.hf_config

583 + quant_config = vllm_config.quant_config

584 multimodal_config = vllm_config.model_config.multimodal_config

585 self.config = config

586 self.multimodal_config = multimodal_config

@@ -590,14 +589,15 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant)

589 self.embed_tokens = VocabParallelEmbedding(

590 config.vocab_size,

591 config.hidden_size,

593 - quant_config=self.quant_config,

592 + quant_config=quant_config,

593 prefix=maybe_prefix(prefix, "model.embed_tokens"),

594 )

595

596 # TODO: Optionally initializes this for supporting input embeddings.

597 self.vision_embed_tokens = Phi3HDImageEmbedding(

598 config,

600 - self.quant_config,

599 + quant_config=quant_config,

600 + multimodal_config=multimodal_config,

601 prefix=maybe_prefix(prefix, "model.vision_embed_tokens"),

602 )

603

vllm/model_executor/models/pixtral.py

+38 -7

@@ -28,7 +28,7 @@ from transformers.models.pixtral.modeling_pixtral import (

28 from transformers.tokenization_utils_base import TextInput

30 from vllm.config import VllmConfig

31 -from vllm.config.multimodal import BaseDummyOptions

31 +from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

32 from vllm.distributed import divide, get_tensor_model_parallel_world_size

33 from vllm.model_executor.layers.activation import get_act_and_mul_fn

34 from vllm.model_executor.layers.conv import Conv2dLayer

@@ -1043,11 +1043,18 @@ class PixtralHFMLP(nn.Module):

1043 self,

1044 config: PixtralVisionConfig,

1045 quant_config: QuantizationConfig | None = None,

1046 + multimodal_config: MultiModalConfig | None = None,

1047 *,

1048 prefix: str = "",

1049 ) -> None:

1050 super().__init__()

1051

1052 + use_data_parallel = (

1053 + multimodal_config.mm_encoder_tp_mode == "data"

1054 + if multimodal_config

1055 + else False

1056 + )

1057 +

1058 assert config.intermediate_size is not None

1059 self.gate_up_proj = MergedColumnParallelLinear(

1060 input_size=config.hidden_size,

@@ -1055,6 +1062,7 @@ class PixtralHFMLP(nn.Module):

1062 bias=False,

1063 quant_config=quant_config,

1064 prefix=f"{prefix}.gate_up_proj",

1065 + disable_tp=use_data_parallel,

1066 )

1067 self.down_proj = RowParallelLinear(

1068 input_size=config.intermediate_size,

@@ -1062,6 +1070,7 @@ class PixtralHFMLP(nn.Module):

1070 bias=False,

1071 quant_config=quant_config,

1072 prefix=f"{prefix}.down_proj",

1073 + disable_tp=use_data_parallel,

1074 )

1075 self.act_and_mul = get_act_and_mul_fn(config.hidden_act)

1076

@@ -1077,6 +1086,7 @@ class PixtralHFAttention(nn.Module):

1086 self,

1087 config: PixtralVisionConfig,

1088 quant_config: QuantizationConfig | None = None,

1089 + multimodal_config: MultiModalConfig | None = None,

1090 *,

1091 prefix: str = "",

1092 ) -> None:

@@ -1085,10 +1095,14 @@ class PixtralHFAttention(nn.Module):

1095 self.config = config

1096 assert not config.hidden_size % config.num_attention_heads

1097 self.total_num_heads = config.num_attention_heads

1088 - tp_size = get_tensor_model_parallel_world_size()

1089 - self.n_heads = divide(config.num_attention_heads, tp_size)

1098 self.head_dim = config.hidden_size // config.num_attention_heads

1099 + assert self.total_num_heads * self.head_dim == config.hidden_size

1100

1101 + use_data_parallel = (

1102 + multimodal_config.mm_encoder_tp_mode == "data"

1103 + if multimodal_config

1104 + else False

1105 + )

1106 self.qkv_proj = QKVParallelLinear(

1107 hidden_size=config.hidden_size,

1108 head_size=self.head_dim,

@@ -1096,15 +1110,21 @@ class PixtralHFAttention(nn.Module):

1110 bias=False,

1111 quant_config=quant_config,

1112 prefix=f"{prefix}.qkv_proj",

1113 + disable_tp=use_data_parallel,

1114 )

1100 - assert self.total_num_heads * self.head_dim == config.hidden_size

1115 self.o_proj = RowParallelLinear(

1116 input_size=config.hidden_size,

1117 output_size=config.hidden_size,

1118 bias=False,

1119 quant_config=quant_config,

1120 prefix=f"{prefix}.o_proj",

1121 + disable_tp=use_data_parallel,

1122 + )

1123 +

1124 + self.tp_size = (

1125 + 1 if use_data_parallel else get_tensor_model_parallel_world_size()

1126 )

1127 + self.n_heads = divide(config.num_attention_heads, self.tp_size)

1128

1129 def forward(

1130 self,

@@ -1147,6 +1167,7 @@ class PixtralHFTransformerBlock(nn.Module):

1167 self,

1168 config: PixtralVisionConfig,

1169 quant_config: QuantizationConfig | None = None,

1170 + multimodal_config: MultiModalConfig | None = None,

1171 *,

1172 prefix: str = "",

1173 ) -> None:

@@ -1154,10 +1175,16 @@ class PixtralHFTransformerBlock(nn.Module):

1175

1176 self.attention_norm = RMSNorm(config.hidden_size, eps=1e-5)

1177 self.attention = PixtralHFAttention(

1157 - config, quant_config=quant_config, prefix=f"{prefix}.attention"

1178 + config,

1179 + quant_config=quant_config,

1180 + multimodal_config=multimodal_config,

1181 + prefix=f"{prefix}.attention",

1182 )

1183 self.feed_forward = PixtralHFMLP(

1160 - config, quant_config=quant_config, prefix=f"{prefix}.feed_forward"

1184 + config,

1185 + quant_config=quant_config,

1186 + multimodal_config=multimodal_config,

1187 + prefix=f"{prefix}.feed_forward",

1188 )

1189 self.ffn_norm = RMSNorm(config.hidden_size, eps=1e-5)

1190

@@ -1183,6 +1210,7 @@ class PixtralHFTransformer(nn.Module):

1210 self,

1211 config: PixtralVisionConfig,

1212 quant_config: QuantizationConfig | None = None,

1213 + multimodal_config: MultiModalConfig | None = None,

1214 *,

1215 num_hidden_layers_override: int | None = None,

1216 prefix: str = "",

@@ -1199,6 +1227,7 @@ class PixtralHFTransformer(nn.Module):

1227 PixtralHFTransformerBlock(

1228 config=config,

1229 quant_config=quant_config,

1230 + multimodal_config=multimodal_config,

1231 prefix=f"{prefix}.layers.{layer_idx}",

1232 )

1233 for layer_idx in range(num_hidden_layers)

@@ -1230,6 +1259,7 @@ class PixtralHFVisionModel(nn.Module):

1259 self,

1260 config: PixtralVisionConfig,

1261 quant_config: QuantizationConfig | None = None,

1262 + multimodal_config: MultiModalConfig | None = None,

1263 *,

1264 num_hidden_layers_override: int | None = None,

1265 require_post_norm: bool | None = None,

@@ -1249,7 +1279,8 @@ class PixtralHFVisionModel(nn.Module):

1279 self.ln_pre = RMSNorm(config.hidden_size, eps=1e-5)

1280 self.transformer = PixtralHFTransformer(

1281 config,

1252 - quant_config,

1282 + quant_config=quant_config,

1283 + multimodal_config=multimodal_config,

1284 num_hidden_layers_override=num_hidden_layers_override,

1285 prefix=f"{prefix}.transformer",

1286 )

vllm/model_executor/models/siglip.py

+73 -55

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 -import math

4 from collections.abc import Callable, Iterable, Mapping

5 from functools import cached_property

6 from typing import Annotated, Literal

@@ -19,7 +18,7 @@ from transformers import (

18 from vllm.attention.layers.encoder_only_attention import EncoderOnlyAttention

19 from vllm.attention.layers.mm_encoder_attention import MMEncoderAttention

20 from vllm.config import VllmConfig

22 -from vllm.config.multimodal import BaseDummyOptions

21 +from vllm.config.multimodal import BaseDummyOptions, MultiModalConfig

22 from vllm.distributed import divide, get_tensor_model_parallel_world_size

23 from vllm.model_executor.layers.activation import get_act_fn

24 from vllm.model_executor.layers.conv import Conv2dLayer

@@ -276,7 +275,7 @@ class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):

275 return image_size // patch_size

276

277

279 -# Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa

278 +# Adapted from https://github.com/huggingface/transformers/blob/v4.57.3/src/transformers/models/siglip/modeling_siglip.py#L216

279 class SiglipVisionEmbeddings(nn.Module):

280 def __init__(self, config: SiglipVisionConfig):

281 super().__init__()

@@ -295,9 +294,7 @@ class SiglipVisionEmbeddings(nn.Module):

294

295 self.num_patches = (self.image_size // self.patch_size) ** 2

296 self.num_positions = self.num_patches

298 - self.position_embedding = VocabParallelEmbedding(

299 - self.num_positions, self.embed_dim

300 - )

297 + self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)

298 self.register_buffer(

299 "position_ids",

300 torch.arange(self.num_positions, dtype=torch.int64).expand((1, -1)),

@@ -307,50 +304,30 @@ class SiglipVisionEmbeddings(nn.Module):

304 def interpolate_pos_encoding(

305 self, embeddings: torch.Tensor, height: int, width: int

306 ) -> torch.Tensor:

310 - """

311 - This method is an adapted method for SigLIP (due to SigLIP not having

312 - class embedding unlike other ViTs) that allows the model to interpolate

313 - the pre-trained position encodings such that it can be usable on higher

314 - resolution images.

315 -

316 - Source:

317 - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174

318 - """

319 - position_embeddings = self.position_embedding.weight.unsqueeze(0)

307 num_patches = embeddings.shape[1]

321 - num_positions = position_embeddings.shape[1]

308 + num_positions = self.position_embedding.weight.shape[1]

309 if num_patches == num_positions and height == width:

323 - return position_embeddings

310 + return self.position_embedding(self.position_ids)

311 +

312 + patch_pos_embed = self.position_embedding.weight.unsqueeze(0)

313

314 dim = embeddings.shape[-1]

326 - height = height // self.patch_size

327 - width = width // self.patch_size

328 - # we add a small number to avoid floating point error

329 - # in the interpolation

330 - # see discussion at https://github.com/facebookresearch/dino/issues/8

331 - height, width = height + 0.1, width + 0.1

332 -

333 - patch_pos_embed = position_embeddings.reshape(

334 - 1, int(math.sqrt(num_positions)), int(math.sqrt(num_positions)), dim

315 +

316 + new_height = height // self.patch_size

317 + new_width = width // self.patch_size

318 +

319 + sqrt_num_positions = int(num_positions**0.5)

320 + patch_pos_embed = patch_pos_embed.reshape(

321 + 1, sqrt_num_positions, sqrt_num_positions, dim

322 )

323 patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)

324 +

325 patch_pos_embed = nn.functional.interpolate(

326 patch_pos_embed,

339 - scale_factor=(

340 - height / math.sqrt(num_positions),

341 - width / math.sqrt(num_positions),

342 - ),

327 + size=(new_height, new_width),

328 mode="bicubic",

329 align_corners=False,

330 )

346 - if (

347 - int(height) != patch_pos_embed.shape[-2]

348 - or int(width) != patch_pos_embed.shape[-1]

349 - ):

350 - raise ValueError(

351 - "Width or height does not match with "

352 - "the interpolated position embeddings"

353 - )

331

332 patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)

333 return patch_pos_embed

@@ -377,6 +354,7 @@ class SiglipAttention(nn.Module):

354 self,

355 config: SiglipVisionConfig | SiglipTextConfig,

356 quant_config: QuantizationConfig | None = None,

357 + multimodal_config: MultiModalConfig | None = None,

358 *,

359 prefix: str = "",

360 attn_cls: type[EncoderOnlyAttention] | type[MMEncoderAttention],

@@ -389,19 +367,25 @@ class SiglipAttention(nn.Module):

367 self.head_dim = self.embed_dim // self.num_heads

368 if self.head_dim * self.num_heads != self.embed_dim:

369 raise ValueError(

392 - f"embed_dim must be divisible by num_heads (got "

393 - "`embed_dim`: {self.embed_dim} and `num_heads`:"

394 - f" {self.num_heads})."

370 + f"embed_dim must be divisible by num_heads "

371 + f"(got `embed_dim`: {self.embed_dim} and "

372 + f"`num_heads`: {self.num_heads})."

373 )

374

375 self.scale = self.head_dim**-0.5

398 - self.dropout = config.attention_dropout

376 +

377 + use_data_parallel = (

378 + multimodal_config.mm_encoder_tp_mode == "data"

379 + if multimodal_config

380 + else False

381 + )

382 self.qkv_proj = QKVParallelLinear(

383 hidden_size=self.embed_dim,

384 head_size=self.head_dim,

385 total_num_heads=self.num_heads,

386 quant_config=quant_config,

387 prefix=f"{prefix}.qkv_proj",

388 + disable_tp=use_data_parallel,

389 )

390

391 self.out_proj = RowParallelLinear(

@@ -409,17 +393,29 @@ class SiglipAttention(nn.Module):

393 output_size=self.embed_dim,

394 quant_config=quant_config,

395 prefix=f"{prefix}.out_proj",

396 + disable_tp=use_data_parallel,

397 )

398

414 - self.tp_size = get_tensor_model_parallel_world_size()

399 + self.tp_size = (

400 + 1 if use_data_parallel else get_tensor_model_parallel_world_size()

401 + )

402 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)

403

417 - self.attn = attn_cls(

418 - self.num_heads_per_partition,

419 - self.head_dim,

420 - self.scale,

421 - prefix=f"{prefix}.attn",

422 - )

404 + if attn_cls == MMEncoderAttention:

405 + self.attn = attn_cls(

406 + self.num_heads_per_partition,

407 + self.head_dim,

408 + self.scale,

409 + prefix=f"{prefix}.attn",

410 + multimodal_config=multimodal_config,

411 + )

412 + else:

413 + self.attn = attn_cls(

414 + self.num_heads_per_partition,

415 + self.head_dim,

416 + self.scale,

417 + prefix=f"{prefix}.attn",

418 + )

419

420 def forward(

421 self,

@@ -439,12 +435,19 @@ class SiglipMLP(nn.Module):

435 self,

436 config: SiglipVisionConfig | SiglipTextConfig,

437 quant_config: QuantizationConfig | None = None,

438 + multimodal_config: MultiModalConfig | None = None,

439 prefix: str = "",

440 ) -> None:

441 super().__init__()

442

443 self.config = config

444 + use_data_parallel = (

445 + multimodal_config.mm_encoder_tp_mode == "data"

446 + if multimodal_config

447 + else False

448 + )

449 self.activation_fn = get_act_fn(config.hidden_act)

450 +

451 # Special handling for BNB and torchao quantization

452 if quant_config and quant_config.get_name() in ["bitsandbytes", "torchao"]:

453 quantizable = True

@@ -454,17 +457,20 @@ class SiglipMLP(nn.Module):

457 quantizable = (

458 config.hidden_size % 64 == 0 and config.intermediate_size % 64 == 0

459 )

460 +

461 self.fc1 = ColumnParallelLinear(

462 config.hidden_size,

463 config.intermediate_size,

464 quant_config=quant_config if quantizable else None,

465 prefix=f"{prefix}.fc1",

466 + disable_tp=use_data_parallel,

467 )

468 self.fc2 = RowParallelLinear(

469 config.intermediate_size,

470 config.hidden_size,

471 quant_config=quant_config if quantizable else None,

472 prefix=f"{prefix}.fc2",

473 + disable_tp=use_data_parallel,

474 )

475

476 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:

@@ -479,6 +485,7 @@ class SiglipEncoderLayer(nn.Module):

485 self,

486 config: SiglipVisionConfig | SiglipTextConfig,

487 quant_config: QuantizationConfig | None = None,

488 + multimodal_config: MultiModalConfig | None = None,

489 *,

490 prefix: str = "",

491 attn_cls: type[EncoderOnlyAttention] | type[MMEncoderAttention],

@@ -490,6 +497,7 @@ class SiglipEncoderLayer(nn.Module):

497 self.self_attn = SiglipAttention(

498 config,

499 quant_config=quant_config,

500 + multimodal_config=multimodal_config,

501 prefix=f"{prefix}.self_attn",

502 attn_cls=attn_cls,

503 )

@@ -497,6 +505,7 @@ class SiglipEncoderLayer(nn.Module):

505 self.mlp = SiglipMLP(

506 config,

507 quant_config=quant_config,

508 + multimodal_config=multimodal_config,

509 prefix=f"{prefix}.mlp",

510 )

511 self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)

@@ -524,6 +533,7 @@ class SiglipEncoder(nn.Module):

533 self,

534 config: SiglipVisionConfig | SiglipTextConfig,

535 quant_config: QuantizationConfig | None = None,

536 + multimodal_config: MultiModalConfig | None = None,

537 num_hidden_layers_override: int | None = None,

538 *,

539 prefix: str = "",

@@ -543,6 +553,7 @@ class SiglipEncoder(nn.Module):

553 SiglipEncoderLayer(

554 config,

555 quant_config=quant_config,

556 + multimodal_config=multimodal_config,

557 prefix=f"{prefix}.layers.{layer_idx}",

558 attn_cls=attn_cls,

559 )

@@ -647,6 +658,7 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):

658 self,

659 config: SiglipVisionConfig,

660 quant_config: QuantizationConfig | None = None,

661 + multimodal_config: MultiModalConfig | None = None,

662 prefix: str = "",

663 ) -> None:

664 super().__init__()

@@ -658,7 +670,10 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):

670 )

671 self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)

672 self.mlp = SiglipMLP(

661 - config=config, quant_config=quant_config, prefix=f"{prefix}.mlp"

673 + config=config,

674 + quant_config=quant_config,

675 + multimodal_config=multimodal_config,

676 + prefix=f"{prefix}.mlp",

677 )

678

679 def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:

@@ -683,6 +698,7 @@ class SiglipVisionTransformer(nn.Module):

698 self,

699 config: SiglipVisionConfig,

700 quant_config: QuantizationConfig | None = None,

701 + multimodal_config: MultiModalConfig | None = None,

702 *,

703 num_hidden_layers_override: int | None = None,

704 require_post_norm: bool | None = None,

@@ -698,6 +714,7 @@ class SiglipVisionTransformer(nn.Module):

714 self.encoder = SiglipEncoder(

715 config,

716 quant_config=quant_config,

717 + multimodal_config=multimodal_config,

718 num_hidden_layers_override=num_hidden_layers_override,

719 prefix=f"{prefix}.encoder",

720 attn_cls=MMEncoderAttention,

@@ -726,6 +743,7 @@ class SiglipVisionTransformer(nn.Module):

743 self.head = SiglipMultiheadAttentionPoolingHead(

744 config=config,

745 quant_config=quant_config,

746 + multimodal_config=multimodal_config,

747 prefix=f"{prefix}.head",

748 )

749

@@ -812,13 +830,11 @@ class SiglipVisionTransformer(nn.Module):

830

831

832 class SiglipVisionModel(nn.Module):

815 - config_class = SiglipVisionConfig

816 - main_input_name = "pixel_values"

817 -

833 def __init__(

834 self,

835 config: SiglipVisionConfig,

836 quant_config: QuantizationConfig | None = None,

837 + multimodal_config: MultiModalConfig | None = None,

838 *,

839 num_hidden_layers_override: int | None = None,

840 require_post_norm: bool | None = None,

@@ -829,7 +845,8 @@ class SiglipVisionModel(nn.Module):

845 self.quant_config = quant_config

846 self.vision_model = SiglipVisionTransformer(

847 config,

832 - quant_config,

848 + quant_config=quant_config,

849 + multimodal_config=multimodal_config,

850 num_hidden_layers_override=num_hidden_layers_override,

851 require_post_norm=require_post_norm,

852 prefix=f"{prefix}.vision_model",

@@ -1023,6 +1040,7 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1040 self.vision_model = SiglipVisionTransformer(

1041 vision_config,

1042 quant_config=quant_config,

1043 + multimodal_config=multimodal_config,

1044 prefix=maybe_prefix(prefix, "vision_model"),

1045 )

1046

vllm/model_executor/models/siglip2navit.py

+3 -5

@@ -11,7 +11,6 @@ from torch.nn import functional as F

11 from transformers import Siglip2VisionConfig

12 from transformers.configuration_utils import PretrainedConfig

14 -from vllm.attention.backends.registry import AttentionBackendEnum

14 from vllm.attention.layers.mm_encoder_attention import MMEncoderAttention

15 from vllm.config import MultiModalConfig

16 from vllm.distributed import divide, get_tensor_model_parallel_world_size

@@ -186,7 +185,6 @@ class Siglip2Attention(nn.Module):

185 multimodal_config: MultiModalConfig | None = None,

186 prefix: str = "",

187 use_data_parallel: bool = False,

189 - attn_backend_override: AttentionBackendEnum | None = None,

188 ):

189 super().__init__()

190 self.config = config

@@ -196,12 +194,11 @@ class Siglip2Attention(nn.Module):

194 if self.head_dim * self.num_heads != self.embed_dim:

195 raise ValueError(

196 f"embed_dim must be divisible by num_heads "

199 - f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"

200 - f" {self.num_heads})."

197 + f"(got `embed_dim`: {self.embed_dim} and "

198 + f"`num_heads`: {self.num_heads})."

199 )

200 self.scale = self.head_dim**-0.5

201 self.dropout = config.attention_dropout

204 - self.is_causal = False

202

203 use_data_parallel = (

204 multimodal_config.mm_encoder_tp_mode == "data"

@@ -233,6 +230,7 @@ class Siglip2Attention(nn.Module):

230 self.attn = MMEncoderAttention(

231 num_heads=self.num_heads_per_partition,

232 head_size=self.head_dim,

233 + scale=self.scale,

234 prefix=f"{prefix}.attn",

235 multimodal_config=multimodal_config,

236 )

vllm/model_executor/models/tarsier.py

+9 -2

@@ -19,7 +19,7 @@ from transformers.models.llava import LlavaProcessor

19 from transformers.processing_utils import ProcessingKwargs, Unpack

20 from transformers.tokenization_utils_base import PreTokenizedInput, TextInput

22 -from vllm.config import VllmConfig

22 +from vllm.config import MultiModalConfig, VllmConfig

23 from vllm.model_executor.layers.activation import get_act_fn

24 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

25 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -346,6 +346,7 @@ def _build_tarsier_hf_processor(

346 def init_vision_tower_for_tarsier(

347 hf_config: TarsierHfConfig, # Use the Tarsier specific config protocol

348 quant_config: QuantizationConfig | None,

349 + multimodal_config: MultiModalConfig | None,

350 *,

351 require_post_norm: bool | None = None,

352 prefix: str = "",

@@ -377,6 +378,7 @@ def init_vision_tower_for_tarsier(

378 return CLIPVisionModel(

379 vision_config,

380 quant_config=quant_config,

381 + multimodal_config=multimodal_config,

382 num_hidden_layers_override=num_hidden_layers_to_init,

383 require_post_norm=require_post_norm,

384 prefix=prefix,

@@ -385,6 +387,7 @@ def init_vision_tower_for_tarsier(

387 return SiglipVisionModel(

388 vision_config,

389 quant_config=quant_config,

390 + multimodal_config=multimodal_config,

391 num_hidden_layers_override=num_hidden_layers_to_init,

392 require_post_norm=require_post_norm,

393 prefix=prefix,

@@ -414,12 +417,16 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

417

418 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

419 super().__init__()

420 +

421 config: TarsierHfConfig = vllm_config.model_config.hf_config

422 quant_config = vllm_config.quant_config

423 + multimodal_config = vllm_config.model_config.multimodal_config

424 +

425 self.config = config # Storing the Tarsier-specific HF config

426 self.vision_tower = init_vision_tower_for_tarsier(

427 config,

422 - quant_config,

428 + quant_config=quant_config,

429 + multimodal_config=multimodal_config,

430 require_post_norm=False,

431 prefix=maybe_prefix(prefix, "vision_tower"),

432 )