e4bb2684bc · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/layers/conv.py

+22 -2

@@ -3,6 +3,7 @@

3 """Conv Layer Class."""

4

5 import math

6 +from typing import Literal

7

8 import torch

9 import torch.nn as nn

@@ -23,11 +24,11 @@ class ConvLayerBase(CustomOp):

24 out_channels: int,

25 kernel_size: int | tuple[int, ...],

26 stride: int | tuple[int, ...] = 1,

26 - padding: int | tuple[int, ...] = 0,

27 + padding: int | tuple[int, ...] | Literal["same", "valid"] = 0,

28 dilation: int | tuple[int, ...] = 1,

29 groups: int = 1,

30 bias: bool = True,

30 - padding_mode: str = "zeros",

31 + padding_mode: Literal["zeros", "reflect", "replicate", "circular"] = "zeros",

32 *,

33 params_dtype: torch.dtype | None = None,

34 ) -> None:

@@ -36,6 +37,22 @@ class ConvLayerBase(CustomOp):

37 if params_dtype is None:

38 params_dtype = torch.get_default_dtype()

39

40 + valid_padding_strings = {"same", "valid"}

41 + if isinstance(padding, str) and padding not in valid_padding_strings:

42 + raise ValueError(

43 + f"Invalid padding string '{padding}'. "

44 + f"Expected one of {valid_padding_strings}."

45 + )

46 +

47 + if padding == "same":

48 + padding = (

49 + kernel_size // 2

50 + if isinstance(kernel_size, int)

51 + else tuple(k // 2 for k in kernel_size)

52 + )

53 + elif padding == "valid":

54 + padding = 0

55 +

56 kernel_size = (

57 (kernel_size,) * self.num_dim

58 if isinstance(kernel_size, int)

@@ -45,6 +62,9 @@ class ConvLayerBase(CustomOp):

62 padding = (padding,) * self.num_dim if isinstance(padding, int) else padding

63 dilation = (dilation,) * self.num_dim if isinstance(dilation, int) else dilation

64

65 + if padding == "same" and any(s != 1 for s in stride):

66 + raise ValueError("padding='same' is not supported for strided convolutions")

67 +

68 self.in_channels = in_channels

69 self.out_channels = out_channels

70 self.kernel_size = kernel_size

vllm/model_executor/models/aimv2.py

+2 -1

@@ -12,6 +12,7 @@ from vllm.attention.layer import MultiHeadAttention

12 from vllm.distributed import get_tensor_model_parallel_world_size

13 from vllm.distributed.utils import divide

14 from vllm.model_executor.layers.activation import SiluAndMul

15 +from vllm.model_executor.layers.conv import Conv2dLayer

16 from vllm.model_executor.layers.layernorm import RMSNorm

17 from vllm.model_executor.layers.linear import (

18 MergedColumnParallelLinear,

@@ -58,7 +59,7 @@ class AIMv2SwiGLUFFN(nn.Module):

59 class AIMv2PatchEmbed(nn.Module):

60 def __init__(self, config: AIMv2Config):

61 super().__init__()

61 - self.proj = nn.Conv2d(

62 + self.proj = Conv2dLayer(

63 config.num_channels,

64 config.hidden_size,

65 kernel_size=(config.patch_size, config.patch_size),

vllm/model_executor/models/blip.py

+2 -1

@@ -12,6 +12,7 @@ from transformers import Blip2VisionConfig, BlipVisionConfig

12 from vllm.attention.layer import MultiHeadAttention

13 from vllm.distributed import divide, get_tensor_model_parallel_world_size

14 from vllm.model_executor.layers.activation import get_act_fn

15 +from vllm.model_executor.layers.conv import Conv2dLayer

16 from vllm.model_executor.layers.linear import (

17 ColumnParallelLinear,

18 QKVParallelLinear,

@@ -47,7 +48,7 @@ class BlipVisionEmbeddings(nn.Module):

48

49 self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim))

50

50 - self.patch_embedding = nn.Conv2d(

51 + self.patch_embedding = Conv2dLayer(

52 in_channels=3,

53 out_channels=self.embed_dim,

54 kernel_size=self.patch_size,

vllm/model_executor/models/chameleon.py

+14 -15

@@ -22,6 +22,7 @@ from vllm.config.multimodal import BaseDummyOptions

22 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size

23 from vllm.logger import init_logger

24 from vllm.model_executor.layers.activation import SiluAndMul

25 +from vllm.model_executor.layers.conv import Conv2dLayer

26 from vllm.model_executor.layers.layernorm import RMSNorm

27 from vllm.model_executor.layers.linear import (

28 MergedColumnParallelLinear,

@@ -549,7 +550,7 @@ class ChameleonVQVAEVectorQuantizer(nn.Module):

550 class ChameleonVQVAEEncoderConvDownsample(nn.Module):

551 def __init__(self, in_channels: int):

552 super().__init__()

552 - self.conv = nn.Conv2d(

553 + self.conv = Conv2dLayer(

554 in_channels, in_channels, kernel_size=3, stride=2, padding=0

555 )

556

@@ -577,23 +578,23 @@ class ChameleonVQVAEEncoderResnetBlock(nn.Module):

578 self.norm1 = torch.nn.GroupNorm(

579 num_groups=32, num_channels=in_channels, eps=1e-6, affine=True

580 )

580 - self.conv1 = torch.nn.Conv2d(

581 + self.conv1 = Conv2dLayer(

582 in_channels, out_channels, kernel_size=3, stride=1, padding=1

583 )

584 self.norm2 = torch.nn.GroupNorm(

585 num_groups=32, num_channels=out_channels, eps=1e-6, affine=True

586 )

587 self.dropout = torch.nn.Dropout(config.dropout)

587 - self.conv2 = torch.nn.Conv2d(

588 + self.conv2 = Conv2dLayer(

589 out_channels, out_channels, kernel_size=3, stride=1, padding=1

590 )

591 if self.in_channels != self.out_channels:

592 if self.use_conv_shortcut:

592 - self.conv_shortcut = torch.nn.Conv2d(

593 + self.conv_shortcut = Conv2dLayer(

594 in_channels, out_channels, kernel_size=3, stride=1, padding=1

595 )

596 else:

596 - self.nin_shortcut = torch.nn.Conv2d(

597 + self.nin_shortcut = Conv2dLayer(

598 in_channels, out_channels, kernel_size=1, stride=1, padding=0

599 )

600

@@ -626,16 +627,16 @@ class ChameleonVQVAEEncoderAttnBlock(nn.Module):

627 self.norm = torch.nn.GroupNorm(

628 num_groups=32, num_channels=in_channels, eps=1e-6, affine=True

629 )

629 - self.q = torch.nn.Conv2d(

630 + self.q = Conv2dLayer(

631 in_channels, in_channels, kernel_size=1, stride=1, padding=0

632 )

632 - self.k = torch.nn.Conv2d(

633 + self.k = Conv2dLayer(

634 in_channels, in_channels, kernel_size=1, stride=1, padding=0

635 )

635 - self.v = torch.nn.Conv2d(

636 + self.v = Conv2dLayer(

637 in_channels, in_channels, kernel_size=1, stride=1, padding=0

638 )

638 - self.proj_out = torch.nn.Conv2d(

639 + self.proj_out = Conv2dLayer(

640 in_channels, in_channels, kernel_size=1, stride=1, padding=0

641 )

642

@@ -681,7 +682,7 @@ class ChameleonVQVAEEncoder(nn.Module):

682 latent_channels = config.latent_channels

683 channel_multiplier = config.channel_multiplier

684

684 - self.conv_in = torch.nn.Conv2d(

685 + self.conv_in = Conv2dLayer(

686 in_channels, base_channels, kernel_size=3, stride=1, padding=1

687 )

688

@@ -738,7 +739,7 @@ class ChameleonVQVAEEncoder(nn.Module):

739 self.norm_out = torch.nn.GroupNorm(

740 num_groups=32, num_channels=block_in, eps=1e-6, affine=True

741 )

741 - self.conv_out = torch.nn.Conv2d(

742 + self.conv_out = Conv2dLayer(

743 block_in,

744 2 * latent_channels if double_latent else latent_channels,

745 kernel_size=3,

@@ -779,10 +780,8 @@ class ChameleonVQVAE(nn.Module):

780 super().__init__()

781 self.encoder = ChameleonVQVAEEncoder(config)

782 self.quantize = ChameleonVQVAEVectorQuantizer(config)

782 - self.quant_conv = torch.nn.Conv2d(config.latent_channels, config.embed_dim, 1)

783 - self.post_quant_conv = torch.nn.Conv2d(

784 - config.embed_dim, config.latent_channels, 1

785 - )

783 + self.quant_conv = Conv2dLayer(config.latent_channels, config.embed_dim, 1)

784 + self.post_quant_conv = Conv2dLayer(config.embed_dim, config.latent_channels, 1)

785 self.eval() # Chameleon's VQ model is frozen

786

787 def encode(

vllm/model_executor/models/deepencoder.py

+8 -5

@@ -19,6 +19,7 @@ import torch.nn.functional as F

19 from transformers import CLIPVisionConfig

20

21 from vllm.attention.layer import MultiHeadAttention

22 +from vllm.model_executor.layers.conv import Conv2dLayer

23 from vllm.model_executor.layers.quantization import QuantizationConfig

24 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

25

@@ -133,14 +134,14 @@ class ImageEncoderViT(nn.Module):

134 self.blocks.append(block)

135

136 self.neck = nn.Sequential(

136 - nn.Conv2d(

137 + Conv2dLayer(

138 embed_dim,

139 out_chans,

140 kernel_size=1,

141 bias=False,

142 ),

143 LayerNorm2d(out_chans),

143 - nn.Conv2d(

144 + Conv2dLayer(

145 out_chans,

146 out_chans,

147 kernel_size=3,

@@ -150,8 +151,10 @@ class ImageEncoderViT(nn.Module):

151 LayerNorm2d(out_chans),

152 )

153

153 - self.net_2 = nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1, bias=False)

154 - self.net_3 = nn.Conv2d(

154 + self.net_2 = Conv2dLayer(

155 + 256, 512, kernel_size=3, stride=2, padding=1, bias=False

156 + )

157 + self.net_3 = Conv2dLayer(

158 512, 1024, kernel_size=3, stride=2, padding=1, bias=False

159 )

160

@@ -500,7 +503,7 @@ class PatchEmbed(nn.Module):

503 """

504 super().__init__()

505

503 - self.proj = nn.Conv2d(

506 + self.proj = Conv2dLayer(

507 in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding

508 )

509

vllm/model_executor/models/dots_ocr.py

+2 -1

@@ -22,6 +22,7 @@ from vllm.distributed.parallel_state import (

22 get_tensor_model_parallel_world_size,

23 )

24 from vllm.model_executor.layers.activation import SiluAndMul

25 +from vllm.model_executor.layers.conv import Conv2dLayer

26 from vllm.model_executor.layers.layernorm import RMSNorm

27 from vllm.model_executor.layers.linear import (

28 ColumnParallelLinear,

@@ -471,7 +472,7 @@ class DotsPatchEmbed(nn.Module):

472 self.temporal_patch_size = config.temporal_patch_size

473 self.embed_dim = config.embed_dim

474 self.config = config

474 - self.proj = nn.Conv2d(

475 + self.proj = Conv2dLayer(

476 config.num_channels,

477 config.embed_dim,

478 kernel_size=(config.patch_size, config.patch_size),

vllm/model_executor/models/glm4_1v.py

+2 -2

@@ -56,7 +56,7 @@ from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

56 from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state

57 from vllm.distributed import utils as dist_utils

58 from vllm.logger import init_logger

59 -from vllm.model_executor.layers.conv import Conv3dLayer

59 +from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer

60 from vllm.model_executor.layers.layernorm import RMSNorm

61 from vllm.model_executor.layers.linear import (

62 ColumnParallelLinear,

@@ -734,7 +734,7 @@ class Glm4vVisionTransformer(nn.Module):

734 self.post_conv_layernorm = RMSNorm(

735 vision_config.hidden_size, eps=vision_config.rms_norm_eps

736 )

737 - self.downsample = nn.Conv2d(

737 + self.downsample = Conv2dLayer(

738 in_channels=vision_config.hidden_size,

739 out_channels=vision_config.out_hidden_size,

740 kernel_size=vision_config.spatial_merge_size,

vllm/model_executor/models/glm4v.py

+3 -2

@@ -24,6 +24,7 @@ from vllm.config import VllmConfig

24 from vllm.config.multimodal import BaseDummyOptions

25 from vllm.distributed import get_tensor_model_parallel_world_size

26 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn

27 +from vllm.model_executor.layers.conv import Conv2dLayer

28 from vllm.model_executor.layers.linear import (

29 ColumnParallelLinear,

30 MergedColumnParallelLinear,

@@ -78,7 +79,7 @@ class GLMVImagePixelInputs(TensorSchema):

79 class EVA2CLIPPatchEmbedding(nn.Module):

80 def __init__(self, config):

81 super().__init__()

81 - self.proj = nn.Conv2d(

82 + self.proj = Conv2dLayer(

83 config.in_channels,

84 config.hidden_size,

85 kernel_size=config.patch_size,

@@ -333,7 +334,7 @@ class EVA2CLIPModel(nn.Module):

334 quant_config=quant_config,

335 prefix=f"{prefix}.linear_proj",

336 )

336 - self.conv = nn.Conv2d(

337 + self.conv = Conv2dLayer(

338 in_channels=vision_config.hidden_size,

339 out_channels=config.hidden_size,

340 kernel_size=2,

vllm/model_executor/models/idefics2_vision_model.py

+2 -1

@@ -30,6 +30,7 @@ from transformers.models.idefics2.configuration_idefics2 import (

30 from vllm.attention.layer import MultiHeadAttention

31 from vllm.distributed import get_tensor_model_parallel_world_size

32 from vllm.model_executor.layers.activation import get_act_fn

33 +from vllm.model_executor.layers.conv import Conv2dLayer

34 from vllm.model_executor.layers.linear import (

35 ColumnParallelLinear,

36 QKVParallelLinear,

@@ -60,7 +61,7 @@ class Idefics2VisionEmbeddings(nn.Module):

61 self.embed_dim = config.hidden_size

62 self.image_size = config.image_size

63 self.patch_size = config.patch_size

63 - self.patch_embedding = nn.Conv2d(

64 + self.patch_embedding = Conv2dLayer(

65 in_channels=config.num_channels,

66 out_channels=self.embed_dim,

67 kernel_size=self.patch_size,

vllm/model_executor/models/intern_vit.py

+2 -1

@@ -24,6 +24,7 @@ from vllm.distributed import (

24 tensor_model_parallel_all_gather,

25 )

26 from vllm.model_executor.layers.activation import get_act_fn

27 +from vllm.model_executor.layers.conv import Conv2dLayer

28 from vllm.model_executor.layers.layernorm import RMSNorm

29 from vllm.model_executor.layers.linear import (

30 ColumnParallelLinear,

@@ -51,7 +52,7 @@ class InternVisionEmbeddings(nn.Module):

52

53 self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim))

54

54 - self.patch_embedding = nn.Conv2d(

55 + self.patch_embedding = Conv2dLayer(

56 in_channels=3,

57 out_channels=self.embed_dim,

58 kernel_size=self.patch_size,

vllm/model_executor/models/interns1_vit.py

+2 -1

@@ -16,6 +16,7 @@ from transformers.utils import torch_int

16

17 from vllm.attention.layer import MultiHeadAttention

18 from vllm.model_executor.layers.activation import get_act_fn

19 +from vllm.model_executor.layers.conv import Conv2dLayer

20 from vllm.model_executor.layers.layernorm import RMSNorm

21 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

22 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -43,7 +44,7 @@ class InternS1VisionPatchEmbeddings(nn.Module):

44 self.num_patches = num_patches

45 self.patch_shape = patch_shape

46

46 - self.projection = nn.Conv2d(

47 + self.projection = Conv2dLayer(

48 num_channels, hidden_size, kernel_size=patch_size, stride=patch_size

49 )

50

vllm/model_executor/models/keye.py

+2 -1

@@ -24,6 +24,7 @@ from vllm.config import VllmConfig

24 from vllm.config.multimodal import BaseDummyOptions

25 from vllm.distributed import get_tensor_model_parallel_world_size

26 from vllm.logger import init_logger

27 +from vllm.model_executor.layers.conv import Conv2dLayer

28 from vllm.model_executor.layers.linear import (

29 ColumnParallelLinear,

30 QKVParallelLinear,

@@ -204,7 +205,7 @@ class KeyeVisionEmbeddings(nn.Module):

205 self.image_size = config.image_size

206 self.patch_size = config.patch_size

207

207 - self.patch_embedding = nn.Conv2d(

208 + self.patch_embedding = Conv2dLayer(

209 in_channels=config.num_channels,

210 out_channels=self.embed_dim,

211 kernel_size=self.patch_size,

vllm/model_executor/models/midashenglm.py

+2 -1

@@ -39,6 +39,7 @@ from vllm.config import VllmConfig

39 from vllm.config.multimodal import BaseDummyOptions

40 from vllm.distributed import get_tensor_model_parallel_world_size

41 from vllm.model_executor.layers.activation import get_act_fn

42 +from vllm.model_executor.layers.conv import Conv2dLayer

43 from vllm.model_executor.layers.linear import (

44 ColumnParallelLinear,

45 QKVParallelLinear,

@@ -120,7 +121,7 @@ class AudioPatchEmbed(nn.Module):

121 self.num_patches = self.grid_size[0] * self.grid_size[1]

122 self.flatten = flatten

123

123 - self.proj = nn.Conv2d(

124 + self.proj = Conv2dLayer(

125 in_chans,

126 embed_dim,

127 kernel_size=self.patch_size,

vllm/model_executor/models/moonvit.py

+2 -1

@@ -53,6 +53,7 @@ from transformers.activations import ACT2FN

53 from transformers.modeling_utils import PreTrainedModel

54 from transformers.utils import is_flash_attn_2_available

55

56 +from vllm.model_executor.layers.conv import Conv2dLayer

57 from vllm.model_executor.layers.linear import ReplicatedLinear

58 from vllm.model_executor.models.utils import maybe_prefix

59 from vllm.transformers_utils.configs.moonvit import MoonViTConfig

@@ -244,7 +245,7 @@ class MoonVisionPatchEmbed(nn.Module):

245 )

246 self.patch_size = patch_size

247

247 - self.proj = nn.Conv2d(

248 + self.proj = Conv2dLayer(

249 in_dim, out_dim, kernel_size=patch_size, stride=patch_size

250 )

251

vllm/model_executor/models/paddleocr_vl.py

+2 -1

@@ -45,6 +45,7 @@ from vllm.config.multimodal import BaseDummyOptions

45 from vllm.distributed import parallel_state

46 from vllm.distributed import utils as dist_utils

47 from vllm.model_executor.layers.activation import get_act_fn

48 +from vllm.model_executor.layers.conv import Conv2dLayer

49 from vllm.model_executor.layers.linear import (

50 ColumnParallelLinear,

51 QKVParallelLinear,

@@ -419,7 +420,7 @@ class SiglipVisionEmbeddings(nn.Module):

420 self.image_size = config.image_size

421 self.patch_size = config.patch_size

422

422 - self.patch_embedding = nn.Conv2d(

423 + self.patch_embedding = Conv2dLayer(

424 in_channels=config.num_channels,

425 out_channels=self.embed_dim,

426 kernel_size=self.patch_size,

vllm/model_executor/models/pixtral.py

+3 -2

@@ -31,6 +31,7 @@ from vllm.config import VllmConfig

31 from vllm.config.multimodal import BaseDummyOptions

32 from vllm.distributed import divide, get_tensor_model_parallel_world_size

33 from vllm.model_executor.layers.activation import get_act_and_mul_fn

34 +from vllm.model_executor.layers.conv import Conv2dLayer

35 from vllm.model_executor.layers.layernorm import RMSNorm

36 from vllm.model_executor.layers.linear import (

37 MergedColumnParallelLinear,

@@ -747,7 +748,7 @@ class VisionTransformer(nn.Module):

748 def __init__(self, args: VisionEncoderArgs):

749 super().__init__()

750 self.args = args

750 - self.patch_conv = nn.Conv2d(

751 + self.patch_conv = Conv2dLayer(

752 in_channels=args.num_channels,

753 out_channels=args.hidden_size,

754 kernel_size=args.patch_size,

@@ -1212,7 +1213,7 @@ class PixtralHFVisionModel(nn.Module):

1213

1214 self.config = config

1215

1215 - self.patch_conv = nn.Conv2d(

1216 + self.patch_conv = Conv2dLayer(

1217 in_channels=config.num_channels,

1218 out_channels=config.hidden_size,

1219 kernel_size=config.patch_size,

vllm/model_executor/models/qwen_vl.py

+2 -1

@@ -25,6 +25,7 @@ from transformers.tokenization_utils_base import TextInput

25 from vllm.config import VllmConfig

26 from vllm.config.multimodal import BaseDummyOptions

27 from vllm.model_executor.layers.activation import get_act_fn

28 +from vllm.model_executor.layers.conv import Conv2dLayer

29 from vllm.model_executor.layers.linear import (

30 ColumnParallelLinear,

31 ReplicatedLinear,

@@ -333,7 +334,7 @@ class VisionTransformer(nn.Module):

334 patch_height, patch_width = self.patch_size = (patch_size, patch_size)

335 self.grid_size = (image_height // patch_height, image_width // patch_width)

336 self.output_dim = output_dim

336 - self.conv1 = nn.Conv2d(

337 + self.conv1 = Conv2dLayer(

338 in_channels=3,

339 out_channels=width,

340 kernel_size=patch_size,

vllm/model_executor/models/siglip.py

+2 -1

@@ -24,6 +24,7 @@ from vllm.config import VllmConfig

24 from vllm.config.multimodal import BaseDummyOptions

25 from vllm.distributed import divide, get_tensor_model_parallel_world_size

26 from vllm.model_executor.layers.activation import get_act_fn

27 +from vllm.model_executor.layers.conv import Conv2dLayer

28 from vllm.model_executor.layers.linear import (

29 ColumnParallelLinear,

30 QKVParallelLinear,

@@ -286,7 +287,7 @@ class SiglipVisionEmbeddings(nn.Module):

287 self.image_size = config.image_size

288 self.patch_size = config.patch_size

289

289 - self.patch_embedding = nn.Conv2d(

290 + self.patch_embedding = Conv2dLayer(

291 in_channels=config.num_channels,

292 out_channels=self.embed_dim,

293 kernel_size=self.patch_size,

vllm/model_executor/models/siglip2navit.py

+3 -2

@@ -16,6 +16,7 @@ from vllm.attention.backends.registry import AttentionBackendEnum

16 from vllm.attention.layer import maybe_get_vit_flash_attn_backend

17 from vllm.distributed import divide, get_tensor_model_parallel_world_size

18 from vllm.model_executor.layers.activation import get_act_fn

19 +from vllm.model_executor.layers.conv import Conv2dLayer

20 from vllm.model_executor.layers.linear import (

21 ColumnParallelLinear,

22 LinearBase,

@@ -67,7 +68,7 @@ class Siglip2VisionEmbeddings(nn.Module):

68 self.position_embedding = nn.Embedding(self.num_patches, self.embed_dim)

69

70 else:

70 - self.patch_embedding = nn.Conv2d(

71 + self.patch_embedding = Conv2dLayer(

72 in_channels=config.num_channels,

73 out_channels=self.embed_dim,

74 kernel_size=self.patch_size,

@@ -99,7 +100,7 @@ class Siglip2VisionEmbeddings(nn.Module):

100 target_dtype = self.patch_embedding.weight.dtype

101 if isinstance(self.patch_embedding, LinearBase):

102 patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype))

102 - elif isinstance(self.patch_embedding, nn.Conv2d):

103 + elif isinstance(self.patch_embedding, Conv2dLayer):

104 pixel_values = pixel_values.view(

105 -1,

106 self.config.num_channels * self.config.temporal_patch_size,

vllm/model_executor/models/step3_vl.py

+4 -3

@@ -20,6 +20,7 @@ from vllm.config import VllmConfig

20 from vllm.config.multimodal import BaseDummyOptions

21 from vllm.distributed import get_tensor_model_parallel_world_size

22 from vllm.model_executor.layers.activation import get_act_fn

23 +from vllm.model_executor.layers.conv import Conv2dLayer

24 from vllm.model_executor.layers.linear import (

25 ColumnParallelLinear,

26 QKVParallelLinear,

@@ -667,7 +668,7 @@ class Step3VisionEmbeddings(nn.Module):

668

669 self.class_embedding = nn.Parameter(torch.randn(1, self.embed_dim))

670

670 - self.patch_embedding = nn.Conv2d(

671 + self.patch_embedding = Conv2dLayer(

672 in_channels=config.num_channels,

673 out_channels=self.embed_dim,

674 kernel_size=self.patch_size,

@@ -950,13 +951,13 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)

951 prefix=maybe_prefix(prefix, "vision_model"),

952 use_data_parallel=self.use_data_parallel,

953 )

953 - self.vit_downsampler = nn.Conv2d(

954 + self.vit_downsampler = Conv2dLayer(

955 config.vision_config.hidden_size,

956 config.vision_config.output_hidden_size,

957 kernel_size=2,

958 stride=config.understand_projector_stride,

959 )

959 - self.vit_downsampler2 = nn.Conv2d(

960 + self.vit_downsampler2 = Conv2dLayer(

961 config.vision_config.output_hidden_size,

962 config.vision_config.output_hidden_size * 2,

963 kernel_size=3,

[Models] Replace all `nn.Conv2d` with vLLM's Conv2dLayer (#28842)