| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py |
| 5 | # -------------------------------------------------------- |
| 6 | # InternVL |
| 7 | # Copyright (c) 2023 OpenGVLab |
| 8 | # Licensed under The MIT License [see LICENSE for details] |
| 9 | # -------------------------------------------------------- |
| 10 | from collections.abc import Iterable |
| 11 | |
| 12 | import torch |
| 13 | import torch.nn as nn |
| 14 | from transformers import PreTrainedConfig |
| 15 | from transformers.utils import torch_int |
| 16 | |
| 17 | from vllm.model_executor.layers.activation import get_act_fn |
| 18 | from vllm.model_executor.layers.attention import MMEncoderAttention |
| 19 | from vllm.model_executor.layers.conv import Conv2dLayer |
| 20 | from vllm.model_executor.layers.layernorm import RMSNorm |
| 21 | from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear |
| 22 | from vllm.model_executor.layers.quantization import QuantizationConfig |
| 23 | |
| 24 | from .utils import AutoWeightsLoader |
| 25 | |
| 26 | NORM2FN = { |
| 27 | "rms_norm": RMSNorm, |
| 28 | "layer_norm": nn.LayerNorm, |
| 29 | } |
| 30 | |
| 31 | |
| 32 | class InternS1VisionPatchEmbeddings(nn.Module): |
| 33 | def __init__(self, config): |
| 34 | super().__init__() |
| 35 | image_size, patch_size = config.image_size, config.patch_size |
| 36 | num_channels, hidden_size = config.num_channels, config.hidden_size |
| 37 | |
| 38 | num_patches = (image_size[1] // patch_size[1]) * ( |
| 39 | image_size[0] // patch_size[0] |
| 40 | ) |
| 41 | patch_shape = (image_size[0] // patch_size[0], image_size[1] // patch_size[1]) |
| 42 | self.image_size = image_size |
| 43 | self.patch_size = patch_size |
| 44 | self.num_channels = num_channels |
| 45 | self.num_patches = num_patches |
| 46 | self.patch_shape = patch_shape |
| 47 | |
| 48 | self.projection = Conv2dLayer( |
| 49 | num_channels, hidden_size, kernel_size=patch_size, stride=patch_size |
| 50 | ) |
| 51 | |
| 52 | def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: |
| 53 | batch_size, num_channels, height, width = pixel_values.shape |
| 54 | if num_channels != self.num_channels: |
| 55 | raise ValueError( |
| 56 | "Make sure that the channel dimension of the pixel values " |
| 57 | "match with the one set in the configuration." |
| 58 | ) |
| 59 | |
| 60 | embeddings = self.projection(pixel_values.to(self.projection.weight.dtype)) |
| 61 | patch_height, patch_width = embeddings.shape[2], embeddings.shape[3] |
| 62 | embeddings = embeddings.flatten(2).transpose(1, 2) |
| 63 | |
| 64 | return embeddings, (patch_height, patch_width) |
| 65 | |
| 66 | |
| 67 | class InternS1VisionEmbeddings(nn.Module): |
| 68 | def __init__(self, config: PreTrainedConfig): |
| 69 | super().__init__() |
| 70 | self.config = config |
| 71 | self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) |
| 72 | if config.use_mask_token: |
| 73 | self.mask_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) |
| 74 | else: |
| 75 | self.mask_token = None |
| 76 | self.patch_embeddings = InternS1VisionPatchEmbeddings(config) |
| 77 | self.patch_size = config.patch_size |
| 78 | self.image_size = ( |
| 79 | config.image_size |
| 80 | if isinstance(config.image_size, Iterable) |
| 81 | else (config.image_size, config.image_size) |
| 82 | ) |
| 83 | num_patches = self.patch_embeddings.num_patches |
| 84 | if config.use_absolute_position_embeddings: |
| 85 | self.position_embeddings = nn.Parameter( |
| 86 | torch.zeros(1, num_patches + 1, config.hidden_size) |
| 87 | ) |
| 88 | else: |
| 89 | self.position_embeddings = None |
| 90 | |
| 91 | def interpolate_pos_encoding( |
| 92 | self, embeddings: torch.Tensor, height: int, width: int |
| 93 | ) -> torch.Tensor: |
| 94 | """This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution |
| 95 | images. This method is also adapted to support torch.jit tracing. |
| 96 | |
| 97 | Adapted from: |
| 98 | - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and |
| 99 | - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 |
| 100 | """ # noqa: E501 |
| 101 | num_patches = embeddings.shape[1] - 1 |
| 102 | num_positions = self.position_embeddings.shape[1] - 1 |
| 103 | |
| 104 | # always interpolate when tracing to ensure the exported model |
| 105 | # works for dynamic input shapes |
| 106 | if ( |
| 107 | not torch.jit.is_tracing() |
| 108 | and num_patches == num_positions |
| 109 | and height == width |
| 110 | ): |
| 111 | return self.position_embeddings |
| 112 | |
| 113 | class_pos_embed = self.position_embeddings[:, :1] |
| 114 | patch_pos_embed = self.position_embeddings[:, 1:] |
| 115 | |
| 116 | dim = embeddings.shape[-1] |
| 117 | |
| 118 | new_height = height // self.patch_size[0] |
| 119 | new_width = width // self.patch_size[1] |
| 120 | |
| 121 | sqrt_num_positions = torch_int(num_positions**0.5) |
| 122 | patch_pos_embed = patch_pos_embed.reshape( |
| 123 | 1, sqrt_num_positions, sqrt_num_positions, dim |
| 124 | ) |
| 125 | patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2) |
| 126 | |
| 127 | patch_pos_embed = nn.functional.interpolate( |
| 128 | patch_pos_embed, |
| 129 | size=(new_height, new_width), |
| 130 | mode="bicubic", |
| 131 | align_corners=False, |
| 132 | ) |
| 133 | |
| 134 | patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) |
| 135 | |
| 136 | return torch.cat((class_pos_embed, patch_pos_embed), dim=1) |
| 137 | |
| 138 | def forward( |
| 139 | self, |
| 140 | pixel_values: torch.Tensor, |
| 141 | bool_masked_pos: torch.BoolTensor | None = None, |
| 142 | ) -> torch.Tensor: |
| 143 | _, _, height, width = pixel_values.shape |
| 144 | embeddings, (patch_height, patch_width) = self.patch_embeddings(pixel_values) |
| 145 | batch_size, seq_len, _ = embeddings.size() |
| 146 | |
| 147 | if bool_masked_pos is not None: |
| 148 | mask_tokens = self.mask_token.expand(batch_size, seq_len, -1) |
| 149 | # replace the masked visual tokens by mask_tokens |
| 150 | w = bool_masked_pos.unsqueeze(-1).type_as(mask_tokens) |
| 151 | embeddings = embeddings * (1 - w) + mask_tokens * w |
| 152 | |
| 153 | cls_tokens = self.cls_token.expand(batch_size, -1, -1) |
| 154 | embeddings = torch.cat((cls_tokens, embeddings), dim=1) |
| 155 | |
| 156 | if self.position_embeddings is not None: |
| 157 | embeddings = embeddings + self.interpolate_pos_encoding( |
| 158 | embeddings, height, width |
| 159 | ) |
| 160 | |
| 161 | return embeddings, (patch_height, patch_width) |
| 162 | |
| 163 | |
| 164 | class InternSdpaAttention(nn.Module): |
| 165 | """Multi-headed attention from 'Attention Is All You Need' paper.""" |
| 166 | |
| 167 | def __init__( |
| 168 | self, |
| 169 | config: PreTrainedConfig, |
| 170 | *, |
| 171 | num_dummy_heads: int = 0, |
| 172 | prefix: str = "", |
| 173 | ) -> None: |
| 174 | super().__init__() |
| 175 | |
| 176 | self.config = config |
| 177 | self.embed_dim = config.hidden_size |
| 178 | self.num_heads = config.num_attention_heads |
| 179 | self.head_dim = self.embed_dim // self.num_heads |
| 180 | if self.head_dim * self.num_heads != self.embed_dim: |
| 181 | raise ValueError( |
| 182 | f"embed_dim must be divisible by num_heads " |
| 183 | f"(got `embed_dim`: {self.embed_dim} and `num_heads`:" |
| 184 | f" {self.num_heads})." |
| 185 | ) |
| 186 | |
| 187 | # Additional dummy heads are used to enable TP for common GPU counts. |
| 188 | self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim |
| 189 | |
| 190 | self.scale = self.head_dim**-0.5 |
| 191 | |
| 192 | self.q_proj = nn.Linear( |
| 193 | self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias |
| 194 | ) |
| 195 | self.k_proj = nn.Linear( |
| 196 | self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias |
| 197 | ) |
| 198 | self.v_proj = nn.Linear( |
| 199 | self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias |
| 200 | ) |
| 201 | |
| 202 | self.qk_normalization = config.use_qk_norm |
| 203 | if self.qk_normalization: |
| 204 | self.q_norm = RMSNorm( |
| 205 | self.dummy_dim, |
| 206 | eps=config.layer_norm_eps, |
| 207 | var_hidden_size=self.embed_dim, |
| 208 | ) |
| 209 | self.k_norm = RMSNorm( |
| 210 | self.dummy_dim, |
| 211 | eps=config.layer_norm_eps, |
| 212 | var_hidden_size=self.embed_dim, |
| 213 | ) |
| 214 | |
| 215 | self.projection_layer = nn.Linear(self.dummy_dim, self.embed_dim) |
| 216 | |
| 217 | # Use unified MMEncoderAttention with automatic backend selection |
| 218 | self.attn = MMEncoderAttention( |
| 219 | self.num_heads, |
| 220 | self.head_dim, |
| 221 | self.scale, |
| 222 | prefix=f"{prefix}.attn", |
| 223 | ) |
| 224 | |
| 225 | def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 226 | """X shape: (B, N, C).""" |
| 227 | q = self.q_proj(x) |
| 228 | k = self.k_proj(x) |
| 229 | v = self.v_proj(x) |
| 230 | |
| 231 | if self.qk_normalization: |
| 232 | q = self.q_norm(q) |
| 233 | k = self.k_norm(k) |
| 234 | |
| 235 | # Use unified MMEncoderAttention with automatic backend selection |
| 236 | x = self.attn(q, k, v) |
| 237 | |
| 238 | x = self.projection_layer(x) |
| 239 | return x |
| 240 | |
| 241 | |
| 242 | class InternS1VisionMLP(nn.Module): |
| 243 | def __init__( |
| 244 | self, |
| 245 | config: PreTrainedConfig, |
| 246 | quant_config: QuantizationConfig | None = None, |
| 247 | prefix: str = "", |
| 248 | ) -> None: |
| 249 | super().__init__() |
| 250 | |
| 251 | self.config = config |
| 252 | self.activation_fn = get_act_fn(config.hidden_act) |
| 253 | self.fc1 = ColumnParallelLinear( |
| 254 | config.hidden_size, |
| 255 | config.intermediate_size, |
| 256 | bias=True, |
| 257 | quant_config=quant_config, |
| 258 | prefix=f"{prefix}.fc1", |
| 259 | ) |
| 260 | self.fc2 = RowParallelLinear( |
| 261 | config.intermediate_size, |
| 262 | config.hidden_size, |
| 263 | bias=True, |
| 264 | quant_config=quant_config, |
| 265 | prefix=f"{prefix}.fc2", |
| 266 | ) |
| 267 | |
| 268 | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: |
| 269 | hidden_states, _ = self.fc1(hidden_states) |
| 270 | hidden_states = self.activation_fn(hidden_states) |
| 271 | hidden_states, _ = self.fc2(hidden_states) |
| 272 | |
| 273 | return hidden_states |
| 274 | |
| 275 | |
| 276 | class InternS1VisionLayer(nn.Module): |
| 277 | def __init__( |
| 278 | self, |
| 279 | config: PreTrainedConfig, |
| 280 | quant_config: QuantizationConfig | None = None, |
| 281 | *, |
| 282 | num_dummy_heads: int = 0, |
| 283 | prefix: str = "", |
| 284 | ) -> None: |
| 285 | super().__init__() |
| 286 | |
| 287 | self.attention = self._init_attn( |
| 288 | config, |
| 289 | quant_config, |
| 290 | num_dummy_heads=num_dummy_heads, |
| 291 | prefix=f"{prefix}.attention", |
| 292 | ) |
| 293 | |
| 294 | self.mlp = InternS1VisionMLP( |
| 295 | config, quant_config=quant_config, prefix=f"{prefix}.mlp" |
| 296 | ) |
| 297 | self.layernorm_before = NORM2FN[config.norm_type]( |
| 298 | config.hidden_size, eps=config.layer_norm_eps |
| 299 | ) |
| 300 | self.layernorm_after = NORM2FN[config.norm_type]( |
| 301 | config.hidden_size, eps=config.layer_norm_eps |
| 302 | ) |
| 303 | |
| 304 | init_values = config.layer_scale_init_value |
| 305 | self.lambda_1 = nn.Parameter( |
| 306 | init_values * torch.ones(config.hidden_size), requires_grad=True |
| 307 | ) |
| 308 | self.lambda_2 = nn.Parameter( |
| 309 | init_values * torch.ones(config.hidden_size), requires_grad=True |
| 310 | ) |
| 311 | |
| 312 | def _init_attn( |
| 313 | self, |
| 314 | config: PreTrainedConfig, |
| 315 | quant_config: QuantizationConfig | None, |
| 316 | *, |
| 317 | num_dummy_heads: int, |
| 318 | prefix: str = "", |
| 319 | ): |
| 320 | return InternSdpaAttention( |
| 321 | config, |
| 322 | num_dummy_heads=num_dummy_heads, |
| 323 | prefix=prefix, |
| 324 | ) |
| 325 | |
| 326 | def forward( |
| 327 | self, |
| 328 | hidden_states: torch.Tensor, |
| 329 | ): |
| 330 | hidden_states = ( |
| 331 | hidden_states |
| 332 | + self.attention(self.layernorm_before(hidden_states)) * self.lambda_1 |
| 333 | ) |
| 334 | |
| 335 | hidden_states = ( |
| 336 | hidden_states |
| 337 | + self.mlp(self.layernorm_after(hidden_states)) * self.lambda_2 |
| 338 | ) |
| 339 | |
| 340 | return hidden_states |
| 341 | |
| 342 | |
| 343 | class InternS1VisionEncoder(nn.Module): |
| 344 | def __init__( |
| 345 | self, |
| 346 | config: PreTrainedConfig, |
| 347 | quant_config: QuantizationConfig | None = None, |
| 348 | *, |
| 349 | num_hidden_layers_override: int | None = None, |
| 350 | num_dummy_heads: int = 0, |
| 351 | prefix: str = "", |
| 352 | ): |
| 353 | super().__init__() |
| 354 | |
| 355 | self.config = config |
| 356 | |
| 357 | if num_hidden_layers_override is None: |
| 358 | num_hidden_layers = config.num_hidden_layers |
| 359 | else: |
| 360 | num_hidden_layers = num_hidden_layers_override |
| 361 | |
| 362 | self.layer = nn.ModuleList( |
| 363 | [ |
| 364 | InternS1VisionLayer( |
| 365 | config, |
| 366 | quant_config, |
| 367 | num_dummy_heads=num_dummy_heads, |
| 368 | prefix=f"{prefix}.layer.{layer_idx}", |
| 369 | ) |
| 370 | for layer_idx in range(num_hidden_layers) |
| 371 | ] |
| 372 | ) |
| 373 | |
| 374 | def forward(self, inputs_embeds: torch.Tensor): |
| 375 | hidden_states = inputs_embeds |
| 376 | for encoder_layer in self.layer: |
| 377 | hidden_states = encoder_layer(hidden_states) |
| 378 | |
| 379 | return hidden_states |
| 380 | |
| 381 | |
| 382 | class InternS1VisionModel(nn.Module): |
| 383 | def __init__( |
| 384 | self, |
| 385 | config: PreTrainedConfig, |
| 386 | quant_config: QuantizationConfig | None = None, |
| 387 | *, |
| 388 | num_hidden_layers_override: int | None = None, |
| 389 | num_dummy_heads: int = 0, |
| 390 | prefix: str = "", |
| 391 | ) -> None: |
| 392 | super().__init__() |
| 393 | |
| 394 | self.config = config |
| 395 | |
| 396 | self.embeddings = InternS1VisionEmbeddings(config) |
| 397 | self.encoder = InternS1VisionEncoder( |
| 398 | config=config, |
| 399 | num_hidden_layers_override=num_hidden_layers_override, |
| 400 | num_dummy_heads=num_dummy_heads, |
| 401 | prefix=f"{prefix}.encoder", |
| 402 | ) |
| 403 | self.layernorm = ( |
| 404 | nn.Identity() |
| 405 | if config.use_mean_pooling |
| 406 | else nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) |
| 407 | ) |
| 408 | |
| 409 | def get_input_embeddings(self): |
| 410 | return self.embeddings.patch_embeddings |
| 411 | |
| 412 | def forward( |
| 413 | self, |
| 414 | pixel_values: torch.Tensor | None = None, |
| 415 | pixel_embeds: torch.Tensor | None = None, |
| 416 | ) -> torch.FloatTensor: |
| 417 | if pixel_values is None and pixel_embeds is None: |
| 418 | raise ValueError("You have to specify pixel_values or pixel_embeds") |
| 419 | |
| 420 | if pixel_embeds is not None: |
| 421 | hidden_states = pixel_embeds |
| 422 | elif pixel_values is not None: |
| 423 | if pixel_values.ndim == 4: |
| 424 | hidden_states, _ = self.embeddings(pixel_values) |
| 425 | else: |
| 426 | raise ValueError(f"wrong pixel_values size: {pixel_values.shape}") |
| 427 | |
| 428 | encoder_outputs = self.encoder(inputs_embeds=hidden_states) |
| 429 | encoder_outputs = self.layernorm(encoder_outputs) |
| 430 | |
| 431 | return encoder_outputs |
| 432 | |
| 433 | def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: |
| 434 | loader = AutoWeightsLoader(self) |
| 435 | return loader.load_weights(weights) |