| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # Copyright 2024 The vLLM team. |
| 5 | # Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. |
| 6 | # |
| 7 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 8 | # you may not use this file except in compliance with the License. |
| 9 | # You may obtain a copy of the License at |
| 10 | # |
| 11 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | # |
| 13 | # Unless required by applicable law or agreed to in writing, software |
| 14 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 16 | # See the License for the specific language governing permissions and |
| 17 | # limitations under the License. |
| 18 | from collections.abc import Iterable, Mapping, Sequence |
| 19 | from typing import Annotated, Any, Literal, TypeAlias |
| 20 | |
| 21 | import regex as re |
| 22 | import torch |
| 23 | import torch.nn as nn |
| 24 | from transformers import ( |
| 25 | BatchFeature, |
| 26 | CLIPVisionConfig, |
| 27 | PreTrainedConfig, |
| 28 | ProcessorMixin, |
| 29 | ) |
| 30 | |
| 31 | from vllm.config import VllmConfig |
| 32 | from vllm.config.multimodal import MultiModalDummyOptions |
| 33 | from vllm.inputs import MultiModalDataDict |
| 34 | from vllm.logger import init_logger |
| 35 | from vllm.model_executor.layers.quantization import QuantizationConfig |
| 36 | from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding |
| 37 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 38 | from vllm.multimodal.inputs import ( |
| 39 | MultiModalFieldConfig, |
| 40 | MultiModalKwargsItems, |
| 41 | ) |
| 42 | from vllm.multimodal.parse import ( |
| 43 | ImageEmbeddingItems, |
| 44 | ImageProcessorItems, |
| 45 | ImageSize, |
| 46 | MultiModalDataItems, |
| 47 | ) |
| 48 | from vllm.multimodal.processing import BaseDummyInputsBuilder |
| 49 | from vllm.multimodal.processing.processor import ( |
| 50 | BaseMultiModalProcessor, |
| 51 | BaseProcessingInfo, |
| 52 | MultiModalPromptUpdates, |
| 53 | PlaceholderFeaturesInfo, |
| 54 | PromptReplacement, |
| 55 | PromptUpdate, |
| 56 | ResolvedPromptUpdate, |
| 57 | cached_encode, |
| 58 | ) |
| 59 | from vllm.sequence import IntermediateTensors |
| 60 | from vllm.utils.tensor_schema import TensorSchema, TensorShape |
| 61 | |
| 62 | from .clip import CLIPVisionModel |
| 63 | from .interfaces import ( |
| 64 | MultiModalEmbeddings, |
| 65 | SupportsMultiModal, |
| 66 | SupportsPP, |
| 67 | SupportsQuant, |
| 68 | _require_is_multimodal, |
| 69 | ) |
| 70 | from .utils import ( |
| 71 | AutoWeightsLoader, |
| 72 | WeightsMapper, |
| 73 | _merge_multimodal_embeddings, |
| 74 | init_vllm_registered_model, |
| 75 | maybe_prefix, |
| 76 | ) |
| 77 | |
| 78 | logger = init_logger(__name__) |
| 79 | |
| 80 | # Cannot find the following 2 numbers from hf config. |
| 81 | _IMAGE_TOKEN_ID = 32044 |
| 82 | |
| 83 | CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig( |
| 84 | dropout=0.0, |
| 85 | hidden_act="quick_gelu", |
| 86 | hidden_size=1024, |
| 87 | image_size=336, |
| 88 | intermediate_size=4096, |
| 89 | num_attention_heads=16, |
| 90 | num_channels=3, |
| 91 | num_hidden_layers=24, |
| 92 | patch_size=14, |
| 93 | projection_dim=768, |
| 94 | ) |
| 95 | |
| 96 | |
| 97 | def _init_img_processor( |
| 98 | hf_config: PreTrainedConfig, |
| 99 | quant_config: QuantizationConfig | None, |
| 100 | prefix: str = "", |
| 101 | ) -> CLIPVisionModel: |
| 102 | clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG |
| 103 | layer_idx = hf_config.img_processor.get("layer_idx", -2) |
| 104 | |
| 105 | # Initialize the CLIP only up to the required feature layer |
| 106 | if layer_idx < 0: |
| 107 | num_hidden_layers = clip_config.num_hidden_layers + layer_idx + 1 |
| 108 | else: |
| 109 | num_hidden_layers = layer_idx + 1 |
| 110 | |
| 111 | img_processor = CLIPVisionModel( |
| 112 | clip_config, |
| 113 | quant_config=quant_config, |
| 114 | num_hidden_layers_override=num_hidden_layers, |
| 115 | prefix=prefix, |
| 116 | ) |
| 117 | |
| 118 | return img_processor |
| 119 | |
| 120 | |
| 121 | class Phi3VImagePixelInputs(TensorSchema): |
| 122 | """Dimensions: |
| 123 | - b: Batch size |
| 124 | - n: Number of images |
| 125 | - p: Number of patches |
| 126 | - h: Height of each patch |
| 127 | - w: Width of each patch |
| 128 | """ |
| 129 | |
| 130 | type: Literal["pixel_values", "image_embeds"] = "pixel_values" |
| 131 | |
| 132 | # Supports either a stacked tensor or a list of (p, 3, h, w) tensors |
| 133 | pixel_values: Annotated[ |
| 134 | torch.Tensor | list[torch.Tensor], |
| 135 | TensorShape( |
| 136 | "bn", "p", 3, "h", "w", dynamic_dims={"p"} |
| 137 | ), # 'p' may vary across items |
| 138 | ] |
| 139 | |
| 140 | # Stacked tensor with height and width for each image |
| 141 | image_sizes: Annotated[torch.Tensor | None, TensorShape("bn", 2)] |
| 142 | |
| 143 | |
| 144 | class Phi3VImageEmbeddingInputs(TensorSchema): |
| 145 | """Dimensions: |
| 146 | - b: Batch size |
| 147 | - n: Number of images |
| 148 | - f: Image feature size (e.g., number of tokens per image) |
| 149 | - h: Hidden size (must match language model backbone) |
| 150 | """ |
| 151 | |
| 152 | type: Literal["image_embeds"] = "image_embeds" |
| 153 | data: Annotated[ |
| 154 | torch.Tensor | list[torch.Tensor], |
| 155 | TensorShape("bn", "f", "h"), |
| 156 | ] |
| 157 | |
| 158 | |
| 159 | Phi3VImageInputs: TypeAlias = Phi3VImagePixelInputs | Phi3VImageEmbeddingInputs |
| 160 | |
| 161 | |
| 162 | # adapted from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_embedding_phi3_v.py |
| 163 | class Phi3HDImageEmbedding(nn.Module): |
| 164 | """Phi3 Image embedding with HD transform.""" |
| 165 | |
| 166 | def __init__( |
| 167 | self, |
| 168 | config: PreTrainedConfig, |
| 169 | quant_config: QuantizationConfig | None, |
| 170 | prefix: str = "", |
| 171 | ) -> None: |
| 172 | super().__init__() |
| 173 | |
| 174 | # n_embed or hidden_size |
| 175 | hidden_size = config.n_embd if hasattr(config, "n_embd") else config.hidden_size |
| 176 | |
| 177 | self.img_processor = _init_img_processor( |
| 178 | config, |
| 179 | quant_config=quant_config, |
| 180 | prefix=f"{prefix}.img_processor", |
| 181 | ) |
| 182 | |
| 183 | image_dim_out = config.img_processor["image_dim_out"] |
| 184 | self.num_img_tokens = config.img_processor["num_img_tokens"] |
| 185 | |
| 186 | self.image_dim_out = image_dim_out |
| 187 | |
| 188 | # global_gn and sub_gn for hd transform, serves as line separator |
| 189 | self.use_hd_transform = config.embd_layer.get("use_hd_transform", False) |
| 190 | self.with_learnable_separator = config.embd_layer.get( |
| 191 | "with_learnable_separator", False |
| 192 | ) |
| 193 | self.hd_transform_order = config.embd_layer.get("hd_transform_order", "glb_sub") |
| 194 | # with_hd_transform and with_learnable_separator should have same value |
| 195 | assert self.use_hd_transform and self.with_learnable_separator |
| 196 | |
| 197 | # 1024 * 4, merge spatial to channel dimension |
| 198 | self.glb_GN = nn.Parameter(torch.empty([1, 1, self.image_dim_out * 4])) |
| 199 | self.sub_GN = nn.Parameter(torch.empty([1, 1, 1, self.image_dim_out * 4])) |
| 200 | |
| 201 | dim_projection = hidden_size |
| 202 | depth = 2 |
| 203 | layers: list[nn.Module] = [nn.Linear(image_dim_out * 4, dim_projection)] |
| 204 | for _ in range(1, depth): |
| 205 | layers.extend([nn.GELU(), nn.Linear(dim_projection, dim_projection)]) |
| 206 | self.img_projection = nn.Sequential(*layers) |
| 207 | |
| 208 | self.type_feature = config.img_processor.get("type_feature", "patch") |
| 209 | |
| 210 | def get_img_features(self, img_embeds: torch.FloatTensor) -> torch.FloatTensor: |
| 211 | type_feature = self.type_feature |
| 212 | |
| 213 | # NOTE: we skip the step to select the vision feature layer since |
| 214 | # this is already done inside the img_processor |
| 215 | img_feature = self.img_processor(img_embeds) |
| 216 | |
| 217 | if type_feature == "patch": |
| 218 | patch_feature = img_feature[:, 1:] |
| 219 | return patch_feature |
| 220 | |
| 221 | if type_feature == "cls_patch": |
| 222 | return img_feature |
| 223 | |
| 224 | raise NotImplementedError(type_feature) |
| 225 | |
| 226 | def forward( |
| 227 | self, pixel_values: torch.FloatTensor, image_sizes: torch.Tensor |
| 228 | ) -> torch.FloatTensor: |
| 229 | """Process image and return vision embeddings. |
| 230 | |
| 231 | pixel_values: (num_images, num_crops, c, h, w) |
| 232 | output: (num_images, num_img_tokens, hidden_size) |
| 233 | """ |
| 234 | num_images, num_crops, c, h, w = pixel_values.shape |
| 235 | pixel_values = pixel_values.flatten(0, 1) |
| 236 | img_features = self.get_img_features(pixel_values) |
| 237 | img_features = img_features.reshape( |
| 238 | num_images, num_crops, -1, self.image_dim_out |
| 239 | ) |
| 240 | image_features_proj = self.hd_feature_transform(img_features, image_sizes) |
| 241 | return image_features_proj |
| 242 | |
| 243 | def hd_feature_transform(self, image_features, image_sizes): |
| 244 | """image_features: (num_images, num_crops+1, 24*24, 1024).""" |
| 245 | assert self.hd_transform_order == "sub_glb", ( |
| 246 | f"hd_transform_order `{self.hd_transform_order}` not implemented" |
| 247 | ) |
| 248 | if isinstance(self.img_projection, nn.Sequential): |
| 249 | target_device = self.img_projection[0].bias.device |
| 250 | target_dtype = self.img_projection[0].bias.dtype |
| 251 | else: # It's a single nn.Linear layer |
| 252 | target_device = self.img_projection.bias.device |
| 253 | target_dtype = self.img_projection.bias.dtype |
| 254 | |
| 255 | global_image_features = image_features[:, 0] # (num_images, 24*24, 1024) |
| 256 | # global feature can be viewed as a special HD case with num_crops 1x1 |
| 257 | global_image_features_hd = self.reshape_hd_patches_2x2merge( |
| 258 | global_image_features, 1, 1 |
| 259 | ) |
| 260 | global_image_features_hd_newline = self.add_image_newline( |
| 261 | global_image_features_hd |
| 262 | ) |
| 263 | |
| 264 | batch_image_features_proj = [] |
| 265 | image_sizes_list = image_sizes.tolist() |
| 266 | # need a for loop to process each image because of different image sizes |
| 267 | # (patch arrangement is different for each image) |
| 268 | for i, img_size in enumerate(image_sizes_list): |
| 269 | h, w = img_size |
| 270 | h_crop = h // 336 |
| 271 | w_crop = w // 336 |
| 272 | num_crops = h_crop * w_crop |
| 273 | |
| 274 | # NOTE: real num_crops is padded |
| 275 | # (num_crops, 24*24, 1024) |
| 276 | sub_image_features = image_features[i, 1 : 1 + num_crops] |
| 277 | sub_image_features_hd = self.reshape_hd_patches_2x2merge( |
| 278 | sub_image_features, h_crop, w_crop |
| 279 | ) |
| 280 | sub_image_features_hd_newline = self.add_image_newline( |
| 281 | sub_image_features_hd |
| 282 | ) |
| 283 | |
| 284 | # [sub features, separator, global features] |
| 285 | image_embeddings = torch.cat( |
| 286 | [ |
| 287 | sub_image_features_hd_newline.squeeze( |
| 288 | 0 |
| 289 | ), # (h_crop*12*(w_crop*12+1), 4096) |
| 290 | self.glb_GN.squeeze(0), |
| 291 | global_image_features_hd_newline[i], |
| 292 | ] |
| 293 | ) |
| 294 | img_proj = self.img_projection( |
| 295 | image_embeddings.to(target_device, target_dtype) |
| 296 | ) |
| 297 | batch_image_features_proj.append(img_proj) |
| 298 | |
| 299 | return batch_image_features_proj |
| 300 | |
| 301 | def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop): |
| 302 | """image_features: (num_images*num_crops, 24*24, 1024) |
| 303 | output: (num_images, h_crop*12, w_crop*12, 4096) |
| 304 | where h_crop*w_crop == num_crops |
| 305 | """ |
| 306 | N, L, C = image_features.shape |
| 307 | assert L == 576 and C == 1024 and N % (h_crop * w_crop) == 0 |
| 308 | num_images = N // (h_crop * w_crop) |
| 309 | H = int(L**0.5) |
| 310 | image_features_hd = ( |
| 311 | image_features.reshape(N, H, H, C) # N, 24, 24, 1024 |
| 312 | .reshape(N, H // 2, 2, H // 2, 2, C) # N, 12, 2, 12, 2, 1024 |
| 313 | .permute(0, 1, 3, 2, 4, 5) # N, 12, 12, 2, 2, 1024 |
| 314 | .reshape(N, -1, 4 * C) # N, 144, 4096 |
| 315 | .reshape( |
| 316 | num_images, h_crop, w_crop, H // 2, H // 2, -1 |
| 317 | ) # n_img, h_crop, w_crop, 12, 12, 4096 |
| 318 | .permute(0, 1, 3, 2, 4, 5) # n_img, h_crop, 12, w_crop, 12, 4096 |
| 319 | .reshape( |
| 320 | num_images, h_crop * H // 2, w_crop * H // 2, 4 * C |
| 321 | ) # n_img, h_crop*12, w_crop*12, 4096 |
| 322 | ) |
| 323 | return image_features_hd |
| 324 | |
| 325 | def add_image_newline(self, image_features_hd): |
| 326 | """image_features_hd: (num_images, h_crop*12, w_crop*12, 4096) |
| 327 | output: (num_images, (h_crop*12) * (w_crop*12+1), 4096) |
| 328 | """ |
| 329 | num_images, h, w, hid_dim = image_features_hd.shape |
| 330 | # add the newline token to the HD image feature patches |
| 331 | newline_embeddings = self.sub_GN.expand( |
| 332 | num_images, h, -1, -1 |
| 333 | ) # (n_img, h, 1, hid_dim) |
| 334 | image_features_hd_newline = torch.cat( |
| 335 | [image_features_hd, newline_embeddings], dim=2 |
| 336 | ).reshape(num_images, -1, hid_dim) |
| 337 | return image_features_hd_newline |
| 338 | |
| 339 | |
| 340 | class Phi3VProcessingInfo(BaseProcessingInfo): |
| 341 | def get_supported_mm_limits(self) -> Mapping[str, int | None]: |
| 342 | return {"image": None} |
| 343 | |
| 344 | def get_num_image_tokens( |
| 345 | self, |
| 346 | *, |
| 347 | image_width: int, |
| 348 | image_height: int, |
| 349 | processor: ProcessorMixin, |
| 350 | ) -> int: |
| 351 | return processor.calc_num_image_tokens_from_image_size( # type: ignore |
| 352 | width=image_width, |
| 353 | height=image_height, |
| 354 | ) |
| 355 | |
| 356 | def get_image_size_with_most_features(self) -> ImageSize: |
| 357 | # Result in the max possible feature size (h:w = 16:1) |
| 358 | return ImageSize(height=8000, width=50) |
| 359 | |
| 360 | |
| 361 | class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]): |
| 362 | def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: |
| 363 | num_images = mm_counts.get("image", 0) |
| 364 | |
| 365 | hf_processor = self.info.get_hf_processor() |
| 366 | image_tokens: list[str] = hf_processor.img_tokens # type: ignore |
| 367 | |
| 368 | return "".join(image_tokens[:num_images]) |
| 369 | |
| 370 | def get_dummy_mm_data( |
| 371 | self, |
| 372 | seq_len: int, |
| 373 | mm_counts: Mapping[str, int], |
| 374 | mm_options: MultiModalDummyOptions, |
| 375 | ) -> MultiModalDataDict: |
| 376 | target_width, target_height = self.info.get_image_size_with_most_features() |
| 377 | |
| 378 | return { |
| 379 | "image": self._get_dummy_images( |
| 380 | width=target_width, |
| 381 | height=target_height, |
| 382 | num_images=mm_counts.get("image", 0), |
| 383 | overrides=mm_options.get("image"), |
| 384 | ) |
| 385 | } |
| 386 | |
| 387 | |
| 388 | class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]): |
| 389 | def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: |
| 390 | return self.dummy_inputs.get_dummy_text(mm_counts) |
| 391 | |
| 392 | def _get_mm_fields_config( |
| 393 | self, |
| 394 | hf_inputs: BatchFeature, |
| 395 | hf_processor_mm_kwargs: Mapping[str, object], |
| 396 | ) -> Mapping[str, MultiModalFieldConfig]: |
| 397 | return dict( |
| 398 | pixel_values=MultiModalFieldConfig.batched("image"), |
| 399 | image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True), |
| 400 | image_embeds=MultiModalFieldConfig.batched("image"), |
| 401 | ) |
| 402 | |
| 403 | def _get_prompt_updates( |
| 404 | self, |
| 405 | mm_items: MultiModalDataItems, |
| 406 | hf_processor_mm_kwargs: Mapping[str, Any], |
| 407 | out_mm_kwargs: MultiModalKwargsItems, |
| 408 | ) -> Sequence[PromptUpdate]: |
| 409 | hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) |
| 410 | tokenizer = self.info.get_tokenizer() |
| 411 | image_tokens: list[str] = hf_processor.img_tokens # type: ignore |
| 412 | |
| 413 | def get_image_token_ids(item_idx: int) -> list[int]: |
| 414 | return cached_encode( |
| 415 | tokenizer, image_tokens[item_idx], add_special_tokens=False |
| 416 | ) |
| 417 | |
| 418 | def get_replacement_phi3v(item_idx: int): |
| 419 | images = mm_items.get_items( |
| 420 | "image", (ImageEmbeddingItems, ImageProcessorItems) |
| 421 | ) |
| 422 | |
| 423 | if isinstance(images, ImageEmbeddingItems): |
| 424 | num_image_tokens = images.get_feature_size(item_idx) |
| 425 | else: |
| 426 | assert isinstance(images, ImageProcessorItems) |
| 427 | image_size = images.get_image_size(item_idx) |
| 428 | num_image_tokens = self.info.get_num_image_tokens( |
| 429 | image_width=image_size.width, |
| 430 | image_height=image_size.height, |
| 431 | processor=hf_processor, |
| 432 | ) |
| 433 | |
| 434 | return [_IMAGE_TOKEN_ID] * num_image_tokens |
| 435 | |
| 436 | return [ |
| 437 | PromptReplacement( |
| 438 | modality="image", |
| 439 | target=get_image_token_ids, |
| 440 | replacement=get_replacement_phi3v, |
| 441 | ) |
| 442 | ] |
| 443 | |
| 444 | def _recompute_cached_prompt_update( |
| 445 | self, |
| 446 | cached_update: ResolvedPromptUpdate, |
| 447 | new_item_idx: int, |
| 448 | ) -> ResolvedPromptUpdate: |
| 449 | new_update = super()._recompute_cached_prompt_update( |
| 450 | cached_update, |
| 451 | new_item_idx, |
| 452 | ) |
| 453 | |
| 454 | if cached_update.modality == "image": |
| 455 | hf_processor = self.info.get_hf_processor() |
| 456 | tokenizer = self.info.get_tokenizer() |
| 457 | image_tokens: list[str] = hf_processor.img_tokens # type: ignore |
| 458 | new_update = new_update.with_target( |
| 459 | cached_encode( |
| 460 | tokenizer, image_tokens[new_item_idx], add_special_tokens=False |
| 461 | ) |
| 462 | ) |
| 463 | |
| 464 | return new_update |
| 465 | |
| 466 | def _apply_prompt_updates( |
| 467 | self, |
| 468 | token_ids: list[int], |
| 469 | mm_prompt_updates: MultiModalPromptUpdates, |
| 470 | ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: |
| 471 | # align to hf behavior when there are images |
| 472 | if len(mm_prompt_updates): |
| 473 | tokenizer = self.info.get_tokenizer() |
| 474 | # to decode token_ids to the original text, we need to |
| 475 | # 1. remove the first bos token |
| 476 | # 2. remove space after each special token |
| 477 | # introduced by the tokenizer |
| 478 | if len(token_ids) and token_ids[0] == tokenizer.bos_token_id: |
| 479 | token_ids = token_ids[1:] |
| 480 | text = tokenizer.decode(token_ids) |
| 481 | for special_token in tokenizer.all_special_tokens: |
| 482 | text = text.replace(f"{special_token} ", special_token) |
| 483 | # perform hf behavior |
| 484 | # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/64f88b6/processing_phi3_v.py#L407 |
| 485 | pattern = r"<\|image_\d+\|>" |
| 486 | prompt_chunks = [ |
| 487 | tokenizer(chunk).input_ids for chunk in re.split(pattern, text) |
| 488 | ] |
| 489 | image_tags = [ |
| 490 | tokenizer(chunk, add_special_tokens=False).input_ids |
| 491 | for chunk in re.findall(pattern, text) |
| 492 | ] |
| 493 | if len(prompt_chunks) > len(image_tags): |
| 494 | image_tags.append([]) |
| 495 | token_ids = [ |
| 496 | e |
| 497 | for sublist in zip(prompt_chunks, image_tags) |
| 498 | for ele in sublist |
| 499 | for e in ele |
| 500 | ] |
| 501 | |
| 502 | token_ids, placeholders = super()._apply_prompt_updates( |
| 503 | token_ids=token_ids, |
| 504 | mm_prompt_updates=mm_prompt_updates, |
| 505 | ) |
| 506 | |
| 507 | # Keep the behavior in line with HF processor |
| 508 | if len(mm_prompt_updates) and ( |
| 509 | token_ids[:2] |
| 510 | == cached_encode(tokenizer, "<s> <|image|>", add_special_tokens=False) |
| 511 | ): |
| 512 | token_ids = [token_ids[0], *token_ids[2:]] |
| 513 | placeholders = { |
| 514 | modality: [ |
| 515 | PlaceholderFeaturesInfo( |
| 516 | modality=p.modality, |
| 517 | item_idx=p.item_idx, |
| 518 | start_idx=p.start_idx - 1, |
| 519 | tokens=p.tokens, |
| 520 | is_embed=p.is_embed, |
| 521 | ) |
| 522 | for p in ps |
| 523 | ] |
| 524 | for modality, ps in placeholders.items() |
| 525 | } |
| 526 | |
| 527 | return token_ids, placeholders |
| 528 | |
| 529 | |
| 530 | @MULTIMODAL_REGISTRY.register_processor( |
| 531 | Phi3VMultiModalProcessor, |
| 532 | info=Phi3VProcessingInfo, |
| 533 | dummy_inputs=Phi3VDummyInputsBuilder, |
| 534 | ) |
| 535 | class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant): |
| 536 | hf_to_vllm_mapper = WeightsMapper( |
| 537 | orig_to_new_prefix={ |
| 538 | "model.vision_embed_tokens.wte": "embed_tokens", |
| 539 | "model.vision_embed_tokens.": "vision_embed_tokens.", |
| 540 | "lm_head.": "language_model.lm_head.", |
| 541 | "model.": "language_model.model.", |
| 542 | } |
| 543 | ) |
| 544 | |
| 545 | @classmethod |
| 546 | def get_placeholder_str(cls, modality: str, i: int) -> str | None: |
| 547 | if modality.startswith("image"): |
| 548 | return f"<|image_{i}|>" |
| 549 | |
| 550 | raise ValueError("Only image modality is supported") |
| 551 | |
| 552 | def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): |
| 553 | super().__init__() |
| 554 | config = vllm_config.model_config.hf_config |
| 555 | quant_config = vllm_config.quant_config |
| 556 | multimodal_config = vllm_config.model_config.multimodal_config |
| 557 | self.config = config |
| 558 | self.multimodal_config = multimodal_config |
| 559 | self.image_token_id = _IMAGE_TOKEN_ID |
| 560 | |
| 561 | with self._mark_tower_model(vllm_config, "image"): |
| 562 | self.embed_tokens = VocabParallelEmbedding( |
| 563 | config.vocab_size, |
| 564 | config.hidden_size, |
| 565 | quant_config=quant_config, |
| 566 | prefix=maybe_prefix(prefix, "model.embed_tokens"), |
| 567 | ) |
| 568 | self.vision_embed_tokens = Phi3HDImageEmbedding( |
| 569 | config, |
| 570 | quant_config=quant_config, |
| 571 | prefix=maybe_prefix(prefix, "model.vision_embed_tokens"), |
| 572 | ) |
| 573 | |
| 574 | with self._mark_language_model(vllm_config): |
| 575 | self.language_model = init_vllm_registered_model( |
| 576 | vllm_config=vllm_config, |
| 577 | # The prefix is empty intentionally because default prefix of |
| 578 | # LlamaForCausalLM is "model" |
| 579 | prefix="", |
| 580 | # We don't directly initialize vLLM's LlamaForCausalLM so we |
| 581 | # can automatically apply embedding wrapper if this model is |
| 582 | # initialized as an embedding model |
| 583 | architectures=["LlamaForCausalLM"], |
| 584 | ) |
| 585 | |
| 586 | self.make_empty_intermediate_tensors = ( |
| 587 | self.language_model.make_empty_intermediate_tensors |
| 588 | ) |
| 589 | |
| 590 | def _parse_and_validate_image_input( |
| 591 | self, **kwargs: object |
| 592 | ) -> Phi3VImageInputs | None: |
| 593 | pixel_values = kwargs.pop("pixel_values", None) |
| 594 | image_sizes = kwargs.pop("image_sizes", None) |
| 595 | image_embeds = kwargs.pop("image_embeds", None) |
| 596 | |
| 597 | if pixel_values is None and image_embeds is None: |
| 598 | return None |
| 599 | |
| 600 | if pixel_values is not None: |
| 601 | return Phi3VImagePixelInputs( |
| 602 | type="pixel_values", |
| 603 | pixel_values=pixel_values, |
| 604 | image_sizes=image_sizes, |
| 605 | resolve_bindings={ |
| 606 | "h": CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size, |
| 607 | "w": CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size, |
| 608 | }, |
| 609 | ) |
| 610 | |
| 611 | if image_embeds is not None: |
| 612 | return Phi3VImageEmbeddingInputs( |
| 613 | type="image_embeds", |
| 614 | data=image_embeds, |
| 615 | ) |
| 616 | |
| 617 | raise AssertionError("This line should be unreachable.") |
| 618 | |
| 619 | def _process_image_input( |
| 620 | self, |
| 621 | image_input: Phi3VImageInputs, |
| 622 | ) -> torch.Tensor: |
| 623 | if image_input["type"] == "image_embeds": |
| 624 | return image_input["data"] |
| 625 | |
| 626 | image_embeds = self.vision_embed_tokens( |
| 627 | image_input["pixel_values"], image_input["image_sizes"] |
| 628 | ) |
| 629 | |
| 630 | return image_embeds |
| 631 | |
| 632 | def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: |
| 633 | image_input = self._parse_and_validate_image_input(**kwargs) |
| 634 | if image_input is None: |
| 635 | return [] |
| 636 | vision_embeddings = self._process_image_input(image_input) |
| 637 | return vision_embeddings |
| 638 | |
| 639 | def embed_input_ids( |
| 640 | self, |
| 641 | input_ids: torch.Tensor, |
| 642 | multimodal_embeddings: MultiModalEmbeddings | None = None, |
| 643 | *, |
| 644 | is_multimodal: torch.Tensor | None = None, |
| 645 | ) -> torch.Tensor: |
| 646 | inputs_embeds = self._embed_text_input_ids( |
| 647 | input_ids, |
| 648 | self.embed_tokens, |
| 649 | is_multimodal=is_multimodal, |
| 650 | ) |
| 651 | |
| 652 | if multimodal_embeddings is None or len(multimodal_embeddings) == 0: |
| 653 | return inputs_embeds |
| 654 | |
| 655 | return _merge_multimodal_embeddings( |
| 656 | inputs_embeds=inputs_embeds, |
| 657 | multimodal_embeddings=multimodal_embeddings, |
| 658 | is_multimodal=_require_is_multimodal(is_multimodal), |
| 659 | ) |
| 660 | |
| 661 | def forward( |
| 662 | self, |
| 663 | input_ids: torch.Tensor | None, |
| 664 | positions: torch.Tensor, |
| 665 | intermediate_tensors: IntermediateTensors | None = None, |
| 666 | inputs_embeds: torch.Tensor | None = None, |
| 667 | **kwargs: object, |
| 668 | ): |
| 669 | if intermediate_tensors is not None: |
| 670 | inputs_embeds = None |
| 671 | |
| 672 | hidden_states = self.language_model.model( |
| 673 | input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds |
| 674 | ) |
| 675 | |
| 676 | return hidden_states |
| 677 | |
| 678 | def compute_logits( |
| 679 | self, |
| 680 | hidden_states: torch.Tensor, |
| 681 | ) -> torch.Tensor | None: |
| 682 | return self.language_model.compute_logits(hidden_states) |
| 683 | |
| 684 | def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: |
| 685 | loader = AutoWeightsLoader(self) |
| 686 | autoloaded_weights = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) |
| 687 | |
| 688 | # The HF config doesn't specify whether these are tied, |
| 689 | # so we detect it this way |
| 690 | if "embed_tokens.weight" not in autoloaded_weights: |
| 691 | self.embed_tokens = self.language_model.model.embed_tokens |
| 692 | autoloaded_weights.add("embed_tokens.weight") |
| 693 | return autoloaded_weights |