| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # adapted from https://huggingface.co/nvidia/NVLM-D-72B/blob/main/modeling_nvlm_d.py |
| 5 | # -------------------------------------------------------- |
| 6 | # NVLM-D |
| 7 | # Copyright (c) 2024 NVIDIA |
| 8 | # Licensed under Apache 2.0 License [see LICENSE for details] |
| 9 | # -------------------------------------------------------- |
| 10 | from collections.abc import Mapping |
| 11 | |
| 12 | import torch |
| 13 | import torch.nn as nn |
| 14 | from transformers import PreTrainedConfig |
| 15 | |
| 16 | from vllm.config.multimodal import MultiModalDummyOptions |
| 17 | from vllm.inputs import MultiModalDataDict |
| 18 | from vllm.model_executor.layers.quantization import QuantizationConfig |
| 19 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 20 | from vllm.multimodal.inputs import BatchedTensorInputs |
| 21 | from vllm.multimodal.parse import ( |
| 22 | ImageEmbeddingItems, |
| 23 | ImageProcessorItems, |
| 24 | MultiModalDataItems, |
| 25 | ) |
| 26 | from vllm.multimodal.processing import ( |
| 27 | PromptReplacement, |
| 28 | PromptUpdateDetails, |
| 29 | cached_encode, |
| 30 | ) |
| 31 | from vllm.transformers_utils.processors.internvl import InternVLImageProcessor |
| 32 | from vllm.transformers_utils.processors.nvlm_d import NVLMProcessor |
| 33 | |
| 34 | from .intern_vit import InternVisionModel |
| 35 | from .internvl import ( |
| 36 | BaseInternVLDummyInputsBuilder, |
| 37 | BaseInternVLMultiModalProcessor, |
| 38 | BaseInternVLProcessingInfo, |
| 39 | InternVLChatModel, |
| 40 | ) |
| 41 | |
| 42 | |
| 43 | class NVLMProcessingInfo(BaseInternVLProcessingInfo): |
| 44 | def get_image_processor(self, **kwargs): |
| 45 | config = self.get_hf_config() |
| 46 | vision_config = config.vision_config |
| 47 | |
| 48 | kwargs = self.ctx.get_merged_mm_kwargs(kwargs) |
| 49 | kwargs.setdefault("image_size", vision_config.image_size) |
| 50 | kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch) |
| 51 | kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch) |
| 52 | kwargs.setdefault("dynamic_image_size", config.dynamic_image_size) |
| 53 | kwargs.setdefault("use_thumbnail", config.use_thumbnail) |
| 54 | |
| 55 | return InternVLImageProcessor(**kwargs) |
| 56 | |
| 57 | def get_hf_processor(self, **kwargs: object) -> NVLMProcessor: |
| 58 | config = self.get_hf_config() |
| 59 | vision_config = config.vision_config |
| 60 | |
| 61 | image_processor = self.get_image_processor(**kwargs) |
| 62 | image_size = image_processor.image_size |
| 63 | patch_size = vision_config.patch_size |
| 64 | downsample_ratio = config.downsample_ratio |
| 65 | image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2)) |
| 66 | |
| 67 | return NVLMProcessor( |
| 68 | tokenizer=self.get_tokenizer(), |
| 69 | image_processor=image_processor, |
| 70 | image_seq_length=image_seq_length, |
| 71 | ) |
| 72 | |
| 73 | |
| 74 | class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo]): |
| 75 | def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: |
| 76 | num_images = mm_counts.get("image", 0) |
| 77 | |
| 78 | # The newline is necessary to separate ">" of the current item |
| 79 | # and "<" of the next item |
| 80 | return "<image>\n" * num_images |
| 81 | |
| 82 | def get_dummy_mm_data( |
| 83 | self, |
| 84 | seq_len: int, |
| 85 | mm_counts: Mapping[str, int], |
| 86 | mm_options: MultiModalDummyOptions, |
| 87 | ) -> MultiModalDataDict: |
| 88 | target_width, target_height = self.info.get_image_size_with_most_features() |
| 89 | |
| 90 | return { |
| 91 | "image": self._get_dummy_images( |
| 92 | width=target_width, |
| 93 | height=target_height, |
| 94 | num_images=mm_counts.get("image", 0), |
| 95 | overrides=mm_options.get("image"), |
| 96 | ) |
| 97 | } |
| 98 | |
| 99 | |
| 100 | class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo]): |
| 101 | def _get_prompt_repl_image( |
| 102 | self, |
| 103 | mm_items: MultiModalDataItems, |
| 104 | hf_processor: NVLMProcessor, |
| 105 | out_mm_data: BatchedTensorInputs, |
| 106 | ): |
| 107 | tokenizer = self.info.get_tokenizer() |
| 108 | |
| 109 | if "image_num_patches" in out_mm_data: |
| 110 | image_num_patches = out_mm_data["image_num_patches"] |
| 111 | assert isinstance(image_num_patches, torch.Tensor) |
| 112 | image_num_patches = image_num_patches.tolist() |
| 113 | elif "image_embeds" in out_mm_data: |
| 114 | # TODO: Use image size information in dictionary embedding inputs |
| 115 | # to compute num_patches (similar to Qwen2-VL) |
| 116 | image_num_patches = [None] * len(out_mm_data["image_embeds"]) |
| 117 | else: |
| 118 | image_num_patches = [] |
| 119 | |
| 120 | def get_replacement_nvlm(item_idx: int): |
| 121 | images = mm_items.get_items( |
| 122 | "image", (ImageEmbeddingItems, ImageProcessorItems) |
| 123 | ) |
| 124 | |
| 125 | if isinstance(images, ImageEmbeddingItems): |
| 126 | feature_size = images.get_feature_size(item_idx) |
| 127 | else: |
| 128 | assert isinstance(images, ImageProcessorItems) |
| 129 | image_size = images.get_image_size(item_idx) |
| 130 | feature_size = self.info.get_num_image_tokens( |
| 131 | image_width=image_size.width, |
| 132 | image_height=image_size.height, |
| 133 | processor=hf_processor, |
| 134 | ) |
| 135 | |
| 136 | num_patches = image_num_patches[item_idx] |
| 137 | if num_patches is not None: |
| 138 | assert isinstance(num_patches, int) |
| 139 | |
| 140 | repl = hf_processor.get_image_repl(num_patches, num_features=feature_size) |
| 141 | |
| 142 | # `repl.full` ends in the special `</Image>` token, so there is |
| 143 | # no BPE merge across the boundary with the newline |
| 144 | newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False) |
| 145 | |
| 146 | return PromptUpdateDetails.select_token_id( |
| 147 | repl.full + newline_ids, hf_processor.ctx_image_token_id |
| 148 | ) |
| 149 | |
| 150 | # See note in dummy data regarding why we have the extra newline |
| 151 | return PromptReplacement( |
| 152 | modality="image", |
| 153 | target=cached_encode(tokenizer, "<image>\n", add_special_tokens=False), |
| 154 | replacement=get_replacement_nvlm, |
| 155 | ) |
| 156 | |
| 157 | |
| 158 | @MULTIMODAL_REGISTRY.register_processor( |
| 159 | NVLMMultiModalProcessor, |
| 160 | info=NVLMProcessingInfo, |
| 161 | dummy_inputs=NVLMDummyInputsBuilder, |
| 162 | ) |
| 163 | class NVLM_D_Model(InternVLChatModel): |
| 164 | def _init_mlp1(self, config: PreTrainedConfig) -> nn.Module: |
| 165 | vit_hidden_size = config.vision_config.hidden_size |
| 166 | llm_intermediate_size = config.text_config.intermediate_size |
| 167 | llm_hidden_size = config.text_config.hidden_size |
| 168 | |
| 169 | return nn.Sequential( |
| 170 | nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio) ** 2), |
| 171 | nn.Linear( |
| 172 | vit_hidden_size * int(1 / self.downsample_ratio) ** 2, |
| 173 | llm_intermediate_size, |
| 174 | bias=False, |
| 175 | ), |
| 176 | nn.GELU(), |
| 177 | nn.Linear(llm_intermediate_size, llm_hidden_size, bias=False), |
| 178 | ) |
| 179 | |
| 180 | def _init_vision_model( |
| 181 | self, |
| 182 | config: PreTrainedConfig, |
| 183 | quant_config: QuantizationConfig | None, |
| 184 | *, |
| 185 | prefix: str, |
| 186 | ): |
| 187 | vision_feature_layer = config.select_layer |
| 188 | if vision_feature_layer < 0: |
| 189 | num_hidden_layers = ( |
| 190 | config.vision_config.num_hidden_layers + vision_feature_layer + 1 |
| 191 | ) |
| 192 | else: |
| 193 | num_hidden_layers = vision_feature_layer + 1 |
| 194 | |
| 195 | # We added additional dummy heads to the original num of heads to |
| 196 | # make the number of heads divisible by 8. |
| 197 | return InternVisionModel( |
| 198 | config.vision_config, |
| 199 | quant_config=quant_config, |
| 200 | num_hidden_layers_override=num_hidden_layers, |
| 201 | num_dummy_heads=7, |
| 202 | prefix=prefix, |
| 203 | ) |