| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # adapted from https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/modeling_h2ovl_chat.py |
| 5 | # https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/image_process.py |
| 6 | # -------------------------------------------------------- |
| 7 | # H2OVL-Mississippi |
| 8 | # Copyright (c) 2024 H2O.AI |
| 9 | # Licensed under Apache 2.0 License [see LICENSE for details] |
| 10 | # -------------------------------------------------------- |
| 11 | |
| 12 | import torch |
| 13 | from transformers import PreTrainedConfig |
| 14 | |
| 15 | from vllm.model_executor.layers.quantization import QuantizationConfig |
| 16 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 17 | from vllm.multimodal.inputs import BatchedTensorInputs, MultiModalKwargsItem |
| 18 | from vllm.multimodal.parse import ( |
| 19 | ImageEmbeddingItems, |
| 20 | ImageProcessorItems, |
| 21 | MultiModalDataItems, |
| 22 | ) |
| 23 | from vllm.multimodal.processing.processor import ( |
| 24 | MultiModalProcessingResult, |
| 25 | ProcessorInputs, |
| 26 | PromptReplacement, |
| 27 | TimingContext, |
| 28 | cached_encode, |
| 29 | ) |
| 30 | from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor |
| 31 | |
| 32 | from .intern_vit import InternVisionModel |
| 33 | from .internvl import ( |
| 34 | BaseInternVLDummyInputsBuilder, |
| 35 | BaseInternVLMultiModalProcessor, |
| 36 | BaseInternVLProcessingInfo, |
| 37 | InternVLChatModel, |
| 38 | ) |
| 39 | |
| 40 | |
| 41 | class H2OVLProcessingInfo(BaseInternVLProcessingInfo): |
| 42 | def get_image_processor(self, **kwargs): |
| 43 | config = self.get_hf_config() |
| 44 | vision_config = config.vision_config |
| 45 | |
| 46 | kwargs = self.ctx.get_merged_mm_kwargs(kwargs) |
| 47 | kwargs.setdefault("image_size", vision_config.image_size) |
| 48 | kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch) |
| 49 | kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch) |
| 50 | kwargs.setdefault("dynamic_image_size", config.dynamic_image_size) |
| 51 | kwargs.setdefault("use_thumbnail", config.use_thumbnail) |
| 52 | kwargs.setdefault("use_msac", config.use_msac) |
| 53 | |
| 54 | return H2OVLImageProcessor(**kwargs) |
| 55 | |
| 56 | def get_hf_processor(self, **kwargs: object) -> H2OVLProcessor: |
| 57 | config = self.get_hf_config() |
| 58 | vision_config = config.vision_config |
| 59 | |
| 60 | image_processor = self.get_image_processor(**kwargs) |
| 61 | image_size = image_processor.image_size |
| 62 | patch_size = vision_config.patch_size |
| 63 | downsample_ratio = config.downsample_ratio |
| 64 | image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2)) |
| 65 | |
| 66 | return H2OVLProcessor( |
| 67 | tokenizer=self.get_tokenizer(), |
| 68 | image_processor=image_processor, |
| 69 | image_seq_length=image_seq_length, |
| 70 | ) |
| 71 | |
| 72 | def get_num_image_tokens( |
| 73 | self, |
| 74 | *, |
| 75 | image_width: int, |
| 76 | image_height: int, |
| 77 | processor: H2OVLProcessor, |
| 78 | use_msac: bool | None = None, |
| 79 | ) -> int: |
| 80 | return processor.get_num_image_tokens( |
| 81 | image_width=image_width, |
| 82 | image_height=image_height, |
| 83 | use_msac=use_msac, |
| 84 | ) |
| 85 | |
| 86 | |
| 87 | class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingInfo]): |
| 88 | def _get_prompt_repl_image( |
| 89 | self, |
| 90 | mm_items: MultiModalDataItems, |
| 91 | hf_processor: H2OVLProcessor, |
| 92 | out_mm_data: BatchedTensorInputs, |
| 93 | ): |
| 94 | tokenizer = self.info.get_tokenizer() |
| 95 | |
| 96 | if "image_num_patches" in out_mm_data: |
| 97 | image_num_patches = out_mm_data["image_num_patches"] |
| 98 | assert isinstance(image_num_patches, torch.Tensor) |
| 99 | image_num_patches = image_num_patches.tolist() |
| 100 | elif "image_embeds" in out_mm_data: |
| 101 | # TODO: Use image size information in dictionary embedding inputs |
| 102 | # to compute num_patches (similar to Qwen2-VL) |
| 103 | image_num_patches = [None] * len(out_mm_data["image_embeds"]) |
| 104 | else: |
| 105 | image_num_patches = [] |
| 106 | |
| 107 | num_images = len(image_num_patches) |
| 108 | |
| 109 | def get_replacement_internvl(item_idx: int): |
| 110 | images = mm_items.get_items( |
| 111 | "image", (ImageEmbeddingItems, ImageProcessorItems) |
| 112 | ) |
| 113 | |
| 114 | if isinstance(images, ImageEmbeddingItems): |
| 115 | feature_size = images.get_feature_size(item_idx) |
| 116 | else: |
| 117 | assert isinstance(images, ImageProcessorItems) |
| 118 | image_size = images.get_image_size(item_idx) |
| 119 | feature_size = self.info.get_num_image_tokens( |
| 120 | image_width=image_size.width, |
| 121 | image_height=image_size.height, |
| 122 | processor=hf_processor, |
| 123 | use_msac=None if num_images == 1 else False, |
| 124 | ) |
| 125 | |
| 126 | num_patches = image_num_patches[item_idx] |
| 127 | if num_patches is not None: |
| 128 | assert isinstance(num_patches, int) |
| 129 | |
| 130 | return hf_processor.get_image_repl(num_patches, num_features=feature_size) |
| 131 | |
| 132 | return PromptReplacement( |
| 133 | modality="image", |
| 134 | target=cached_encode(tokenizer, "<image>", add_special_tokens=False), |
| 135 | replacement=get_replacement_internvl, |
| 136 | ) |
| 137 | |
| 138 | def _cached_apply_hf_processor( |
| 139 | self, |
| 140 | inputs: ProcessorInputs, |
| 141 | timing_ctx: TimingContext, |
| 142 | ) -> MultiModalProcessingResult: |
| 143 | # The processor logic is different for len(images) <= 1 vs > 1 |
| 144 | # Since the processing cache assumes that the processor output is |
| 145 | # invariant of how many images are passed per prompt, we only |
| 146 | # perform caching for the most common case |
| 147 | if inputs.mm_data_items.get_count("image", strict=False) > 1: |
| 148 | return self._apply_hf_processor(inputs, timing_ctx) |
| 149 | |
| 150 | return super()._cached_apply_hf_processor(inputs, timing_ctx) |
| 151 | |
| 152 | |
| 153 | @MULTIMODAL_REGISTRY.register_processor( |
| 154 | H2OVLMultiModalProcessor, |
| 155 | info=H2OVLProcessingInfo, |
| 156 | dummy_inputs=BaseInternVLDummyInputsBuilder, |
| 157 | ) |
| 158 | class H2OVLChatModel(InternVLChatModel): |
| 159 | def _init_vision_model( |
| 160 | self, |
| 161 | config: PreTrainedConfig, |
| 162 | quant_config: QuantizationConfig | None, |
| 163 | *, |
| 164 | prefix: str, |
| 165 | ): |
| 166 | vision_feature_layer = config.select_layer |
| 167 | if vision_feature_layer < 0: |
| 168 | num_hidden_layers = ( |
| 169 | config.vision_config.num_hidden_layers + vision_feature_layer + 1 |
| 170 | ) |
| 171 | else: |
| 172 | num_hidden_layers = vision_feature_layer + 1 |
| 173 | |
| 174 | return InternVisionModel( |
| 175 | config.vision_config, |
| 176 | quant_config=quant_config, |
| 177 | num_hidden_layers_override=num_hidden_layers, |
| 178 | prefix=prefix, |
| 179 | ) |
| 180 | |
| 181 | def get_mm_lora_token_counts( |
| 182 | self, |
| 183 | *, |
| 184 | modality: str, |
| 185 | mm_kwargs: MultiModalKwargsItem | None, |
| 186 | num_mm_embeds: int, |
| 187 | ) -> tuple[int, int | None]: |
| 188 | del modality, mm_kwargs |
| 189 | if num_mm_embeds <= 0 or self.num_image_token <= 0: |
| 190 | return 0, 0 |
| 191 | |
| 192 | num_patches = num_mm_embeds // self.num_image_token |
| 193 | tower_tokens = num_patches * (self.patch_tokens + 1) |
| 194 | return tower_tokens, num_patches * self.num_image_token |