| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for H2OVL's multimodal preprocessing kwargs.""" |
| 4 | |
| 5 | from collections.abc import Mapping |
| 6 | |
| 7 | import pytest |
| 8 | from PIL import Image |
| 9 | from transformers import PreTrainedConfig |
| 10 | |
| 11 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 12 | from vllm.multimodal.image import rescale_image_size |
| 13 | from vllm.multimodal.processing import BaseMultiModalProcessor |
| 14 | |
| 15 | from ....conftest import ImageTestAssets |
| 16 | from ...utils import build_model_context |
| 17 | |
| 18 | |
| 19 | def _get_expected_num_patches( |
| 20 | config: PreTrainedConfig, |
| 21 | image: Image.Image, |
| 22 | num_imgs: int, |
| 23 | min_num: int, |
| 24 | max_num: int, |
| 25 | ): |
| 26 | from vllm.transformers_utils.processors.h2ovl import ( |
| 27 | calculate_h2ovl_targets, |
| 28 | get_h2ovl_target_ratios, |
| 29 | ) |
| 30 | |
| 31 | width, height = image.size |
| 32 | |
| 33 | # Calculate the expected number of blocks |
| 34 | if num_imgs == 1 and config.use_msac: |
| 35 | # First pass |
| 36 | blocks1, _, _, aspect_ratio = calculate_h2ovl_targets( |
| 37 | orig_width=width, |
| 38 | orig_height=height, |
| 39 | target_ratios=get_h2ovl_target_ratios( |
| 40 | min_num=1, |
| 41 | max_num=max_num, |
| 42 | prior_aspect_ratio=None, |
| 43 | ), |
| 44 | image_size=config.vision_config.image_size, |
| 45 | use_thumbnail=False, # Thumbnail is handled separately |
| 46 | ) |
| 47 | |
| 48 | # Second pass |
| 49 | blocks2, _, _, _ = calculate_h2ovl_targets( |
| 50 | orig_width=width, |
| 51 | orig_height=height, |
| 52 | target_ratios=get_h2ovl_target_ratios( |
| 53 | min_num=3, |
| 54 | max_num=max_num, |
| 55 | prior_aspect_ratio=aspect_ratio, |
| 56 | ), |
| 57 | image_size=config.vision_config.image_size, |
| 58 | use_thumbnail=False, |
| 59 | ) |
| 60 | |
| 61 | # Add thumbnail if use_thumbnail is True and total_blocks > 1 |
| 62 | if config.use_thumbnail: |
| 63 | blocks1 += 1 if blocks1 > 1 else 0 |
| 64 | blocks2 += 1 if blocks2 > 1 else 0 |
| 65 | |
| 66 | # Total blocks is the sum of blocks from both passes minus |
| 67 | # overlapping |
| 68 | total_blocks = blocks1 + blocks2 - 1 |
| 69 | |
| 70 | return total_blocks |
| 71 | |
| 72 | blocks, _, _, _ = calculate_h2ovl_targets( |
| 73 | orig_width=width, |
| 74 | orig_height=height, |
| 75 | target_ratios=get_h2ovl_target_ratios( |
| 76 | min_num, |
| 77 | max_num, |
| 78 | prior_aspect_ratio=None, |
| 79 | ), |
| 80 | image_size=config.vision_config.image_size, |
| 81 | use_thumbnail=False, |
| 82 | ) |
| 83 | expected_num_patches = blocks |
| 84 | |
| 85 | if config.use_thumbnail and expected_num_patches > 1: |
| 86 | expected_num_patches += 1 |
| 87 | |
| 88 | return expected_num_patches |
| 89 | |
| 90 | |
| 91 | def _run_check( |
| 92 | processor: BaseMultiModalProcessor, |
| 93 | images: list[Image.Image], |
| 94 | min_num: int, |
| 95 | max_num: int, |
| 96 | mm_processor_kwargs: Mapping[str, object], |
| 97 | ): |
| 98 | tokenizer = processor.info.get_tokenizer() |
| 99 | config = processor.info.get_hf_config() |
| 100 | |
| 101 | prompt = "<image>" * len(images) |
| 102 | mm_data = {"image": images} |
| 103 | |
| 104 | total_expected_num_patches = sum( |
| 105 | _get_expected_num_patches(config, image, len(images), min_num, max_num) |
| 106 | for image in images |
| 107 | ) |
| 108 | |
| 109 | processed_inputs = processor( |
| 110 | prompt, |
| 111 | mm_items=processor.info.parse_mm_data(mm_data), |
| 112 | hf_processor_mm_kwargs=mm_processor_kwargs, |
| 113 | ) |
| 114 | |
| 115 | # Ensure we have the right number of placeholders per num_crops size |
| 116 | image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>") |
| 117 | img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) |
| 118 | pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape |
| 119 | |
| 120 | assert img_tok_count == 256 * total_expected_num_patches |
| 121 | assert pixel_shape[0] == total_expected_num_patches |
| 122 | |
| 123 | |
| 124 | @pytest.mark.parametrize( |
| 125 | "model_id", |
| 126 | [ |
| 127 | "h2oai/h2ovl-mississippi-800m", |
| 128 | "h2oai/h2ovl-mississippi-2b", |
| 129 | ], |
| 130 | ) |
| 131 | @pytest.mark.parametrize( |
| 132 | "size_factors", |
| 133 | [ |
| 134 | # Single-scale |
| 135 | [1.0], |
| 136 | # Single-scale, batched |
| 137 | [1.0, 1.0, 1.0], |
| 138 | # Multi-scale |
| 139 | [0.25, 0.5, 1.0], |
| 140 | [4.0, 2.0, 1.0], |
| 141 | ], |
| 142 | ) |
| 143 | @pytest.mark.parametrize( |
| 144 | ("min_dynamic_patch", "max_dynamic_patch"), |
| 145 | [(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)], |
| 146 | ) |
| 147 | @pytest.mark.parametrize("dynamic_image_size", [True, False]) |
| 148 | @pytest.mark.parametrize("kwargs_on_init", [True, False]) |
| 149 | def test_processor_override( |
| 150 | model_id: str, |
| 151 | image_assets: ImageTestAssets, |
| 152 | size_factors: list[int], |
| 153 | min_dynamic_patch: int, |
| 154 | max_dynamic_patch: int, |
| 155 | dynamic_image_size: bool | None, |
| 156 | kwargs_on_init: bool, |
| 157 | ): |
| 158 | mm_processor_kwargs = { |
| 159 | "min_dynamic_patch": min_dynamic_patch, |
| 160 | "max_dynamic_patch": max_dynamic_patch, |
| 161 | "dynamic_image_size": dynamic_image_size, |
| 162 | } |
| 163 | |
| 164 | ctx = build_model_context( |
| 165 | model_id, |
| 166 | mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None, |
| 167 | limit_mm_per_prompt={"image": len(size_factors)}, |
| 168 | ) |
| 169 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 170 | hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs |
| 171 | |
| 172 | min_num = min_dynamic_patch if dynamic_image_size else 1 |
| 173 | max_num = max_dynamic_patch if dynamic_image_size else 1 |
| 174 | |
| 175 | _run_check( |
| 176 | processor, |
| 177 | [rescale_image_size(image_assets[0].pil_image, f) for f in size_factors], |
| 178 | min_num, |
| 179 | max_num, |
| 180 | hf_processor_mm_kwargs, |
| 181 | ) |