| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import itertools |
| 5 | from functools import partial |
| 6 | |
| 7 | import pytest |
| 8 | from PIL import Image |
| 9 | from pqdm.threads import pqdm |
| 10 | |
| 11 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 12 | from vllm.multimodal.parse import ImageSize |
| 13 | from vllm.multimodal.processing import BaseMultiModalProcessor |
| 14 | from vllm.tokenizers.hf import maybe_make_thread_pool |
| 15 | |
| 16 | from ...utils import build_model_context |
| 17 | |
| 18 | |
| 19 | def _validate_image_max_tokens_one( |
| 20 | processor: BaseMultiModalProcessor, |
| 21 | max_tokens: int, |
| 22 | failed_size_excs: list[tuple[ImageSize, Exception]], |
| 23 | image_size: ImageSize, |
| 24 | ) -> None: |
| 25 | info = processor.info |
| 26 | feature_size = info.get_num_image_tokens( |
| 27 | image_width=image_size.width, image_height=image_size.height |
| 28 | ) |
| 29 | |
| 30 | try: |
| 31 | assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}" |
| 32 | except Exception as exc: |
| 33 | failed_size_excs.append((image_size, exc)) |
| 34 | |
| 35 | |
| 36 | @pytest.mark.skip( |
| 37 | "This test takes around 5 minutes to run. Comment this out to run it manually." |
| 38 | ) |
| 39 | @pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]) |
| 40 | def test_processor_max_tokens(model_id): |
| 41 | ctx = build_model_context( |
| 42 | model_id, |
| 43 | mm_processor_kwargs=None, |
| 44 | limit_mm_per_prompt={"image": 1}, |
| 45 | ) |
| 46 | |
| 47 | # Avoid tokenizer already borrowed error |
| 48 | maybe_make_thread_pool(ctx.tokenizer) |
| 49 | |
| 50 | processor = MULTIMODAL_REGISTRY.create_processor( |
| 51 | ctx.model_config, |
| 52 | tokenizer=ctx.tokenizer, |
| 53 | ) |
| 54 | |
| 55 | info = processor.info |
| 56 | |
| 57 | seen_aspect_ratios = set[float]() |
| 58 | image_sizes = list[ImageSize]() |
| 59 | |
| 60 | # The aspect ratio of the grid layout is between 1 and 6 |
| 61 | # NOTE: Assumes that feature size calculation is the same if we |
| 62 | # swap the width and height of the image |
| 63 | for w, h in itertools.product(range(32, 4096), repeat=2): |
| 64 | aspect_ratio = w / h |
| 65 | if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios: |
| 66 | image_sizes.append(ImageSize(w, h)) |
| 67 | seen_aspect_ratios.add(aspect_ratio) |
| 68 | |
| 69 | failed_size_excs = list[tuple[ImageSize, Exception]]() |
| 70 | |
| 71 | validate_one = partial( |
| 72 | _validate_image_max_tokens_one, |
| 73 | processor, |
| 74 | info.get_max_image_tokens(), # type: ignore |
| 75 | failed_size_excs, |
| 76 | ) |
| 77 | pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes") |
| 78 | |
| 79 | if failed_size_excs: |
| 80 | msg = "Found failing image sizes:" + "\n========\n".join( |
| 81 | f"[{size}]\n{exc}" for size, exc in failed_size_excs |
| 82 | ) |
| 83 | raise AssertionError(msg) |
| 84 | |
| 85 | |
| 86 | def _validate_image_prompt_replacements_one( |
| 87 | processor: BaseMultiModalProcessor, |
| 88 | num_imgs: int, |
| 89 | failed_size_excs: list[tuple[ImageSize, Exception]], |
| 90 | image_size: ImageSize, |
| 91 | ) -> None: |
| 92 | prompt = "<image>" * num_imgs |
| 93 | image = Image.new("RGB", size=image_size) |
| 94 | mm_data = {"image": [image] * num_imgs} |
| 95 | |
| 96 | try: |
| 97 | # The processor will throw an error if there is a mismatch |
| 98 | # in the prompt replacements |
| 99 | processed_inputs = processor( |
| 100 | prompt, |
| 101 | mm_items=processor.info.parse_mm_data(mm_data), |
| 102 | hf_processor_mm_kwargs={}, |
| 103 | ) |
| 104 | |
| 105 | image_placeholders = processed_inputs["mm_placeholders"]["image"] |
| 106 | assert len(image_placeholders) == num_imgs |
| 107 | |
| 108 | first_placeholder = image_placeholders[0] |
| 109 | |
| 110 | assert first_placeholder.offset == 0 |
| 111 | assert ( |
| 112 | first_placeholder.length |
| 113 | == len(processed_inputs["prompt_token_ids"]) // num_imgs |
| 114 | ) |
| 115 | except Exception as exc: |
| 116 | failed_size_excs.append((image_size, exc)) |
| 117 | |
| 118 | |
| 119 | def _test_image_prompt_replacements( |
| 120 | processor, |
| 121 | *, |
| 122 | num_imgs: int, |
| 123 | image_sizes: list[ImageSize], |
| 124 | ) -> None: |
| 125 | """Ensure LlavaOnevisionMultiModalProcessor |
| 126 | handles prompt replacement properly for input images. |
| 127 | """ |
| 128 | failed_size_excs = list[tuple[ImageSize, Exception]]() |
| 129 | |
| 130 | validate_one = partial( |
| 131 | _validate_image_prompt_replacements_one, |
| 132 | processor, |
| 133 | num_imgs, |
| 134 | failed_size_excs, |
| 135 | ) |
| 136 | pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes") |
| 137 | |
| 138 | if failed_size_excs: |
| 139 | msg = "Found failing image sizes:" + "\n========\n".join( |
| 140 | f"[{size}]\n{exc}" for size, exc in failed_size_excs |
| 141 | ) |
| 142 | raise AssertionError(msg) |
| 143 | |
| 144 | |
| 145 | @pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]) |
| 146 | @pytest.mark.parametrize("num_imgs", [1, 2]) |
| 147 | def test_processor_prompt_replacements_regression(model_id, num_imgs): |
| 148 | ctx = build_model_context( |
| 149 | model_id, |
| 150 | mm_processor_kwargs=None, |
| 151 | limit_mm_per_prompt={"image": num_imgs}, |
| 152 | ) |
| 153 | |
| 154 | # Avoid tokenizer already borrowed error |
| 155 | maybe_make_thread_pool(ctx.tokenizer) |
| 156 | |
| 157 | processor = MULTIMODAL_REGISTRY.create_processor( |
| 158 | ctx.model_config, |
| 159 | tokenizer=ctx.tokenizer, |
| 160 | ) |
| 161 | |
| 162 | image_ratios = [ |
| 163 | (171, 152), |
| 164 | (184, 161), |
| 165 | (198, 176), |
| 166 | (333, 296), |
| 167 | (369, 328), |
| 168 | (488, 183), |
| 169 | (2560, 1669), |
| 170 | ] |
| 171 | image_sizes = [ |
| 172 | size for w, h in image_ratios for size in [ImageSize(w, h), ImageSize(h, w)] |
| 173 | ] |
| 174 | |
| 175 | _test_image_prompt_replacements( |
| 176 | processor, |
| 177 | num_imgs=num_imgs, |
| 178 | image_sizes=image_sizes, |
| 179 | ) |
| 180 | |
| 181 | |
| 182 | @pytest.mark.skip( |
| 183 | "This test takes around 2 hours to run. Comment this out to run it manually." |
| 184 | ) |
| 185 | @pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]) |
| 186 | @pytest.mark.parametrize("num_imgs", [1]) |
| 187 | def test_processor_prompt_replacements_all(model_id, num_imgs): |
| 188 | ctx = build_model_context( |
| 189 | model_id, |
| 190 | mm_processor_kwargs=None, |
| 191 | limit_mm_per_prompt={"image": num_imgs}, |
| 192 | ) |
| 193 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 194 | |
| 195 | seen_aspect_ratios = set[float]() |
| 196 | image_sizes = list[ImageSize]() |
| 197 | |
| 198 | # The aspect ratio of the grid layout is between 1 and 6 |
| 199 | # NOTE: Assumes that feature size calculation is the same if we |
| 200 | # swap the width and height of the image |
| 201 | for w, h in itertools.product(range(64, 1024), repeat=2): |
| 202 | aspect_ratio = w / h |
| 203 | if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios: |
| 204 | image_sizes.append(ImageSize(w, h)) |
| 205 | seen_aspect_ratios.add(aspect_ratio) |
| 206 | |
| 207 | _test_image_prompt_replacements( |
| 208 | processor, |
| 209 | num_imgs=num_imgs, |
| 210 | image_sizes=image_sizes, |
| 211 | ) |