| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from unittest.mock import patch |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | from vllm.assets.image import ImageAsset |
| 8 | from vllm.config import ModelConfig |
| 9 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 10 | from vllm.multimodal.cache import MultiModalProcessorOnlyCache |
| 11 | |
| 12 | |
| 13 | @pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]) |
| 14 | def test_multimodal_processor(model_id): |
| 15 | mm_processor = MULTIMODAL_REGISTRY.create_processor( |
| 16 | ModelConfig(model=model_id, model_impl="transformers") |
| 17 | ) |
| 18 | |
| 19 | image_pil = ImageAsset("cherry_blossom").pil_image |
| 20 | mm_data = {"image": image_pil} |
| 21 | str_prompt = "<|im_start|>user <image>\nWhat is the content of this image?<|im_end|><|im_start|>assistant\n" # noqa: E501 |
| 22 | str_processed_inputs = mm_processor( |
| 23 | prompt=str_prompt, |
| 24 | mm_items=mm_processor.info.parse_mm_data(mm_data), |
| 25 | hf_processor_mm_kwargs={}, |
| 26 | ) |
| 27 | |
| 28 | ids_prompt = [ |
| 29 | 151644, |
| 30 | 872, |
| 31 | 220, |
| 32 | 151646, |
| 33 | 198, |
| 34 | 3838, |
| 35 | 374, |
| 36 | 279, |
| 37 | 2213, |
| 38 | 315, |
| 39 | 419, |
| 40 | 2168, |
| 41 | 30, |
| 42 | 151645, |
| 43 | 151644, |
| 44 | 77091, |
| 45 | 198, |
| 46 | ] |
| 47 | ids_processed_inputs = mm_processor( |
| 48 | prompt=ids_prompt, |
| 49 | mm_items=mm_processor.info.parse_mm_data(mm_data), |
| 50 | hf_processor_mm_kwargs={}, |
| 51 | ) |
| 52 | |
| 53 | assert ( |
| 54 | str_processed_inputs["prompt_token_ids"] |
| 55 | == ids_processed_inputs["prompt_token_ids"] |
| 56 | ) |
| 57 | |
| 58 | |
| 59 | def _process_two_images(separator: str): |
| 60 | model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" |
| 61 | mm_processor = MULTIMODAL_REGISTRY.create_processor( |
| 62 | ModelConfig(model=model_id, model_impl="transformers") |
| 63 | ) |
| 64 | |
| 65 | image = ImageAsset("cherry_blossom").pil_image |
| 66 | prompt = ( |
| 67 | f"<|im_start|>user <image>{separator}<image>\n" |
| 68 | "What do these images show?<|im_end|><|im_start|>assistant\n" |
| 69 | ) |
| 70 | |
| 71 | return mm_processor( |
| 72 | prompt=prompt, |
| 73 | mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}), |
| 74 | hf_processor_mm_kwargs={}, |
| 75 | ) |
| 76 | |
| 77 | |
| 78 | def test_image_multiple_inputs(): |
| 79 | """Multiple images per prompt are each detected as a separate placeholder |
| 80 | and multi-modal item by the Transformers modelling backend.""" |
| 81 | result = _process_two_images(separator="\n and ") |
| 82 | |
| 83 | assert len(result["mm_placeholders"]["image"]) == 2 |
| 84 | assert len(result["mm_kwargs"]["image"]) == 2 |
| 85 | |
| 86 | |
| 87 | def test_image_adjacent_inputs(): |
| 88 | """Adjacent images stay separate placeholders rather than merging into one.""" |
| 89 | result = _process_two_images(separator="") |
| 90 | |
| 91 | assert len(result["mm_placeholders"]["image"]) == 2 |
| 92 | assert len(result["mm_kwargs"]["image"]) == 2 |
| 93 | |
| 94 | |
| 95 | def test_batch_padding_removed_from_image_items(): |
| 96 | """Emu3 pads every image up to the largest in the batch, which would leave an |
| 97 | item's data dependent on what it was processed with and so uncacheable.""" |
| 98 | mm_processor = MULTIMODAL_REGISTRY.create_processor( |
| 99 | ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers") |
| 100 | ) |
| 101 | image_token = mm_processor.info.get_hf_processor().image_token |
| 102 | |
| 103 | images = [ |
| 104 | ImageAsset("cherry_blossom").pil_image, |
| 105 | ImageAsset("cherry_blossom").pil_image.resize((256, 1024)), |
| 106 | ] |
| 107 | result = mm_processor( |
| 108 | prompt=f"{image_token} and {image_token}", |
| 109 | mm_items=mm_processor.info.parse_mm_data({"image": images}), |
| 110 | hf_processor_mm_kwargs={}, |
| 111 | ) |
| 112 | |
| 113 | items = result["mm_kwargs"]["image"] |
| 114 | shapes = set() |
| 115 | for item in items: |
| 116 | height, width = item["image_sizes"].data.flatten().tolist() |
| 117 | pixel_values = item["pixel_values"].data |
| 118 | assert tuple(pixel_values.shape[-2:]) == (height, width) |
| 119 | shapes.add(tuple(pixel_values.shape)) |
| 120 | |
| 121 | # Both images would have been padded to a common shape had they been kept |
| 122 | assert len(shapes) == 2 |
| 123 | |
| 124 | |
| 125 | def _process_one_gemma3_image(): |
| 126 | mm_processor = MULTIMODAL_REGISTRY.create_processor( |
| 127 | ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers") |
| 128 | ) |
| 129 | boi_token = mm_processor.info.get_hf_processor().boi_token |
| 130 | return mm_processor( |
| 131 | prompt=f"{boi_token} What is this?", |
| 132 | mm_items=mm_processor.info.parse_mm_data( |
| 133 | {"image": ImageAsset("cherry_blossom").pil_image} |
| 134 | ), |
| 135 | hf_processor_mm_kwargs={}, |
| 136 | ) |
| 137 | |
| 138 | |
| 139 | def test_non_embedding_tokens_excluded_from_placeholders(): |
| 140 | """Gemma3 wraps each image in text that carries no embeddings, which must be |
| 141 | inside the placeholder range but masked out of it.""" |
| 142 | result = _process_one_gemma3_image() |
| 143 | |
| 144 | (placeholder,) = result["mm_placeholders"]["image"] |
| 145 | assert placeholder.is_embed is not None |
| 146 | assert 0 < int(placeholder.is_embed.sum()) < placeholder.length |
| 147 | |
| 148 | |
| 149 | def test_tokens_structuring_an_image_are_masked_not_dropped(): |
| 150 | """SmolVLM splits each image into tiles introduced by tokens carrying no |
| 151 | embeddings. Those belong inside the placeholder and masked out, because the token |
| 152 | count the processor reports is over the whole span. Idefics3 also refuses a prompt |
| 153 | holding `<image>` when no images are passed, which is how the prompt has to be |
| 154 | tokenized before splicing in the expansion.""" |
| 155 | model_id = "HuggingFaceTB/SmolVLM-256M-Instruct" |
| 156 | mm_processor = MULTIMODAL_REGISTRY.create_processor( |
| 157 | ModelConfig(model=model_id, model_impl="transformers") |
| 158 | ) |
| 159 | result = mm_processor( |
| 160 | prompt="<image>What is this?", |
| 161 | mm_items=mm_processor.info.parse_mm_data( |
| 162 | {"image": ImageAsset("cherry_blossom").pil_image} |
| 163 | ), |
| 164 | hf_processor_mm_kwargs={}, |
| 165 | ) |
| 166 | |
| 167 | (placeholder,) = result["mm_placeholders"]["image"] |
| 168 | assert placeholder.is_embed is not None |
| 169 | assert 0 < int(placeholder.is_embed.sum()) < placeholder.length |
| 170 | |
| 171 | |
| 172 | def test_missing_replacement_offsets_names_the_processor(): |
| 173 | """A processor that reports no replacement offsets cannot be served, which must |
| 174 | be said plainly rather than surfacing later as a field config mismatch.""" |
| 175 | model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" |
| 176 | mm_processor = MULTIMODAL_REGISTRY.create_processor( |
| 177 | ModelConfig(model=model_id, model_impl="transformers") |
| 178 | ) |
| 179 | hf_processor_cls = type(mm_processor.info.get_hf_processor()) |
| 180 | hf_call = hf_processor_cls.__call__ |
| 181 | |
| 182 | def without_offsets(self, *args, **kwargs): |
| 183 | hf_inputs = hf_call(self, *args, **kwargs) |
| 184 | hf_inputs.pop("text_replacement_offsets", None) |
| 185 | return hf_inputs |
| 186 | |
| 187 | with ( |
| 188 | patch.object(hf_processor_cls, "__call__", without_offsets), |
| 189 | pytest.raises(ValueError, match="LlavaOnevisionProcessor returned no"), |
| 190 | ): |
| 191 | mm_processor( |
| 192 | prompt="<image>\nWhat is the content of this image?", |
| 193 | mm_items=mm_processor.info.parse_mm_data( |
| 194 | {"image": ImageAsset("cherry_blossom").pil_image} |
| 195 | ), |
| 196 | hf_processor_mm_kwargs={}, |
| 197 | ) |
| 198 | |
| 199 | |
| 200 | def test_text_only_prompt(): |
| 201 | """An image model still accepts a prompt with no images.""" |
| 202 | model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" |
| 203 | mm_processor = MULTIMODAL_REGISTRY.create_processor( |
| 204 | ModelConfig(model=model_id, model_impl="transformers") |
| 205 | ) |
| 206 | |
| 207 | result = mm_processor( |
| 208 | prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n", |
| 209 | mm_items=mm_processor.info.parse_mm_data({}), |
| 210 | hf_processor_mm_kwargs={}, |
| 211 | ) |
| 212 | |
| 213 | assert len(result["prompt_token_ids"]) > 0 |
| 214 | assert not result["mm_placeholders"] |
| 215 | |
| 216 | |
| 217 | def test_repeated_image_hits_the_processor_cache(): |
| 218 | """Check that mm caching is actually working.""" |
| 219 | model_config = ModelConfig( |
| 220 | model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers" |
| 221 | ) |
| 222 | model_config.multimodal_config.mm_processor_cache_gb = 4 |
| 223 | mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) |
| 224 | cache = MultiModalProcessorOnlyCache(model_config) |
| 225 | image = ImageAsset("cherry_blossom").pil_image |
| 226 | |
| 227 | def process(): |
| 228 | return mm_processor( |
| 229 | prompt="<image>\nWhat is this?", |
| 230 | mm_items=mm_processor.info.parse_mm_data({"image": image}), |
| 231 | hf_processor_mm_kwargs={}, |
| 232 | cache=cache, |
| 233 | ) |
| 234 | |
| 235 | first, second = process(), process() |
| 236 | |
| 237 | assert cache.make_stats().hits > 0 |
| 238 | assert first["prompt_token_ids"] == second["prompt_token_ids"] |
| 239 | assert first["mm_hashes"] == second["mm_hashes"] |
| 240 | |
| 241 | |
| 242 | @pytest.mark.parametrize( |
| 243 | ("model_id", "prompt"), |
| 244 | [ |
| 245 | ("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "<image>\nWhat is this?"), |
| 246 | ("google/gemma-3-4b-it", "<start_of_image> What is this?"), |
| 247 | ("HuggingFaceTB/SmolVLM-256M-Instruct", "<image>What is this?"), |
| 248 | pytest.param( |
| 249 | "BAAI/Emu3-Chat-hf", |
| 250 | "<image> and more text", |
| 251 | marks=pytest.mark.xfail( |
| 252 | reason="Emu3Processor prepends its BOS token only when images are " |
| 253 | "passed, so the unexpanded prompt vLLM tokenizes never " |
| 254 | "gets one. Fixed by huggingface/transformers#47924, unreleased.", |
| 255 | strict=False, |
| 256 | ), |
| 257 | ), |
| 258 | ], |
| 259 | ) |
| 260 | def test_spliced_prompt_matches_hf_expansion(model_id, prompt): |
| 261 | """The prompt is tokenized without any multi-modal data and the expansion spliced |
| 262 | in, so its token ids have to come out the same as the ones the HF processor |
| 263 | produces itself.""" |
| 264 | mm_processor = MULTIMODAL_REGISTRY.create_processor( |
| 265 | ModelConfig(model=model_id, model_impl="transformers") |
| 266 | ) |
| 267 | info = mm_processor.info |
| 268 | image = ImageAsset("cherry_blossom").pil_image |
| 269 | |
| 270 | hf_processor = info.get_hf_processor() |
| 271 | prompt_ids = info.get_tokenizer().encode( |
| 272 | prompt, **info.default_tok_params.get_encode_kwargs() |
| 273 | ) |
| 274 | hf_ids = info.ctx.call_hf_processor( |
| 275 | hf_processor, |
| 276 | dict(text=hf_processor.decode(prompt_ids), images=[image]), |
| 277 | dict(truncation=False, add_special_tokens=False), |
| 278 | )["input_ids"][0].tolist() |
| 279 | |
| 280 | result = mm_processor( |
| 281 | prompt=prompt, |
| 282 | mm_items=info.parse_mm_data({"image": image}), |
| 283 | hf_processor_mm_kwargs={}, |
| 284 | ) |
| 285 | assert result["prompt_token_ids"] == hf_ids |
| 286 | |
| 287 | |
| 288 | def test_nested_image_fields_split_per_image(): |
| 289 | """Idefics3 returns image fields with a leading batch dimension, putting the rows |
| 290 | belonging to each image one dimension further in. Slicing the batch dimension |
| 291 | instead handed the first image every row and the second an empty tensor.""" |
| 292 | model_id = "HuggingFaceTB/SmolVLM-256M-Instruct" |
| 293 | mm_processor = MULTIMODAL_REGISTRY.create_processor( |
| 294 | ModelConfig(model=model_id, model_impl="transformers") |
| 295 | ) |
| 296 | image = ImageAsset("cherry_blossom").pil_image |
| 297 | result = mm_processor( |
| 298 | prompt="<image> and <image>", |
| 299 | mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}), |
| 300 | hf_processor_mm_kwargs={}, |
| 301 | ) |
| 302 | |
| 303 | items = result["mm_kwargs"]["image"] |
| 304 | assert len(items) == 2 |
| 305 | for item in items: |
| 306 | pixel_values = item["pixel_values"].data |
| 307 | assert pixel_values.shape[1] == int(item["num_image_patches"].data) |