| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for Llama4's multimodal preprocessing kwargs.""" |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 8 | |
| 9 | from ....conftest import ImageTestAssets |
| 10 | from ...utils import build_model_context |
| 11 | |
| 12 | |
| 13 | @pytest.mark.parametrize("model_id", ["meta-llama/Llama-4-Scout-17B-16E-Instruct"]) |
| 14 | @pytest.mark.parametrize("mm_processor_kwargs", [{}]) |
| 15 | @pytest.mark.parametrize("num_imgs", [1, 5]) |
| 16 | @pytest.mark.parametrize("mm_processor_cache_gb", [0, 4]) |
| 17 | @pytest.mark.parametrize("tokenized_prompt", [True, False]) |
| 18 | def test_processor_override( |
| 19 | image_assets: ImageTestAssets, |
| 20 | model_id: str, |
| 21 | mm_processor_kwargs: dict, |
| 22 | num_imgs: int, |
| 23 | mm_processor_cache_gb: int, |
| 24 | tokenized_prompt: bool, |
| 25 | ): |
| 26 | """Ensure llama4 processor works properly.""" |
| 27 | ctx = build_model_context( |
| 28 | model_id, |
| 29 | mm_processor_kwargs=mm_processor_kwargs, |
| 30 | limit_mm_per_prompt={"image": num_imgs}, |
| 31 | mm_processor_cache_gb=mm_processor_cache_gb, |
| 32 | ) |
| 33 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 34 | config = processor.info.get_hf_config() |
| 35 | tokenizer = processor.info.get_tokenizer() |
| 36 | hf_processor = processor.info.get_hf_processor() |
| 37 | vocab = tokenizer.get_vocab() |
| 38 | |
| 39 | prompt = ( |
| 40 | "<|begin_of_text|><|header_start|>user<|header_end|>" |
| 41 | + "<|image|>" * num_imgs |
| 42 | + "<|eot|><|header_start|>assistant<|header_end|>" |
| 43 | ) |
| 44 | mm_data = { |
| 45 | "image": [ |
| 46 | image_assets[(i % len(image_assets))].pil_image for i in range(num_imgs) |
| 47 | ] |
| 48 | } |
| 49 | if tokenized_prompt: |
| 50 | prompt = tokenizer.encode(prompt) |
| 51 | |
| 52 | processed_inputs = processor( |
| 53 | prompt, |
| 54 | mm_items=processor.info.parse_mm_data(mm_data), |
| 55 | hf_processor_mm_kwargs=mm_processor_kwargs, |
| 56 | ) |
| 57 | mm_data = processed_inputs["mm_kwargs"].get_data() |
| 58 | |
| 59 | # place holder replacements |
| 60 | prompt_token_ids = processed_inputs["prompt_token_ids"] |
| 61 | assert prompt_token_ids.count(config.boi_token_index) == num_imgs |
| 62 | assert prompt_token_ids.count(config.eoi_token_index) == num_imgs |
| 63 | assert prompt_token_ids.count(vocab[hf_processor.image_token]) == num_imgs |
| 64 | aspect_ratios = mm_data["aspect_ratios"] |
| 65 | num_x_separators = num_y_separators = 0 |
| 66 | for tiles_y, tiles_x in aspect_ratios: |
| 67 | if tiles_x * tiles_y > 1: |
| 68 | num_x_separators += (tiles_x - 1) * tiles_y |
| 69 | num_y_separators += tiles_y |
| 70 | assert prompt_token_ids.count(vocab[hf_processor.tile_token]) == num_x_separators |
| 71 | assert ( |
| 72 | prompt_token_ids.count(vocab[hf_processor.tile_global_token]) |
| 73 | == num_y_separators |
| 74 | ) |
| 75 | |
| 76 | # image token offsets |
| 77 | img_locs = processed_inputs["mm_placeholders"].get("image", []) |
| 78 | assert len(img_locs) == num_imgs |
| 79 | assert [img_loc.offset for img_loc in img_locs] == [ |
| 80 | i for i, v in enumerate(prompt_token_ids) if v == config.boi_token_index |
| 81 | ] |
| 82 | |
| 83 | # patch sizes and masks |
| 84 | num_patches_per_chunk = processor.info.get_patch_per_chunk(config.vision_config) |
| 85 | assert ( |
| 86 | prompt_token_ids.count(config.image_token_index) |
| 87 | == sum(mm_data["patches_per_image"]) * num_patches_per_chunk |
| 88 | ) |
| 89 | assert len(mm_data["pixel_values"]) == sum(mm_data["patches_per_image"]) |