| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for Nemotron-Nano-VL's multimodal preprocessing kwargs.""" |
| 4 | |
| 5 | from collections.abc import Mapping |
| 6 | |
| 7 | import pytest |
| 8 | from PIL import Image |
| 9 | from transformers import PreTrainedConfig |
| 10 | |
| 11 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 12 | from vllm.multimodal.image import rescale_image_size |
| 13 | from vllm.multimodal.processing import BaseMultiModalProcessor |
| 14 | |
| 15 | from ....conftest import ImageTestAssets |
| 16 | from ...utils import build_model_context |
| 17 | |
| 18 | |
| 19 | def _get_expected_num_patches( |
| 20 | config: PreTrainedConfig, |
| 21 | image: Image.Image, |
| 22 | num_imgs: int, |
| 23 | min_num: int, |
| 24 | max_num: int, |
| 25 | ): |
| 26 | from vllm.transformers_utils.processors.nemotron_vl import ( |
| 27 | calculate_nemotron_vl_targets, |
| 28 | get_nemotron_vl_target_ratios, |
| 29 | ) |
| 30 | |
| 31 | width, height = image.size |
| 32 | |
| 33 | blocks, _, _ = calculate_nemotron_vl_targets( |
| 34 | orig_width=width, |
| 35 | orig_height=height, |
| 36 | target_ratios=get_nemotron_vl_target_ratios( |
| 37 | min_num, |
| 38 | max_num, |
| 39 | ), |
| 40 | image_size=config.force_image_size, |
| 41 | use_thumbnail=False, |
| 42 | ) |
| 43 | expected_num_patches = blocks |
| 44 | |
| 45 | if config.use_thumbnail and expected_num_patches > 1: |
| 46 | expected_num_patches += 1 |
| 47 | |
| 48 | return expected_num_patches |
| 49 | |
| 50 | |
| 51 | def _run_check( |
| 52 | processor: BaseMultiModalProcessor, |
| 53 | images: list[Image.Image], |
| 54 | min_num: int, |
| 55 | max_num: int, |
| 56 | mm_processor_kwargs: Mapping[str, object], |
| 57 | ): |
| 58 | tokenizer = processor.info.get_tokenizer() |
| 59 | config = processor.info.get_hf_config() |
| 60 | image_processor = processor.info.get_image_processor() |
| 61 | |
| 62 | config.use_thumbnail = image_processor.use_thumbnail |
| 63 | prompt = "<image>" * len(images) |
| 64 | mm_data = {"image": images} |
| 65 | |
| 66 | total_expected_num_patches = sum( |
| 67 | _get_expected_num_patches(config, image, len(images), min_num, max_num) |
| 68 | for image in images |
| 69 | ) |
| 70 | print(total_expected_num_patches) |
| 71 | processed_inputs = processor( |
| 72 | prompt, |
| 73 | mm_items=processor.info.parse_mm_data(mm_data), |
| 74 | hf_processor_mm_kwargs=mm_processor_kwargs, |
| 75 | ) |
| 76 | |
| 77 | # Ensure we have the right number of placeholders per num_crops size |
| 78 | image_token_id = tokenizer.convert_tokens_to_ids("<image>") |
| 79 | img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) |
| 80 | pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape |
| 81 | print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape) |
| 82 | assert img_tok_count == 256 * total_expected_num_patches |
| 83 | assert pixel_shape[0] == total_expected_num_patches |
| 84 | |
| 85 | |
| 86 | @pytest.mark.parametrize("model_id", ["nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"]) |
| 87 | @pytest.mark.parametrize( |
| 88 | "size_factors", |
| 89 | [ |
| 90 | # Single-scale |
| 91 | [1.0], |
| 92 | # Single-scale, batched |
| 93 | [1.0, 1.0, 1.0], |
| 94 | # Multi-scale |
| 95 | [0.25, 0.5, 1.0], |
| 96 | [4.0, 2.0, 1.0], |
| 97 | ], |
| 98 | ) |
| 99 | @pytest.mark.parametrize( |
| 100 | ("min_dynamic_patch", "max_dynamic_patch"), |
| 101 | [(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)], |
| 102 | ) |
| 103 | @pytest.mark.parametrize("dynamic_image_size", [True, False]) |
| 104 | @pytest.mark.parametrize("kwargs_on_init", [True, False]) |
| 105 | def test_processor_override( |
| 106 | model_id: str, |
| 107 | image_assets: ImageTestAssets, |
| 108 | size_factors: list[int], |
| 109 | min_dynamic_patch: int, |
| 110 | max_dynamic_patch: int, |
| 111 | dynamic_image_size: bool | None, |
| 112 | kwargs_on_init: bool, |
| 113 | ): |
| 114 | mm_processor_kwargs = { |
| 115 | "min_dynamic_patch": min_dynamic_patch, |
| 116 | "max_dynamic_patch": max_dynamic_patch, |
| 117 | "dynamic_image_size": dynamic_image_size, |
| 118 | } |
| 119 | |
| 120 | ctx = build_model_context( |
| 121 | model_id, |
| 122 | mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None, |
| 123 | limit_mm_per_prompt={"image": len(size_factors)}, |
| 124 | ) |
| 125 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 126 | hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs |
| 127 | |
| 128 | min_num = min_dynamic_patch if dynamic_image_size else 1 |
| 129 | max_num = max_dynamic_patch if dynamic_image_size else 1 |
| 130 | |
| 131 | _run_check( |
| 132 | processor, |
| 133 | [rescale_image_size(image_assets[0].pil_image, f) for f in size_factors], |
| 134 | min_num, |
| 135 | max_num, |
| 136 | hf_processor_mm_kwargs, |
| 137 | ) |