| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | """Tests for MiniCPMV's multimodal preprocessing.""" |
| 5 | |
| 6 | import numpy as np |
| 7 | import pytest |
| 8 | |
| 9 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 10 | from vllm.multimodal.parse import ImageSize |
| 11 | |
| 12 | from ...utils import build_model_context |
| 13 | |
| 14 | |
| 15 | @pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) |
| 16 | def test_get_hf_processor_for_same_model_different_kwargs(model_id: str): |
| 17 | """Calls with different kwargs must not reuse stale processor instances.""" |
| 18 | ctx = build_model_context( |
| 19 | model_id, |
| 20 | limit_mm_per_prompt={"image": 1}, |
| 21 | ) |
| 22 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 23 | info = processor.info |
| 24 | processor_1 = info.get_hf_processor(max_slice_nums=1) |
| 25 | processor_2 = info.get_hf_processor(max_slice_nums=2) |
| 26 | assert processor_1.image_processor.max_slice_nums == 1 |
| 27 | assert processor_2.image_processor.max_slice_nums == 2 |
| 28 | |
| 29 | |
| 30 | @pytest.mark.parametrize( |
| 31 | "model_ids", |
| 32 | [ |
| 33 | ("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4"), |
| 34 | ("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6"), |
| 35 | ], |
| 36 | ) |
| 37 | def test_image_processor_for_different_models(model_ids): |
| 38 | first_model_id, second_model_id = model_ids |
| 39 | |
| 40 | first_ctx = build_model_context( |
| 41 | first_model_id, |
| 42 | limit_mm_per_prompt={"image": 1}, |
| 43 | ) |
| 44 | first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config) |
| 45 | first_image_processor = first_processor.info.get_image_processor() |
| 46 | |
| 47 | second_ctx = build_model_context( |
| 48 | second_model_id, |
| 49 | limit_mm_per_prompt={"image": 1}, |
| 50 | ) |
| 51 | second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config) |
| 52 | second_image_processor = second_processor.info.get_image_processor() |
| 53 | |
| 54 | second_processor.info.get_sliced_grid(ImageSize(width=128, height=128)) |
| 55 | |
| 56 | assert type(first_image_processor) is not type(second_image_processor) |
| 57 | assert ( |
| 58 | type(first_image_processor).__module__ |
| 59 | != type(second_image_processor).__module__ |
| 60 | ) |
| 61 | |
| 62 | |
| 63 | @pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) |
| 64 | def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str): |
| 65 | ctx = build_model_context( |
| 66 | model_id, |
| 67 | limit_mm_per_prompt={"image": 1}, |
| 68 | ) |
| 69 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 70 | tokenizer = ctx.get_tokenizer() |
| 71 | |
| 72 | messages = [ |
| 73 | {"role": "user", "content": "(<image>./</image>)\nWhat is in this image?"} |
| 74 | ] |
| 75 | prompt = tokenizer.apply_chat_template( |
| 76 | messages, |
| 77 | tokenize=False, |
| 78 | add_generation_prompt=True, |
| 79 | ) |
| 80 | image = np.zeros((768, 1024, 3), dtype=np.uint8) |
| 81 | |
| 82 | mm_items = processor.info.parse_mm_data({"image": [image]}) |
| 83 | processed = processor( |
| 84 | prompt, |
| 85 | mm_items=mm_items, |
| 86 | hf_processor_mm_kwargs={}, |
| 87 | ) |
| 88 | image_placeholders = processed["mm_placeholders"].get("image", []) |
| 89 | assert len(image_placeholders) == 1 |
| 90 | assert image_placeholders[0].length > 0 |