| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import pytest |
| 4 | import torch |
| 5 | |
| 6 | from vllm.model_executor.models.minicpmv import MiniCPMV2_6 |
| 7 | from vllm.model_executor.models.minicpmv4_6 import ( |
| 8 | MiniCPMV4_6ForConditionalGeneration, |
| 9 | ) |
| 10 | |
| 11 | pytestmark = pytest.mark.skip_global_cleanup |
| 12 | |
| 13 | |
| 14 | def make_model() -> MiniCPMV2_6: |
| 15 | return object.__new__(MiniCPMV2_6) |
| 16 | |
| 17 | |
| 18 | def make_model_4_6() -> MiniCPMV4_6ForConditionalGeneration: |
| 19 | model = object.__new__(MiniCPMV4_6ForConditionalGeneration) |
| 20 | model._process_vision_input = lambda vision_input, use_vit_merger=None: [ |
| 21 | vision_input["image_embeds"] |
| 22 | ] |
| 23 | return model |
| 24 | |
| 25 | |
| 26 | def test_video_embeds_reach_the_vision_parser(): |
| 27 | embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8) |
| 28 | |
| 29 | modalities = make_model()._parse_and_validate_multimodal_inputs(video_embeds=embeds) |
| 30 | |
| 31 | video_input = modalities["videos"] |
| 32 | assert video_input is not None |
| 33 | assert video_input["type"] == "image_embeds" |
| 34 | assert torch.equal(video_input["image_embeds"], embeds) |
| 35 | |
| 36 | |
| 37 | def test_video_embeds_are_embedded_by_4_6(): |
| 38 | embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8) |
| 39 | |
| 40 | embeddings = make_model_4_6().embed_multimodal(video_embeds=embeds) |
| 41 | |
| 42 | assert len(embeddings) == 1 |
| 43 | assert torch.equal(embeddings[0], embeds) |
| 44 | |
| 45 | |
| 46 | def test_image_and_video_embeds_stay_in_their_own_modality(): |
| 47 | image_embeds = torch.zeros(1, 4, 8) |
| 48 | video_embeds = torch.ones(1, 4, 8) |
| 49 | |
| 50 | modalities = make_model()._parse_and_validate_multimodal_inputs( |
| 51 | image_embeds=image_embeds, |
| 52 | video_embeds=video_embeds, |
| 53 | ) |
| 54 | |
| 55 | assert torch.equal(modalities["images"]["image_embeds"], image_embeds) |
| 56 | assert torch.equal(modalities["videos"]["image_embeds"], video_embeds) |