| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | |
| 6 | from tests.models.utils import GenerateModelInfo |
| 7 | |
| 8 | from .ppl_utils import vqa_ppl_test |
| 9 | |
| 10 | MODELS = [ |
| 11 | ( |
| 12 | GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=9.810060501098633), |
| 13 | 28, |
| 14 | {}, |
| 15 | ), |
| 16 | ( |
| 17 | GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=22.17719078063965), |
| 18 | 28, |
| 19 | {}, |
| 20 | ), |
| 21 | ( |
| 22 | GenerateModelInfo("Qwen/Qwen3-VL-4B-Instruct", hf_ppl=11.916606903076172), |
| 23 | 32, |
| 24 | {}, |
| 25 | ), |
| 26 | ( |
| 27 | GenerateModelInfo("Qwen/Qwen3.5-4B", hf_ppl=6.478141784667969), |
| 28 | 32, |
| 29 | {"enable_prefix_caching": False}, |
| 30 | ), |
| 31 | ] |
| 32 | |
| 33 | |
| 34 | def _pixel_factor(factor: int): |
| 35 | """`factor = patch_size * merge_size` for Qwen models""" |
| 36 | return {"min_pixels": factor**2, "max_pixels": 1280 * factor**2} |
| 37 | |
| 38 | |
| 39 | @pytest.mark.parametrize("model_info,factor,extra_kwargs", MODELS) |
| 40 | @pytest.mark.parametrize("mm_device_do_normalize", [True, False]) |
| 41 | def test_ppl( |
| 42 | hf_runner, |
| 43 | vllm_runner, |
| 44 | model_info: GenerateModelInfo, |
| 45 | factor: int, |
| 46 | extra_kwargs: dict, |
| 47 | mm_device_do_normalize: bool, |
| 48 | ): |
| 49 | vqa_ppl_test( |
| 50 | hf_runner, |
| 51 | vllm_runner, |
| 52 | model_info, |
| 53 | vllm_extra_kwargs={ |
| 54 | "mm_device_do_normalize": mm_device_do_normalize, |
| 55 | **extra_kwargs, |
| 56 | }, |
| 57 | mm_processor_kwargs=_pixel_factor(factor), |
| 58 | ) |