| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | |
| 6 | from tests.models.utils import GenerateModelInfo |
| 7 | |
| 8 | from .ppl_utils import vqa_ppl_test |
| 9 | |
| 10 | MODELS = [ |
| 11 | (GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28), |
| 12 | ] |
| 13 | |
| 14 | |
| 15 | def _pixel_size(factor: int): |
| 16 | # Match max image token size with qwen's PPL test |
| 17 | return { |
| 18 | "size": { |
| 19 | "shortest_edge": 2 * factor**2, |
| 20 | "longest_edge": 2 * 1280 * factor**2, |
| 21 | } |
| 22 | } |
| 23 | |
| 24 | |
| 25 | @pytest.mark.parametrize("model_info,factor", MODELS) |
| 26 | @pytest.mark.parametrize("mm_device_do_normalize", [True, False]) |
| 27 | def test_ppl( |
| 28 | hf_runner, |
| 29 | vllm_runner, |
| 30 | model_info: GenerateModelInfo, |
| 31 | factor: int, |
| 32 | mm_device_do_normalize: bool, |
| 33 | ): |
| 34 | vqa_ppl_test( |
| 35 | hf_runner, |
| 36 | vllm_runner, |
| 37 | model_info, |
| 38 | vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize}, |
| 39 | mm_processor_kwargs=_pixel_size(factor), |
| 40 | ) |