| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | |
| 6 | from tests.models.utils import GenerateModelInfo |
| 7 | |
| 8 | from .ppl_utils import wikitext_ppl_test |
| 9 | |
| 10 | MODELS = [ |
| 11 | # for Qwen3 |
| 12 | GenerateModelInfo("Qwen/Qwen3-0.6B", hf_ppl=23.864173889160156), |
| 13 | GenerateModelInfo("Qwen/Qwen3-0.6B-FP8", hf_ppl=24.313045501708984), |
| 14 | # for Qwen3.5 |
| 15 | GenerateModelInfo("Qwen/Qwen3.5-0.8B", hf_ppl=19.38858413696289), |
| 16 | ] |
| 17 | |
| 18 | |
| 19 | @pytest.mark.parametrize("model_info", MODELS) |
| 20 | def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo): |
| 21 | vllm_extra_kwargs = {} |
| 22 | if model_info.name == "Qwen/Qwen3.5-0.8B": |
| 23 | vllm_extra_kwargs["language_model_only"] = True |
| 24 | # Qwen3.5 is a hybrid model; its default mamba cache mode ('align') |
| 25 | # for prefix caching requires chunked prefill. |
| 26 | vllm_extra_kwargs["enable_chunked_prefill"] = True |
| 27 | |
| 28 | wikitext_ppl_test( |
| 29 | hf_runner, vllm_runner, model_info, vllm_extra_kwargs=vllm_extra_kwargs |
| 30 | ) |