| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import pytest |
| 4 | |
| 5 | from tests.models.utils import GenerateModelInfo |
| 6 | |
| 7 | from .ppl_utils import wikitext_ppl_test |
| 8 | |
| 9 | MODELS = [ |
| 10 | GenerateModelInfo("google/gemma-2b", hf_ppl=21.48524284362793), |
| 11 | GenerateModelInfo("google/gemma-2-2b", hf_ppl=102.59290313720703), |
| 12 | GenerateModelInfo("google/gemma-3-4b-it", hf_ppl=27.79648208618164), |
| 13 | ] |
| 14 | |
| 15 | |
| 16 | @pytest.mark.parametrize("model_info", MODELS) |
| 17 | def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo): |
| 18 | wikitext_ppl_test(hf_runner, vllm_runner, model_info) |