| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import pytest |
| 4 | |
| 5 | from tests.models.utils import GenerateModelInfo |
| 6 | |
| 7 | from .ppl_utils import wikitext_ppl_test |
| 8 | |
| 9 | MODELS = [GenerateModelInfo("openai-community/gpt2-large", hf_ppl=19.457056045532227)] |
| 10 | |
| 11 | |
| 12 | @pytest.mark.parametrize("model_info", MODELS) |
| 13 | def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo): |
| 14 | bf16_ppl = wikitext_ppl_test(hf_runner, vllm_runner, model_info) |
| 15 | fp32_ppl = wikitext_ppl_test( |
| 16 | hf_runner, |
| 17 | vllm_runner, |
| 18 | model_info, |
| 19 | vllm_extra_kwargs={"hf_overrides": {"head_dtype": "float32"}}, |
| 20 | ) |
| 21 | |
| 22 | differ = ((fp32_ppl - bf16_ppl) / bf16_ppl) * 100 |
| 23 | print("fp32 head difference (%):", differ) |
| 24 | assert differ < 0 |