| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Test quark-quantized {MXFP4, FP8} mixed precision models. |
| 4 | |
| 5 | Run `pytest tests/quantization/test_mixed_precision.py`. |
| 6 | |
| 7 | """ |
| 8 | |
| 9 | import importlib |
| 10 | import importlib.metadata |
| 11 | import importlib.util |
| 12 | from dataclasses import dataclass |
| 13 | |
| 14 | import lm_eval |
| 15 | import pytest |
| 16 | from packaging import version |
| 17 | |
| 18 | from tests.utils import ( |
| 19 | multi_gpu_only, |
| 20 | ) |
| 21 | |
| 22 | QUARK_MXFP4_AVAILABLE = importlib.util.find_spec("quark") is not None and version.parse( |
| 23 | importlib.metadata.version("amd-quark") |
| 24 | ) >= version.parse("0.8.99") |
| 25 | |
| 26 | |
| 27 | @dataclass |
| 28 | class ModelCase: |
| 29 | model_id: str |
| 30 | tp: int |
| 31 | |
| 32 | |
| 33 | @dataclass |
| 34 | class EvaluationConfig: |
| 35 | model_name: str |
| 36 | |
| 37 | def get_model_args(self) -> str: |
| 38 | return ( |
| 39 | f"pretrained={self.model_name}," |
| 40 | "tensor_parallel_size=4,dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=False" |
| 41 | ) |
| 42 | |
| 43 | |
| 44 | TEST_CONFIGS = { |
| 45 | # Mixed-precision (AMP) model |
| 46 | # - Demonstrates end-to-end pipeline functionality |
| 47 | "amd/Qwen3-8B-WMXFP4FP8-AMXFP4FP8-AMP-KVFP8": {"arc_challenge": 0.52, "mmlu": 0.72}, |
| 48 | # Non-mixed-precision (PTQ) model |
| 49 | # - Reference for pipeline compatibility verification -> No conflicts or breakings |
| 50 | "amd/Llama-2-70b-chat-hf_FP8_MLPerf_V2": { |
| 51 | "arc_challenge": 0.53, |
| 52 | "mmlu": 0.61, |
| 53 | }, |
| 54 | } |
| 55 | |
| 56 | |
| 57 | @pytest.mark.parametrize("model_name, accuracy_numbers", TEST_CONFIGS.items()) |
| 58 | @pytest.mark.skipif(not QUARK_MXFP4_AVAILABLE, reason="amd-quark>=0.9 is not available") |
| 59 | @multi_gpu_only(num_gpus=4) |
| 60 | def test_mixed_precision_model_accuracies(model_name: str, accuracy_numbers: dict): |
| 61 | results = lm_eval.simple_evaluate( |
| 62 | model="vllm", |
| 63 | model_args=EvaluationConfig(model_name).get_model_args(), |
| 64 | tasks=list(accuracy_numbers.keys()), |
| 65 | batch_size=8, |
| 66 | ) |
| 67 | |
| 68 | rtol = 0.05 |
| 69 | |
| 70 | for task, expect_accuracy in accuracy_numbers.items(): |
| 71 | measured_accuracy = results["results"][task]["acc,none"] |
| 72 | assert ( |
| 73 | measured_accuracy - rtol < expect_accuracy |
| 74 | and measured_accuracy + rtol > expect_accuracy |
| 75 | ), f"Expected: {expect_accuracy} | Measured: {measured_accuracy}" |