| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests whether Marlin models can be loaded from the autogptq config. |
| 4 | |
| 5 | Run `pytest tests/quantization/test_configs.py --forked`. |
| 6 | """ |
| 7 | |
| 8 | import pytest |
| 9 | |
| 10 | from vllm.config import ModelConfig |
| 11 | from vllm.platforms import current_platform |
| 12 | |
| 13 | # Model Id // Quantization Arg // Expected Type |
| 14 | MODEL_ARG_EXPTYPES = [ |
| 15 | # AUTOGPTQ |
| 16 | # compat: autogptq <=0.7.1 is_marlin_format: bool |
| 17 | # Model Serialized in Exllama Format. |
| 18 | ("TheBloke/Llama-2-7B-Chat-GPTQ", None, "auto_gptq"), |
| 19 | ( |
| 20 | "TheBloke/Llama-2-7B-Chat-GPTQ", |
| 21 | "marlin", |
| 22 | "auto_gptq" if current_platform.is_cuda_alike() else "ERROR", |
| 23 | ), |
| 24 | ("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "auto_gptq"), |
| 25 | ("TheBloke/Llama-2-7B-Chat-GPTQ", "awq", "ERROR"), |
| 26 | # compat: autogptq >=0.8.0 use checkpoint_format: str |
| 27 | # Model Serialized in Exllama Format. |
| 28 | ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", None, "auto_gptq"), |
| 29 | ( |
| 30 | "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", |
| 31 | "marlin", |
| 32 | "auto_gptq" if current_platform.is_cuda_alike() else "ERROR", |
| 33 | ), |
| 34 | ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"), |
| 35 | ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"), |
| 36 | # AUTOAWQ |
| 37 | # AutoAWQConfig.override_quantization_method() returns "auto_awq" for AWQ models |
| 38 | # when user_quant is None, "awq", "awq_marlin", "marlin", or "auto_awq" |
| 39 | ( |
| 40 | "TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", |
| 41 | None, |
| 42 | "auto_awq", |
| 43 | ), |
| 44 | ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "auto_awq"), |
| 45 | ( |
| 46 | "TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", |
| 47 | "marlin", |
| 48 | "auto_awq" if current_platform.is_cuda_alike() else "ERROR", |
| 49 | ), |
| 50 | ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"), |
| 51 | ] |
| 52 | |
| 53 | |
| 54 | @pytest.mark.parametrize("model_arg_exptype", MODEL_ARG_EXPTYPES) |
| 55 | def test_auto_gptq(model_arg_exptype: tuple[str, None, str]) -> None: |
| 56 | model_path, quantization_arg, expected_type = model_arg_exptype |
| 57 | |
| 58 | try: |
| 59 | model_config = ModelConfig(model_path, quantization=quantization_arg) |
| 60 | found_quantization_type = model_config.quantization |
| 61 | except ValueError: |
| 62 | found_quantization_type = "ERROR" |
| 63 | |
| 64 | assert found_quantization_type == expected_type, ( |
| 65 | f"Expected quant_type == {expected_type} for {model_path}, " |
| 66 | f"but found {found_quantization_type} " |
| 67 | f"for no --quantization {quantization_arg} case" |
| 68 | ) |