@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa2a33b893
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 3, 15:15 UTC

[Quant] Consolidate GPTQ: rename gptq_marlin.py to auto_gptq.py (#38288)

Signed-off-by: Chengyi Nie <cnie@roblox.com> Co-authored-by: Chengyi Nie <cnie@roblox.com> Co-authored-by: Cursor <cursoragent@cursor.com>

Chengyi Nie committed May 14, 2026 at 17:25 UTC fa2a33b893d8ede4234911f912738ade601738ac
21 files changed +171 -602
.buildkite/hardware_tests/cpu.yaml
+1 -1
@@ -61,7 +61,7 @@ steps:
61 source_file_dependencies:
62 - csrc/cpu/
63 - vllm/model_executor/layers/quantization/cpu_wna16.py
64 - - vllm/model_executor/layers/quantization/gptq_marlin.py
64 + - vllm/model_executor/layers/quantization/auto_gptq.py
65 - vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
66 - vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py
67 - vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py
tests/models/quantization/test_gptq_marlin.py
+3 -6
@@ -1,10 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Compares the outputs of gptq vs gptq_marlin.
3 +"""Tests AutoGPTQ (GPTQ with Marlin kernels) output correctness.
4
5 -Note: GPTQ and Marlin do not have bitwise correctness.
6 -As a result, in this test, we just confirm that the top selected tokens of the
7 -Marlin/GPTQ models are in the top 5 selections of each other.
5 Note: Marlin internally uses locks to synchronize the threads. This can
6 result in very slight nondeterminism for Marlin. As a result, we re-run the test
7 up to 3 times to see if we pass.
@@ -36,10 +33,10 @@ MODELS = [
33
34 @pytest.mark.flaky(reruns=3)
35 @pytest.mark.skipif(
39 - not is_quant_method_supported("gptq_marlin")
36 + not is_quant_method_supported("auto_gptq")
37 or current_platform.is_rocm()
38 or not current_platform.is_cuda(),
42 - reason="gptq_marlin is not supported on this GPU type.",
39 + reason="auto_gptq is not supported on this GPU type.",
40 )
41 @pytest.mark.parametrize("model", MODELS)
42 @pytest.mark.parametrize("dtype", ["half", "bfloat16"])
tests/quantization/test_auto_gptq.py new
+56
@@ -0,0 +1,56 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests that the auto_gptq quantization method works correctly.
4 +
5 +Run `pytest tests/quantization/test_auto_gptq.py -v -s`.
6 +"""
7 +
8 +import pytest
9 +import torch
10 +
11 +from tests.quantization.utils import is_quant_method_supported
12 +from vllm.model_executor.layers.quantization.auto_gptq import (
13 + AutoGPTQConfig,
14 + AutoGPTQLinearMethod,
15 +)
16 +
17 +PROMPT = "On the surface of Mars, we found"
18 +
19 +MODELS = [
20 + "TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
21 +]
22 +
23 +
24 +@pytest.mark.skipif(
25 + not is_quant_method_supported("auto_gptq"),
26 + reason="auto_gptq is not supported on this GPU type.",
27 +)
28 +@pytest.mark.parametrize("model_id", MODELS)
29 +def test_auto_gptq_quantization_method(vllm_runner, model_id: str, monkeypatch):
30 + """Test that quantization='auto_gptq' loads and runs correctly."""
31 + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
32 +
33 + with vllm_runner(
34 + model_id,
35 + dtype=torch.float16,
36 + quantization="auto_gptq",
37 + max_model_len=2048,
38 + enforce_eager=True,
39 + ) as llm:
40 +
41 + def check_model(model):
42 + for name, submodule in model.named_modules():
43 + if name == "model.layers.0.self_attn.qkv_proj":
44 + assert isinstance(submodule.quant_method, AutoGPTQLinearMethod)
45 + break
46 +
47 + llm.apply_model(check_model)
48 +
49 + outputs = llm.generate_greedy([PROMPT], max_tokens=8)
50 + assert outputs
51 + assert len(outputs[0][1]) > 0
52 +
53 +
54 +def test_auto_gptq_config_get_name():
55 + """Test that AutoGPTQConfig.get_name() returns 'auto_gptq'."""
56 + assert AutoGPTQConfig.get_name() == "auto_gptq"
tests/quantization/test_configs.py
+6 -14
@@ -24,31 +24,23 @@ MODEL_ARG_EXPTYPES = [
24 # AUTOGPTQ
25 # compat: autogptq <=0.7.1 is_marlin_format: bool
26 # Model Serialized in Exllama Format.
27 - (
28 - "TheBloke/Llama-2-7B-Chat-GPTQ",
29 - None,
30 - "gptq_marlin" if current_platform.is_cuda() else "gptq",
31 - ),
27 + ("TheBloke/Llama-2-7B-Chat-GPTQ", None, "auto_gptq"),
28 (
29 "TheBloke/Llama-2-7B-Chat-GPTQ",
30 "marlin",
35 - "gptq_marlin" if current_platform.is_cuda() else "ERROR",
31 + "auto_gptq" if current_platform.is_cuda() else "ERROR",
32 ),
37 - ("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "gptq"),
33 + ("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "auto_gptq"),
34 ("TheBloke/Llama-2-7B-Chat-GPTQ", "awq", "ERROR"),
35 # compat: autogptq >=0.8.0 use checkpoint_format: str
36 # Model Serialized in Exllama Format.
41 - (
42 - "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
43 - None,
44 - "gptq_marlin" if current_platform.is_cuda() else "gptq",
45 - ),
37 + ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", None, "auto_gptq"),
38 (
39 "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
40 "marlin",
49 - "gptq_marlin" if current_platform.is_cuda() else "ERROR",
41 + "auto_gptq" if current_platform.is_cuda() else "ERROR",
42 ),
51 - ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "gptq"),
43 + ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"),
44 ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
45 # AUTOAWQ
46 (
tests/quantization/test_gptq_dynamic.py
+9 -19
@@ -3,46 +3,36 @@
3 """Tests whether gptq models with dynamic quantized can be loaded.
4
5 Run `pytest tests/quantization/test_gptq_dynamic.py --forked`.
6 +
7 +Note: Only symmetric GPTQ models are supported after consolidation to Marlin.
8 """
9
10 import pytest
11 import torch
12
13 from vllm.model_executor.layers.linear import UnquantizedLinearMethod
12 -from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
13 -from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinLinearMethod
14 +from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
15 from vllm.model_executor.layers.quantization.utils.gptq_utils import (
16 get_dynamic_override,
17 )
17 -from vllm.platforms import current_platform
18
19 PROMPT = "On the surface of Mars, we found"
20
21 # The first layer is quantized using bits=4, group_size=128
22 # The second layer is quantized using bits=8, group_size=32
23 # All other layers (layer index >= 2) are not quantized
24 -MODEL_QUANT = [
25 - (
26 - "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
27 - current_platform.is_cuda(),
28 - ),
29 - (
30 - "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse",
31 - False,
32 - ),
24 +# Note: Only symmetric models are supported with Marlin kernels
25 +MODELS = [
26 + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
27 ]
28
29
36 -@pytest.mark.parametrize("model_id, use_marlin_kernel", MODEL_QUANT)
37 -def test_gptq_with_dynamic(
38 - vllm_runner, model_id: str, use_marlin_kernel: bool, monkeypatch
39 -):
30 +@pytest.mark.parametrize("model_id", MODELS)
31 +def test_gptq_with_dynamic(vllm_runner, model_id: str, monkeypatch):
32 # `LLM.apply_model` requires pickling a function.
33 monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
34
43 - linear_method_cls = (
44 - GPTQMarlinLinearMethod if use_marlin_kernel else (GPTQLinearMethod)
45 - )
35 + linear_method_cls = AutoGPTQLinearMethod
36
37 with vllm_runner(
38 model_id, dtype=torch.float16, max_model_len=2048, enforce_eager=True
tests/quantization/test_gptq_v2.py
+8 -11
@@ -3,6 +3,9 @@
3 """Tests whether vllm correctly load and run gptq_v2 format checkpoints.
4
5 Run `pytest tests/quantization/test_gptq_v2.py --forked`.
6 +
7 +Note: 2/3-bit GPTQ models are no longer supported after the consolidation
8 +to Marlin kernels. Only 4/8-bit symmetric GPTQ models are supported.
9 """
10
11 import pytest
@@ -10,9 +13,10 @@ import torch
13 from transformers import AutoTokenizer
14
15 from vllm import SamplingParams
13 -from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
16 +from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
17
18 # A dummy small model quantized by GPTQModel, stored in GPTQ v2 format
19 +# Note: This is a 2-bit model which is no longer supported with Marlin kernels
20 MODELS = ["XXXXyu/Qwen3-1.7B-w2g64-gptq_v2"]
21
22 # Generate multiple sequences for testing, because an 1.7B 2-bit model
@@ -20,27 +24,19 @@ MODELS = ["XXXXyu/Qwen3-1.7B-w2g64-gptq_v2"]
24 N_SEQ = 5
25
26
27 +@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation")
28 @pytest.mark.parametrize("model_id", MODELS)
29 def test_model_load(vllm_runner, model_id, monkeypatch):
30 # `LLM.apply_model` requires pickling a function.
31 monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
32
28 - # Only check the default GPTQ linear method (used for 2/3-bit models).
29 - # 4/8-bit linear methods like Marlin already support gptq_v2.
30 - linear_method_cls = GPTQLinearMethod
31 -
33 with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm:
34
35 def check_model(model_id):
36 for name, submodule in model_id.named_modules():
37 # Could check more modules if necessary
38 if name == "model_id.layers.0.self_attn.qkv_proj":
38 - assert isinstance(submodule.quant_method, linear_method_cls)
39 -
40 - config = submodule.quant_method.quant_config
41 - assert config.checkpoint_format == "gptq_v2"
42 - assert submodule.quant_method.use_v2_format
43 -
39 + assert isinstance(submodule.quant_method, AutoGPTQLinearMethod)
40 # Just break since currently we only check 1 module
41 break
42
@@ -48,6 +44,7 @@ def test_model_load(vllm_runner, model_id, monkeypatch):
44 llm.apply_model(check_model)
45
46
47 +@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation")
48 @pytest.mark.parametrize("model_id", MODELS)
49 def test_model_inference(vllm_runner, model_id):
50 # Prepare prompt to test the model's generation result.
tests/quantization/test_lm_head.py
+2 -3
@@ -8,8 +8,7 @@ Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`.
8 import pytest
9 import torch
10
11 -from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
12 -from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinLinearMethod
11 +from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
12 from vllm.model_executor.layers.vocab_parallel_embedding import (
13 UnquantizedEmbeddingMethod,
14 )
@@ -40,7 +39,7 @@ def test_lm_head(
39 if lm_head_quantized:
40 assert isinstance(
41 lm_head_layer.quant_method,
43 - (GPTQLinearMethod, GPTQMarlinLinearMethod),
42 + AutoGPTQLinearMethod,
43 )
44 else:
45 assert isinstance(
tests/weight_loading/models-large.txt
+2 -2
@@ -2,7 +2,7 @@ compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
2 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
3 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
4 compressed-tensors, nm-testing/test-w4a16-mixtral-actorder-group, main
5 -gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
6 -gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, gptq-8bit-128g-actorder_True
5 +gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
6 +gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, gptq-8bit-128g-actorder_True
7 awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main
8 compressed-tensors, RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16, main
\ No newline at end of file
tests/weight_loading/models.txt
-6
@@ -1,9 +1,3 @@
1 -gptq_marlin, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
2 -gptq_marlin, TheBloke/Llama-2-7B-GPTQ, main
3 -gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
4 -gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
5 -gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
6 -gptq_marlin, TechxGenus/gemma-1.1-2b-it-GPTQ, main
1 gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
2 gptq, TheBloke/Llama-2-7B-GPTQ, main
3 gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
vllm/config/model.py
+2
@@ -961,6 +961,8 @@ class ModelConfig:
961 # `override_quantization_method` method) must be checked in order
962 # of preference (this is particularly important for GPTQ).
963 overrides = [
964 + "auto_gptq",
965 + "gptq",
966 "gptq_marlin",
967 "awq_marlin",
968 "inc",
vllm/model_executor/layers/fused_moe/layer.py
+1 -1
@@ -409,7 +409,7 @@ class FusedMoE(PluggableLayer):
409 }
410 # need full intermediate size pre-sharding for WNA16 act order
411 if self.quant_method.__class__.__name__ in (
412 - "GPTQMarlinMoEMethod",
412 + "AutoGPTQMoEMethod",
413 "CompressedTensorsWNA16MarlinMoEMethod",
414 "CompressedTensorsWNA16MoEMethod",
415 ):
vllm/model_executor/layers/fused_moe/oracle/int_wna16.py
+6 -6
@@ -27,7 +27,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
27 )
28
29 if TYPE_CHECKING:
30 - from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinConfig
30 + from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
31
32 logger = init_logger(__name__)
33
@@ -212,7 +212,7 @@ def make_wna16_moe_kernel(
212
213 def _process_weights_marlin(
214 layer: torch.nn.Module,
215 - quant_config: "GPTQMarlinConfig",
215 + quant_config: "AutoGPTQConfig",
216 input_dtype: torch.dtype | None,
217 w13_qweight: torch.Tensor,
218 w2_qweight: torch.Tensor,
@@ -417,13 +417,13 @@ def convert_to_wna16_moe_kernel_format(
417 WNA16MoEBackend.MARLIN,
418 WNA16MoEBackend.BATCHED_MARLIN,
419 ):
420 - from vllm.model_executor.layers.quantization.gptq_marlin import (
421 - GPTQMarlinConfig,
420 + from vllm.model_executor.layers.quantization.auto_gptq import (
421 + AutoGPTQConfig,
422 )
423
424 - if not isinstance(quant_config, GPTQMarlinConfig):
424 + if not isinstance(quant_config, AutoGPTQConfig):
425 raise TypeError(
426 - "Marlin WNA16 MoE backend requires GPTQMarlinConfig, got "
426 + "Marlin WNA16 MoE backend requires AutoGPTQConfig, got "
427 f"{type(quant_config).__name__}."
428 )
429 return _process_weights_marlin(
vllm/model_executor/layers/linear.py
+1 -2
@@ -48,12 +48,11 @@ WEIGHT_LOADER_V2_SUPPORTED = [
48 "CompressedTensorsLinearTransformMethod",
49 "AWQMarlinLinearMethod",
50 "AWQLinearMethod",
51 - "GPTQMarlinLinearMethod",
51 + "AutoGPTQLinearMethod",
52 "Fp8LinearMethod",
53 "MarlinLinearMethod",
54 "GPTQMarlin24LinearMethod",
55 "TPUInt8LinearMethod",
56 - "GPTQLinearMethod",
56 "FBGEMMFp8LinearMethod",
57 "ModelOptFp8LinearMethod",
58 "ModelOptFp8PcPtLinearMethod",
vllm/model_executor/layers/quantization/__init__.py
+6 -5
@@ -19,9 +19,10 @@ QuantizationMethods = Literal[
19 "modelopt_mxfp8",
20 "modelopt_mixed",
21 "gguf",
22 + "auto_gptq",
23 + "gptq",
24 "gptq_marlin",
25 "awq_marlin",
24 - "gptq",
26 "humming",
27 "compressed-tensors",
28 "bitsandbytes",
@@ -114,6 +115,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
115 from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
116 from vllm.model_executor.models.deepseek_v4 import DeepseekV4FP8Config
117
118 + from .auto_gptq import AutoGPTQConfig
119 from .awq import AWQConfig
120 from .awq_marlin import AWQMarlinConfig
121 from .bitsandbytes import BitsAndBytesConfig
@@ -126,8 +128,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
128 from .fp8 import Fp8Config
129 from .fp_quant import FPQuantConfig
130 from .gguf import GGUFConfig
129 - from .gptq import GPTQConfig
130 - from .gptq_marlin import GPTQMarlinConfig
131 from .humming import HummingConfig
132 from .inc import INCConfig
133 from .modelopt import (
@@ -151,9 +151,10 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
151 "modelopt_mxfp8": ModelOptMxFp8Config,
152 "modelopt_mixed": ModelOptMixedPrecisionConfig,
153 "gguf": GGUFConfig,
154 - "gptq_marlin": GPTQMarlinConfig,
154 + "auto_gptq": AutoGPTQConfig,
155 + "gptq": AutoGPTQConfig,
156 + "gptq_marlin": AutoGPTQConfig,
157 "awq_marlin": AWQMarlinConfig,
156 - "gptq": GPTQConfig,
158 "compressed-tensors": CompressedTensorsConfig,
159 "bitsandbytes": BitsAndBytesConfig,
160 "experts_int8": ExpertsInt8Config,
vllm/model_executor/layers/quantization/auto_gptq.py renamed
+28 -60
@@ -41,7 +41,6 @@ from vllm.model_executor.layers.quantization.utils.gptq_utils import (
41 override_config,
42 )
43 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
44 - check_marlin_supported,
44 check_moe_marlin_supports_layer,
45 get_marlin_input_dtype,
46 marlin_make_workspace_new,
@@ -60,7 +59,6 @@ from vllm.model_executor.parameter import (
59 PackedvLLMParameter,
60 RowvLLMParameter,
61 )
63 -from vllm.platforms import current_platform
62 from vllm.scalar_type import scalar_types
63 from vllm.transformers_utils.config import get_safetensors_params_metadata
64 from vllm.utils.collection_utils import is_list_of
@@ -69,7 +67,7 @@ logger = init_logger(__name__)
67
68
69 def get_moe_quant_method(
72 - config: "GPTQMarlinConfig",
70 + config: "AutoGPTQConfig",
71 layer: RoutedExperts,
72 prefix: str,
73 moe_method_cls: type,
@@ -94,8 +92,8 @@ def get_moe_quant_method(
92 return moe_method_cls(cloned_config, layer.moe_config)
93
94
97 -class GPTQMarlinConfig(QuantizationConfig):
98 - """Config class for GPTQ Marlin"""
95 +class AutoGPTQConfig(QuantizationConfig):
96 + """Config class for AutoGPTQ quantization using Marlin kernels."""
97
98 # (num_bits, is_sym) -> quant_type
99 TYPE_MAP = {
@@ -167,7 +165,7 @@ class GPTQMarlinConfig(QuantizationConfig):
165
166 def __repr__(self) -> str:
167 return (
170 - f"GPTQMarlinConfig(quant_type={self.quant_type}, "
168 + f"AutoGPTQConfig(quant_type={self.quant_type}, "
169 f"group_size={self.group_size}, "
170 f"desc_act={self.desc_act}, "
171 f"lm_head_quantized={self.lm_head_quantized}, "
@@ -177,7 +175,7 @@ class GPTQMarlinConfig(QuantizationConfig):
175
176 @classmethod
177 def get_name(cls) -> QuantizationMethods:
180 - return "gptq_marlin"
178 + return "auto_gptq"
179
180 @classmethod
181 def get_supported_act_dtypes(cls) -> list[torch.dtype]:
@@ -185,14 +183,14 @@ class GPTQMarlinConfig(QuantizationConfig):
183
184 @classmethod
185 def get_min_capability(cls) -> int:
188 - return 75
186 + return 60
187
188 @classmethod
189 def get_config_filenames(cls) -> list[str]:
190 return ["quantize_config.json"]
191
192 @classmethod
195 - def from_config(cls, config: dict[str, Any]) -> "GPTQMarlinConfig":
193 + def from_config(cls, config: dict[str, Any]) -> "AutoGPTQConfig":
194 dynamic = cls.get_from_keys_or(config, ["dynamic"], default={})
195 dynamic = {} if dynamic is None else dynamic
196
@@ -219,27 +217,22 @@ class GPTQMarlinConfig(QuantizationConfig):
217 def override_quantization_method(
218 cls, hf_quant_cfg, user_quant, hf_config=None
219 ) -> QuantizationMethods | None:
222 - can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg)
220 + """Override to use AutoGPTQ for compatible GPTQ models."""
221 + quant_method = hf_quant_cfg.get("quant_method", "").lower()
222
224 - is_valid_user_quant = (
225 - user_quant is None or user_quant == "marlin" or user_quant == "gptq_marlin"
223 + if quant_method != "gptq":
224 + return None
225 +
226 + is_valid_user_quant = user_quant is None or user_quant in (
227 + "gptq",
228 + "gptq_marlin",
229 + "auto_gptq",
230 + "marlin",
231 )
232
228 - if can_convert and is_valid_user_quant:
229 - msg = (
230 - "The model is convertible to {} during runtime."
231 - " Using {} kernel.".format(cls.get_name(), cls.get_name())
232 - )
233 - logger.info(msg)
233 + if is_valid_user_quant:
234 return cls.get_name()
235
236 - if can_convert and user_quant == "gptq":
237 - logger.info(
238 - "Detected that the model can run with gptq_marlin"
239 - ", however you specified quantization=gptq explicitly,"
240 - " so forcing gptq. Use quantization=gptq_marlin for"
241 - " faster inference"
242 - )
236 return None
237
238 def get_quant_method(
@@ -257,7 +250,7 @@ class GPTQMarlinConfig(QuantizationConfig):
250 layer, prefix
251 )
252 moe_quant_method = get_moe_quant_method(
260 - self, layer, prefix, GPTQMarlinMoEMethod
253 + self, layer, prefix, AutoGPTQMoEMethod
254 )
255 if moe_quant_method is None:
256 return None
@@ -265,38 +258,13 @@ class GPTQMarlinConfig(QuantizationConfig):
258 return moe_quant_method
259
260 quant_method = get_linear_quant_method(
268 - self, layer, prefix, GPTQMarlinLinearMethod
261 + self, layer, prefix, AutoGPTQLinearMethod
262 )
263 if quant_method is None:
264 return None
265 quant_method.input_dtype = get_marlin_input_dtype(prefix)
266 return quant_method
267
275 - @classmethod
276 - def is_gptq_marlin_compatible(cls, quant_config: dict[str, Any]):
277 - quant_method = quant_config.get("quant_method", "").lower()
278 - num_bits = quant_config.get("bits")
279 - group_size = quant_config.get("group_size")
280 - sym = quant_config.get("sym")
281 - desc_act = quant_config.get("desc_act")
282 -
283 - if not (current_platform.is_cuda() or current_platform.is_cpu()):
284 - return False
285 -
286 - if quant_method != "gptq":
287 - return False
288 -
289 - # Marlin conversion is only valid if required properties are found
290 - if num_bits is None or group_size is None or sym is None or desc_act is None:
291 - return False
292 -
293 - if (num_bits, sym) not in cls.TYPE_MAP:
294 - return False
295 -
296 - return check_marlin_supported(
297 - quant_type=cls.TYPE_MAP[(num_bits, sym)], group_size=group_size
298 - )
299 -
268 def apply_vllm_mapper(self, hf_to_vllm_mapper):
269 if self.modules_in_block_to_quantize is not None:
270 self.modules_in_block_to_quantize = hf_to_vllm_mapper.apply_list(
@@ -331,16 +299,16 @@ class GPTQMarlinConfig(QuantizationConfig):
299 self.modules_in_block_to_quantize = list(quant_layers)
300
301
334 -class GPTQMarlinLinearMethod(LinearMethodBase):
335 - """Linear method for GPTQ Marlin.
302 +class AutoGPTQLinearMethod(LinearMethodBase):
303 + """Linear method for AutoGPTQ using Marlin kernels.
304
305 Args:
338 - quant_config: The GPTQ Marlin quantization config.
306 + quant_config: The AutoGPTQ quantization config.
307 """
308
309 _kernel_backends_being_used: set[str] = set()
310
343 - def __init__(self, quant_config: GPTQMarlinConfig) -> None:
311 + def __init__(self, quant_config: AutoGPTQConfig) -> None:
312 self.quant_config = quant_config
313 self.input_dtype = None
314 self.quant_type = self.quant_config.quant_type
@@ -382,7 +350,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
350 kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
351
352 if kernel_type.__name__ not in self._kernel_backends_being_used:
385 - logger.info("Using %s for GPTQMarlinLinearMethod", kernel_type.__name__)
353 + logger.info("Using %s for AutoGPTQLinearMethod", kernel_type.__name__)
354 self._kernel_backends_being_used.add(kernel_type.__name__)
355
356 # Normalize group_size
@@ -492,12 +460,12 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
460 return self.kernel.apply_weights(layer, x, bias)
461
462
495 -class GPTQMarlinMoEMethod(FusedMoEMethodBase):
463 +class AutoGPTQMoEMethod(FusedMoEMethodBase):
464 """MoE Marlin method with quantization."""
465
466 def __init__(
467 self,
500 - quant_config: GPTQMarlinConfig,
468 + quant_config: AutoGPTQConfig,
469 moe: FusedMoEConfig,
470 ) -> None:
471 super().__init__(moe)
@@ -509,7 +477,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
477 quant_type = scalar_types.uint8b128
478 scale = kInt8StaticGroupScale
479 else:
512 - raise ValueError("GPTQMarlinMoEMethod only supports int4 and int8 now.")
480 + raise ValueError("AutoGPTQMoEMethod only supports int4 and int8 now.")
481 self.input_dtype = None
482 self.use_marlin = True
483 weight_key = QuantKey(quant_type, scale)
vllm/model_executor/layers/quantization/gptq.py deleted
-399
@@ -1,399 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import enum
5 -from enum import Enum
6 -from fractions import Fraction
7 -from typing import TYPE_CHECKING, Any, Union
8 -
9 -import torch
10 -from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
11 -from torch.nn.parameter import Parameter
12 -from transformers import PretrainedConfig
13 -
14 -from vllm import _custom_ops as ops
15 -from vllm.logger import init_logger
16 -from vllm.model_executor.layers.fused_moe import RoutedExperts
17 -from vllm.model_executor.layers.linear import LinearMethodBase
18 -from vllm.model_executor.layers.quantization.base_config import (
19 - QuantizationConfig,
20 - QuantizeMethodBase,
21 -)
22 -from vllm.model_executor.layers.quantization.utils.gptq_utils import (
23 - get_linear_quant_method,
24 -)
25 -from vllm.model_executor.parameter import (
26 - ChannelQuantScaleParameter,
27 - GroupQuantScaleParameter,
28 - PackedColumnParameter,
29 - PackedvLLMParameter,
30 - RowvLLMParameter,
31 -)
32 -from vllm.transformers_utils.config import get_safetensors_params_metadata
33 -from vllm.utils.collection_utils import is_list_of
34 -
35 -if TYPE_CHECKING:
36 - from vllm.model_executor.layers.quantization import QuantizationMethods
37 - from vllm.model_executor.models.utils import WeightsMapper
38 -else:
39 - QuantizationMethods = str
40 -
41 -logger = init_logger(__name__)
42 -
43 -
44 -class GPTQConfig(QuantizationConfig):
45 - """Config class for GPTQ.
46 -
47 - Reference: https://arxiv.org/abs/2210.17323
48 - """
49 -
50 - def __init__(
51 - self,
52 - weight_bits: int,
53 - group_size: int,
54 - desc_act: bool,
55 - lm_head_quantized: bool,
56 - dynamic: dict[str, dict[str, int | bool]],
57 - autoround_version: str = "",
58 - modules_in_block_to_quantize: list[str] | None = None,
59 - checkpoint_format: str = "",
60 - ) -> None:
61 - # GPTQModel use `dynamic` config property to allow per module
62 - # quantization config so each module can be individually optimized.
63 - # Format is dict[str, dict] where key is a regex string that can
64 - # perform both positive ("+:" prefixed) or negative ("-:" prefixed)
65 - # matching of a module.
66 - # Default to positive match, override base quant config mode, if no
67 - # prefix is used. Value is in dict format of field key and override
68 - # value.
69 - # Negative matching will skip quantization init for this module
70 - # entirely:
71 - # non-quantized inference. More details and quantization examples can be
72 - # found at: https://github.com/ModelCloud/GPTQModel
73 - # Example:
74 - # # last 1/2 of the layers 10-21 has 8bit vs 4bit for 0-9
75 - # # last 1/4 of the layers 16-21 has 8bit and group_size 64
76 - # dynamic = {
77 - # #`.*\.` matches the layers_node prefix
78 - # # positive match layer 10-15
79 - # r"+:.*\.(?:1[0-5])\..*": {"bits": 8,},
80 - # # positive match layer 16-21
81 - # r"+:.*\.(?:1[6-9]|20|21)\..*": {"bits": 8, "group_size": 64,},
82 - # r"-:.*\.moe\..*": {}, # negative match (skip) all `moe` layers
83 - # }
84 - super().__init__()
85 - self.dynamic = dynamic
86 -
87 - self.weight_bits = weight_bits
88 - self.group_size = group_size
89 - self.desc_act = desc_act
90 - self.lm_head_quantized = lm_head_quantized
91 - self.pack_factor = Fraction(32, self.weight_bits)
92 - if self.weight_bits not in [2, 3, 4, 8]:
93 - raise ValueError(
94 - "Currently, only 2/3/4/8-bit weight quantization is "
95 - f"supported for GPTQ, but got {self.weight_bits} bits."
96 - )
97 - # Somehow gptq_gemm 4-bit is buggy, maybe fix it in the future.
98 - # For now, show a warning, since gptq_marlin will be used by default.
99 - if self.weight_bits == 4:
100 - logger.warning_once(
101 - "Currently, the 4-bit gptq_gemm kernel for GPTQ is buggy. "
102 - "Please switch to gptq_marlin."
103 - )
104 -
105 - self.modules_in_block_to_quantize = modules_in_block_to_quantize or []
106 -
107 - # used to identify GPTQ model quantized by autoround
108 - self.autoround_version = autoround_version
109 -
110 - # GPTQ v1 and v2 format deals with zero points differently.
111 - # Currently GPTQModel stores v1 format checkpoints by default,
112 - # but provides the option to set `format="gptq_v2"` in `QuantizeConfig`.
113 - self.checkpoint_format = checkpoint_format
114 -
115 - def __repr__(self) -> str:
116 - return (
117 - f"GPTQConfig(weight_bits={self.weight_bits}, "
118 - f"group_size={self.group_size}, "
119 - f"desc_act={self.desc_act}), "
120 - f"lm_head_quantized={self.lm_head_quantized}, "
121 - f"dynamic={self.dynamic}, "
122 - f"modules_in_block_to_quantize={self.modules_in_block_to_quantize}), "
123 - f"checkpoint_format={self.checkpoint_format})"
124 - )
125 -
126 - @classmethod
127 - def get_name(cls) -> QuantizationMethods:
128 - return "gptq"
129 -
130 - @classmethod
131 - def get_supported_act_dtypes(cls) -> list[torch.dtype]:
132 - return [torch.half]
133 -
134 - @classmethod
135 - # Need to figure it out
136 - def get_min_capability(cls) -> int:
137 - return 60
138 -
139 - @classmethod
140 - def get_config_filenames(cls) -> list[str]:
141 - return ["quantize_config.json"]
142 -
143 - @classmethod
144 - def from_config(cls, config: dict[str, Any]) -> "GPTQConfig":
145 - dynamic = cls.get_from_keys_or(config, ["dynamic"], default={})
146 - dynamic = {} if dynamic is None else dynamic
147 -
148 - weight_bits = cls.get_from_keys(config, ["bits"])
149 - group_size = cls.get_from_keys(config, ["group_size"])
150 - desc_act = cls.get_from_keys(config, ["desc_act"])
151 - lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
152 - autoround_version = cls.get_from_keys_or(
153 - config, ["autoround_version"], default=""
154 - )
155 - modules_in_block_to_quantize = cls.get_from_keys_or(
156 - config, ["modules_in_block_to_quantize"], default=None
157 - )
158 - checkpoint_format = cls.get_from_keys_or(
159 - config, ["checkpoint_format"], default=""
160 - )
161 - return cls(
162 - weight_bits,
163 - group_size,
164 - desc_act,
165 - lm_head_quantized,
166 - dynamic,
167 - autoround_version,
168 - modules_in_block_to_quantize,
169 - checkpoint_format,
170 - )
171 -
172 - def get_quant_method(
173 - self, layer: torch.nn.Module, prefix: str
174 - ) -> Union["GPTQLinearMethod", "QuantizeMethodBase"] | None:
175 - if isinstance(layer, RoutedExperts):
176 - # GPTQ MoE support: fall back to MoeWNA16 for broad compatibility
177 - from .moe_wna16 import MoeWNA16Config
178 -
179 - # TODO: maybe update this for GPTQv2 format checkpoints
180 - config = {
181 - "quant_method": "gptq",
182 - "bits": self.weight_bits,
183 - "group_size": self.group_size,
184 - "sym": True, # GPTQ typically uses symmetric quantization
185 - "lm_head": False,
186 - }
187 - return MoeWNA16Config.from_config(config).get_quant_method(layer, prefix)
188 -
189 - return get_linear_quant_method(self, layer, prefix, GPTQLinearMethod)
190 -
191 - def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
192 - if self.modules_in_block_to_quantize is not None:
193 - self.modules_in_block_to_quantize = hf_to_vllm_mapper.apply_list(
194 - self.modules_in_block_to_quantize
195 - )
196 -
197 - def maybe_update_config(
198 - self,
199 - model_name: str,
200 - hf_config: PretrainedConfig | None = None,
201 - revision: str | None = None,
202 - ):
203 - if self.modules_in_block_to_quantize:
204 - if is_list_of(self.modules_in_block_to_quantize, list):
205 - # original modules_in_block_to_quantize: list[list[str]]
206 - # flatten original modules_in_block_to_quantize
207 - self.modules_in_block_to_quantize = [
208 - item
209 - for sublist in self.modules_in_block_to_quantize
210 - for item in sublist
211 - ]
212 - return
213 -
214 - unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]
215 - metadata = get_safetensors_params_metadata(model_name, revision=revision)
216 - quant_layers: set[str] = {
217 - param_name.rsplit(".", 1)[0]
218 - for param_name, info in metadata.items()
219 - if (dtype := info.get("dtype", None))
220 - and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes
221 - }
222 - self.modules_in_block_to_quantize = list(quant_layers)
223 -
224 -
225 -class ExllamaState(Enum):
226 - UNUSED = enum.auto()
227 - UNINITIALIZED = enum.auto()
228 - READY = enum.auto()
229 -
230 -
231 -class GPTQLinearMethod(LinearMethodBase):
232 - """Linear method for GPTQ.
233 -
234 - Args:
235 - quant_config: The GPTQ quantization config.
236 - """
237 -
238 - def __init__(self, quant_config: GPTQConfig):
239 - self.quant_config = quant_config
240 -
241 - # GPTQ v1 and v2 format deals with zero points differently
242 - self.use_v2_format = quant_config.checkpoint_format == "gptq_v2"
243 -
244 - def create_weights(
245 - self,
246 - layer: torch.nn.Module,
247 - input_size_per_partition: int,
248 - output_partition_sizes: list[int],
249 - input_size: int,
250 - output_size: int,
251 - params_dtype: torch.dtype,
252 - **extra_weight_attrs,
253 - ):
254 - del output_size # Unused.
255 - weight_loader = extra_weight_attrs.get("weight_loader")
256 - if input_size_per_partition % self.quant_config.group_size != 0:
257 - raise ValueError(
258 - "The input size is not aligned with the quantized "
259 - "weight shape. This can be caused by too large "
260 - "tensor parallel size."
261 - )
262 - output_size_per_partition = sum(output_partition_sizes)
263 - if output_size_per_partition % self.quant_config.pack_factor.numerator != 0:
264 - raise ValueError(
265 - "The output size is not aligned with the quantized "
266 - "weight shape. This can be caused by too large "
267 - "tensor parallel size."
268 - )
269 -
270 - if self.quant_config.group_size != -1:
271 - group_size = self.quant_config.group_size
272 - else:
273 - group_size = input_size
274 - exllama_state = ExllamaState.UNINITIALIZED
275 - scale_and_zero_size = input_size // group_size
276 - scale_and_zero_input_dim = None
277 - if (
278 - input_size != input_size_per_partition
279 - and self.quant_config.group_size != -1
280 - ):
281 - # For act-order models, we cannot use Exllama for row parallel layer
282 - if self.quant_config.desc_act:
283 - exllama_state = ExllamaState.UNUSED
284 - else:
285 - # we need to partition qzeros and scales for exllama kernel
286 - scale_and_zero_size = input_size_per_partition // group_size
287 - scale_and_zero_input_dim = 0
288 -
289 - qweight = PackedvLLMParameter(
290 - data=torch.empty(
291 - input_size_per_partition // self.quant_config.pack_factor,
292 - output_size_per_partition,
293 - dtype=torch.int32,
294 - ),
295 - input_dim=0,
296 - output_dim=1,
297 - packed_dim=0,
298 - packed_factor=self.quant_config.pack_factor,
299 - weight_loader=weight_loader,
300 - )
301 -
302 - g_idx = RowvLLMParameter(
303 - data=torch.tensor(
304 - [
305 - i // self.quant_config.group_size
306 - for i in range(input_size_per_partition)
307 - ],
308 - dtype=torch.int32,
309 - ),
310 - input_dim=0,
311 - weight_loader=weight_loader,
312 - )
313 - qzeros_args = {
314 - "data": torch.empty(
315 - scale_and_zero_size,
316 - output_size_per_partition // self.quant_config.pack_factor,
317 - dtype=torch.int32,
318 - ),
319 - "weight_loader": weight_loader,
320 - }
321 - weight_scale_args = {
322 - "data": torch.empty(
323 - scale_and_zero_size,
324 - output_size_per_partition,
325 - dtype=params_dtype,
326 - ),
327 - "weight_loader": weight_loader,
328 - }
329 - if scale_and_zero_input_dim is None:
330 - scales = ChannelQuantScaleParameter(output_dim=1, **weight_scale_args)
331 - qzeros = PackedColumnParameter(
332 - output_dim=1,
333 - packed_dim=1,
334 - packed_factor=self.quant_config.pack_factor,
335 - **qzeros_args,
336 - )
337 -
338 - else:
339 - scales = GroupQuantScaleParameter(
340 - output_dim=1, input_dim=0, **weight_scale_args
341 - )
342 - qzeros = PackedvLLMParameter(
343 - input_dim=0,
344 - output_dim=1,
345 - packed_dim=1,
346 - packed_factor=self.quant_config.pack_factor,
347 - **qzeros_args,
348 - )
349 -
350 - layer.register_parameter("qweight", qweight)
351 - layer.register_parameter("g_idx", g_idx)
352 - layer.register_parameter("qzeros", qzeros)
353 - layer.register_parameter("scales", scales)
354 -
355 - layer.exllama_state = exllama_state
356 -
357 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
358 - # for torch.compile
359 - layer.qzeros = Parameter(layer.qzeros.data, requires_grad=False)
360 - layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
361 - layer.g_idx = Parameter(layer.g_idx.data, requires_grad=False)
362 - layer.scales = Parameter(layer.scales.data, requires_grad=False)
363 -
364 - # exllama needs to shuffle the weight after the weight is loaded
365 - # here we do the shuffle on first forward pass
366 - if layer.exllama_state == ExllamaState.UNINITIALIZED:
367 - if self.quant_config.desc_act:
368 - layer.g_idx.data = torch.argsort(layer.g_idx).to(torch.int)
369 - else:
370 - layer.g_idx.data = torch.empty(
371 - (0,), dtype=torch.int, device=layer.g_idx.device
372 - )
373 - layer.exllama_state = ExllamaState.READY
374 - ops.gptq_shuffle(layer.qweight, layer.g_idx, self.quant_config.weight_bits)
375 -
376 - def apply(
377 - self,
378 - layer: torch.nn.Module,
379 - x: torch.Tensor,
380 - bias: torch.Tensor | None = None,
381 - ) -> torch.Tensor:
382 - out_shape = x.shape[:-1] + (layer.qweight.shape[-1],)
383 - reshaped_x = x.reshape(-1, x.shape[-1])
384 -
385 - # GPTQ v1 and v2 format checkpoints deals with zero points differently,
386 - # and require different gemm kernels.
387 - output = ops.gptq_gemm(
388 - reshaped_x,
389 - layer.qweight,
390 - layer.qzeros,
391 - layer.scales,
392 - layer.g_idx,
393 - layer.exllama_state == ExllamaState.READY,
394 - self.use_v2_format,
395 - self.quant_config.weight_bits,
396 - )
397 - if bias is not None:
398 - output.add_(bias)
399 - return output.reshape(out_shape)
vllm/model_executor/layers/quantization/inc.py
+12 -21
@@ -356,13 +356,13 @@ class INCConfig(QuantizationConfig):
356 else:
357 use_marlin = False
358 if use_marlin:
359 - from vllm.model_executor.layers.quantization.gptq_marlin import (
360 - GPTQMarlinConfig,
361 - GPTQMarlinLinearMethod,
362 - GPTQMarlinMoEMethod,
359 + from vllm.model_executor.layers.quantization.auto_gptq import (
360 + AutoGPTQConfig,
361 + AutoGPTQLinearMethod,
362 + AutoGPTQMoEMethod,
363 )
364
365 - quant_args_marlin = GPTQMarlinConfig(
365 + quant_args_marlin = AutoGPTQConfig(
366 weight_bits=weight_bits,
367 group_size=group_size,
368 is_sym=sym,
@@ -371,23 +371,10 @@ class INCConfig(QuantizationConfig):
371 dynamic={},
372 full_config={},
373 )
374 - else:
375 - from vllm.model_executor.layers.quantization.gptq import (
376 - GPTQConfig,
377 - GPTQLinearMethod,
378 - )
379 -
380 - quant_args = GPTQConfig(
381 - weight_bits=weight_bits,
382 - group_size=group_size,
383 - lm_head_quantized=False,
384 - desc_act=False,
385 - dynamic={},
386 - )
374
375 if isinstance(layer, RoutedExperts):
376 if use_marlin:
390 - return GPTQMarlinMoEMethod(quant_args_marlin, layer.moe_config)
377 + return AutoGPTQMoEMethod(quant_args_marlin, layer.moe_config)
378 else:
379 from vllm.model_executor.layers.quantization.moe_wna16 import (
380 MoeWNA16Config,
@@ -406,9 +393,13 @@ class INCConfig(QuantizationConfig):
393
394 if isinstance(layer, (LinearBase, ParallelLMHead)):
395 if use_marlin:
409 - return GPTQMarlinLinearMethod(quant_args_marlin)
396 + return AutoGPTQLinearMethod(quant_args_marlin)
397 else:
411 - return GPTQLinearMethod(quant_args)
398 + raise NotImplementedError(
399 + f"INC quantization with bits={weight_bits}, sym={sym} "
400 + "is not supported. Only 4-bit and 8-bit symmetric "
401 + "quantization is supported with Marlin kernels."
402 + )
403
404 return None
405
vllm/model_executor/layers/quantization/moe_wna16.py
+7 -14
@@ -60,10 +60,9 @@ class MoeWNA16Config(QuantizationConfig):
60 # Avoid circular import
61 from vllm.model_executor.layers.quantization.awq import AWQConfig
62 from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
63 - from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinConfig
63
64 if self.linear_quant_method == "gptq":
66 - self.use_marlin = GPTQMarlinConfig.is_gptq_marlin_compatible(full_config)
65 + pass
66 elif self.linear_quant_method in ("awq", "awq_marlin"):
67 capability_tuple = current_platform.get_device_capability()
68 device_capability = (
@@ -172,24 +171,18 @@ class MoeWNA16Config(QuantizationConfig):
171 return UnquantizedLinearMethod()
172 elif isinstance(layer, LinearBase):
173 # Avoid circular import
174 + from vllm.model_executor.layers.quantization.auto_gptq import (
175 + AutoGPTQConfig,
176 + )
177 from vllm.model_executor.layers.quantization.awq import AWQConfig
178 from vllm.model_executor.layers.quantization.awq_marlin import (
179 AWQMarlinConfig,
180 )
179 - from vllm.model_executor.layers.quantization.gptq import GPTQConfig
180 - from vllm.model_executor.layers.quantization.gptq_marlin import (
181 - GPTQMarlinConfig,
182 - )
181
182 if self.linear_quant_method == "gptq":
185 - if self.use_marlin:
186 - return GPTQMarlinConfig.from_config(
187 - self.full_config
188 - ).get_quant_method(layer, prefix)
189 - else:
190 - return GPTQConfig.from_config(self.full_config).get_quant_method(
191 - layer, prefix
192 - )
183 + return AutoGPTQConfig.from_config(self.full_config).get_quant_method(
184 + layer, prefix
185 + )
186 elif self.linear_quant_method in ("awq", "awq_marlin"):
187 if self.use_marlin and check_marlin_supports_layer(
188 layer, self.group_size
vllm/model_executor/layers/quantization/utils/gptq_utils.py
+17 -28
@@ -2,7 +2,6 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from collections.abc import Mapping
4 from copy import deepcopy
5 -from fractions import Fraction
5 from types import MappingProxyType
6 from typing import TYPE_CHECKING
7
@@ -16,16 +15,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
15 )
16
17 if TYPE_CHECKING:
19 - from ..gptq import GPTQConfig
20 - from ..gptq_marlin import GPTQMarlinConfig
18 + from ..auto_gptq import AutoGPTQConfig
19 else:
22 - GPTQConfig = object
23 - GPTQMarlinConfig = object
20 + AutoGPTQConfig = object
21
22
23 # Match dynamic rules with module name (prefix) and override quantize
24 # config if module (prefix) matches a rule
28 -def override_config(config: GPTQConfig | GPTQMarlinConfig, prefix: str):
25 +def override_config(config: AutoGPTQConfig, prefix: str):
26 weight_bits = get_dynamic_override(config, prefix, "bits", config.weight_bits)
27 if isinstance(weight_bits, int):
28 config.weight_bits = weight_bits
@@ -36,31 +33,23 @@ def override_config(config: GPTQConfig | GPTQMarlinConfig, prefix: str):
33 if isinstance(desc_act, bool):
34 config.desc_act = desc_act
35
39 - config.pack_factor = Fraction(32, config.weight_bits) # packed into int32
40 - if config.get_name() == "gptq_marlin":
41 - assert isinstance(config, GPTQMarlinConfig)
42 - is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
43 - if isinstance(is_sym, bool):
44 - config.is_sym = is_sym
45 -
46 - if (config.weight_bits, config.is_sym) not in config.TYPE_MAP:
47 - raise ValueError(
48 - "Unsupported quantization config: "
49 - f"bits={config.weight_bits}, sym={config.is_sym}"
50 - )
36 + config.pack_factor = 32 // config.weight_bits # packed into int32
37 + assert isinstance(config, AutoGPTQConfig)
38 + is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
39 + if isinstance(is_sym, bool):
40 + config.is_sym = is_sym
41
52 - config.quant_type = config.TYPE_MAP[(config.weight_bits, config.is_sym)]
53 - elif config.get_name() == "gptq":
54 - assert isinstance(config, GPTQConfig)
55 - if config.weight_bits not in [2, 3, 4, 8]:
56 - raise ValueError(
57 - "Currently, only 2/3/4/8-bit weight quantization is "
58 - f"supported for GPTQ, but got {config.weight_bits} bits."
59 - )
42 + if (config.weight_bits, config.is_sym) not in config.TYPE_MAP:
43 + raise ValueError(
44 + "Unsupported quantization config: "
45 + f"bits={config.weight_bits}, sym={config.is_sym}"
46 + )
47 +
48 + config.quant_type = config.TYPE_MAP[(config.weight_bits, config.is_sym)]
49
50
51 def get_dynamic_override(
63 - config: GPTQConfig | GPTQMarlinConfig,
52 + config: AutoGPTQConfig,
53 layer_name: str,
54 key: str | None = None,
55 default_value: int | bool | None = None,
@@ -126,7 +115,7 @@ def is_layer_gptq_quantized(
115
116
117 def get_linear_quant_method(
129 - config: GPTQConfig | GPTQMarlinConfig,
118 + config: AutoGPTQConfig,
119 layer: torch.nn.Module,
120 prefix: str,
121 linear_method_cls: type,
vllm/model_executor/models/openpangu_vl.py
+2 -3
@@ -44,8 +44,7 @@ from vllm.model_executor.layers.linear import (
44 RowParallelLinear,
45 )
46 from vllm.model_executor.layers.quantization import QuantizationConfig
47 -from vllm.model_executor.layers.quantization.gptq import GPTQConfig
48 -from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinConfig
47 +from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
48 from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
49 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50 from vllm.model_executor.models.interfaces import (
@@ -883,7 +882,7 @@ class OpenPanguVLForConditionalGeneration(
882 self.image_std = tuple(image_processor.image_std)
883
884 def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
886 - if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
885 + if isinstance(quant_config, AutoGPTQConfig):
886 return None
887 return quant_config
888
vllm/platforms/rocm.py
+2 -1
@@ -407,7 +407,8 @@ class RocmPlatform(Platform):
407 "awq",
408 "awq_marlin", # will be overwritten with awq
409 "gptq",
410 - "gptq_marlin", # will be overwritten with gptq
410 + "gptq_marlin",
411 + "auto_gptq",
412 "fp8",
413 "deepseek_v4_fp8",
414 "compressed-tensors",