A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc] Add support for new autogptq checkpoint_format (#3689)
Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>
Qubitium committed
Apr 2, 2024 at 07:32 UTC
7d4e1b85e78574acb5c682fa9fe1d3dfa5f092d7
2 files changed
+83
-13
tests/quantization/test_autogptq_marlin_configs.py
new
+68
@@ -0,0 +1,68 @@
1
+"""Tests whether Marlin models can be loaded from the autogptq config.
2
+
3
+Run `pytest tests/quantization/test_autogptq_marlin_configs.py --forked`.
4
+"""
5
+
6
+from dataclasses import dataclass
7
+
8
+import pytest
9
+
10
+from vllm.config import ModelConfig
11
+
12
+
13
+@dataclass
14
+class ModelPair:
15
+ model_marlin: str
16
+ model_gptq: str
17
+
18
+
19
+# Model Id // Expected Kernel
20
+MODELS_QUANT_TYPE = [
21
+ # compat: autogptq <=0.7.1 is_marlin_format: bool
22
+ ("neuralmagic/TinyLlama-1.1B-Chat-v1.0-marlin", "marlin"),
23
+ ("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq"),
24
+ # compat: autogptq >=0.8.0 use checkpoint_format: str
25
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-Marlin-4bit", "marlin"),
26
+ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq")
27
+]
28
+
29
+
30
+@pytest.mark.parametrize("model_quant_type", MODELS_QUANT_TYPE)
31
+def test_auto_gptq(model_quant_type: str, ) -> None:
32
+ model_path, quant_type = model_quant_type
33
+
34
+ model_config_no_quant_arg = ModelConfig(
35
+ model_path,
36
+ model_path,
37
+ tokenizer_mode="auto",
38
+ trust_remote_code=False,
39
+ download_dir=None,
40
+ load_format="dummy",
41
+ seed=0,
42
+ dtype="float16",
43
+ revision=None,
44
+ quantization=None # case 1
45
+ )
46
+
47
+ model_config_quant_arg = ModelConfig(
48
+ model_path,
49
+ model_path,
50
+ tokenizer_mode="auto",
51
+ trust_remote_code=False,
52
+ download_dir=None,
53
+ load_format="dummy",
54
+ seed=0,
55
+ dtype="float16",
56
+ revision=None,
57
+ quantization="gptq" # case 2
58
+ )
59
+
60
+ assert model_config_no_quant_arg.quantization == quant_type, (
61
+ f"Expected quant_type == {quant_type} for {model_path}, "
62
+ f"but found {model_config_no_quant_arg.quantization} "
63
+ "for no --quantization None case")
64
+
65
+ assert model_config_quant_arg.quantization == quant_type, (
66
+ f"Expected quant_type == {quant_type} for {model_path}, "
67
+ f"but found {model_config_quant_arg.quantization} "
68
+ "for --quantization gptq case")
vllm/config.py
+15
-13
@@ -171,26 +171,28 @@ class ModelConfig:
171
self.quantization = self.quantization.lower()
172
173
# Parse quantization method from the HF model config, if available.
174
- hf_quant_config = getattr(self.hf_config, "quantization_config", None)
175
- if hf_quant_config is not None:
176
- hf_quant_method = str(hf_quant_config["quant_method"]).lower()
177
-
178
- # If the GPTQ model is serialized in marlin format, use marlin.
179
- if (hf_quant_method == "gptq"
180
- and "is_marlin_format" in hf_quant_config
181
- and hf_quant_config["is_marlin_format"]):
174
+ quant_cfg = getattr(self.hf_config, "quantization_config", None)
175
+ if quant_cfg is not None:
176
+ quant_method = quant_cfg.get("quant_method", "").lower()
177
+ # compat: autogptq >=0.8.0 use checkpoint_format: str
178
+ # compat: autogptq <=0.7.1 is_marlin_format: bool
179
+ is_format_marlin = (quant_cfg.get("checkpoint_format") == "marlin"
180
+ or quant_cfg.get("is_marlin_format", False))
181
+
182
+ # Use marlin if the GPTQ model is serialized in marlin format.
183
+ if quant_method == "gptq" and is_format_marlin:
184
logger.info("The model is serialized in Marlin format. "
185
"Using Marlin kernel.")
184
- hf_quant_method = "marlin"
186
+ quant_method = "marlin"
187
if self.quantization == "gptq":
186
- self.quantization = hf_quant_method
188
+ self.quantization = quant_method
189
190
if self.quantization is None:
189
- self.quantization = hf_quant_method
190
- elif self.quantization != hf_quant_method:
191
+ self.quantization = quant_method
192
+ elif self.quantization != quant_method:
193
raise ValueError(
194
"Quantization method specified in the model config "
193
- f"({hf_quant_method}) does not match the quantization "
195
+ f"({quant_method}) does not match the quantization "
196
f"method specified in the `quantization` argument "
197
f"({self.quantization}).")
198