A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Hardware][XPU] AWQ/GPTQ support for xpu backend (#10107)
Signed-off-by: yan ma <yan.ma@intel.com>
Yan Ma committed
Nov 19, 2024 at 02:18 UTC
6b2d25efc78f21867ca37e3f707c5a94f906478f
7 files changed
+146
-52
docs/source/quantization/supported_hardware.rst
+4
-4
@@ -27,7 +27,7 @@ The table below shows the compatibility of various quantization implementations
27
- ✅︎
28
- ✅︎
29
- ✗
30
- - ✗
30
+ - ✅︎
31
- ✅︎
32
- ✗
33
- ✗
@@ -38,8 +38,8 @@ The table below shows the compatibility of various quantization implementations
38
- ✅︎
39
- ✅︎
40
- ✗
41
- - ✗
42
- - ✗
41
+ - ✅︎
42
+ - ✅︎
43
- ✗
44
- ✗
45
* - Marlin (GPTQ/AWQ/FP8)
@@ -129,4 +129,4 @@ Notes:
129
130
Please note that this compatibility chart may be subject to change as vLLM continues to evolve and expand its support for different hardware platforms and quantization methods.
131
132
-For the most up-to-date information on hardware support and quantization methods, please check the `quantization directory <https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization>`_ or consult with the vLLM development team.
\ No newline at end of file
132
+For the most up-to-date information on hardware support and quantization methods, please check the `quantization directory <https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization>`_ or consult with the vLLM development team.
tests/quantization/test_ipex_quant.py
+6
-4
@@ -1,5 +1,5 @@
1
"""Test model set-up and inference for quantized HF models supported
2
- on the CPU backend using IPEX (including AWQ).
2
+ on the CPU/GPU backend using IPEX (including AWQ/GPTQ).
3
4
Validating the configuration and printing results for manual checking.
5
@@ -11,13 +11,15 @@ import pytest
11
from vllm.platforms import current_platform
12
13
MODELS = [
14
- "casperhansen/llama-3-8b-instruct-awq",
14
+ "AMead10/Llama-3.2-1B-Instruct-AWQ",
15
+ "shuyuej/Llama-3.2-1B-Instruct-GPTQ", # with g_idx
16
]
17
DTYPE = ["bfloat16"]
18
19
19
-@pytest.mark.skipif(not current_platform.is_cpu(),
20
- reason="only supports the CPU backend.")
20
+@pytest.mark.skipif(not current_platform.is_cpu()
21
+ and not current_platform.is_xpu(),
22
+ reason="only supports Intel CPU/XPU backend.")
23
@pytest.mark.parametrize("model", MODELS)
24
@pytest.mark.parametrize("dtype", DTYPE)
25
def test_ipex_quant(vllm_runner, model, dtype):
vllm/model_executor/layers/linear.py
+1
-1
@@ -27,7 +27,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
27
"AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
28
"MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
29
"TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod",
30
- "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod"
30
+ "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod", "IPEXGPTQLinearMethod"
31
]
32
33
vllm/model_executor/layers/quantization/gptq.py
-1
@@ -210,7 +210,6 @@ class GPTQLinearMethod(LinearMethodBase):
210
211
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
212
# for torch.compile
213
- layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
213
layer.qzeros = Parameter(layer.qzeros.data, requires_grad=False)
214
layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
215
layer.g_idx = Parameter(layer.g_idx.data, requires_grad=False)
vllm/model_executor/layers/quantization/gptq_marlin.py
+4
@@ -23,6 +23,7 @@ from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
23
PackedColumnParameter,
24
PackedvLLMParameter,
25
RowvLLMParameter)
26
+from vllm.platforms import current_platform
27
from vllm.scalar_type import scalar_types
28
29
logger = init_logger(__name__)
@@ -134,6 +135,9 @@ class GPTQMarlinConfig(QuantizationConfig):
135
sym = quant_config.get("sym")
136
desc_act = quant_config.get("desc_act")
137
138
+ if not current_platform.is_cuda():
139
+ return False
140
+
141
if quant_method != "gptq":
142
return False
143
vllm/model_executor/layers/quantization/ipex_quant.py
+128
-41
@@ -2,21 +2,26 @@ from typing import Any, Dict, List, Optional
2
3
import torch
4
5
-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
6
-from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
5
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
6
+ UnquantizedLinearMethod)
7
+from vllm.model_executor.layers.quantization.awq import (AWQLinearMethod,
8
+ is_layer_skipped_awq)
9
from vllm.model_executor.layers.quantization.base_config import (
10
QuantizationConfig)
11
+from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
12
from vllm.platforms import current_platform
13
14
+MIN_IPEX_VERSION = "2.5.0"
15
+
16
17
class IPEXConfig(QuantizationConfig):
13
- """INT8 quantization config class using IPEX for the CPU backend,
14
- including AWQ.
18
+ """INT8 quantization config class using IPEX for the CPU/XPU backend,
19
+ including AWQ, GPTQ.
20
"""
21
22
IPEX_QUANT_METHOD_MAP = {
23
"awq": 1,
19
- "gptq": 2,
24
+ "gptq": 0,
25
}
26
27
def __init__(
@@ -24,29 +29,30 @@ class IPEXConfig(QuantizationConfig):
29
method: str,
30
weight_bits: int,
31
group_size: int,
32
+ modules_to_not_convert: Optional[List[str]] = None,
33
+ desc_act: Optional[bool] = None,
34
+ lm_head_quantized: Optional[bool] = None,
35
) -> None:
36
self.method = method
37
self.weight_bits = weight_bits
38
self.group_size = group_size
39
+ self.modules_to_not_convert = modules_to_not_convert or []
40
+ self.desc_act = desc_act
41
+ self.lm_head_quantized = lm_head_quantized
42
self.pack_factor = 32 // self.weight_bits
43
44
if self.weight_bits not in [4]:
45
raise ValueError(f"IPEX quantization supports weight bits [4], "
46
f"but got {self.weight_bits}.")
47
37
- if self.method == "awq":
38
- self.quant_method = IPEXAWQLinearMethod
39
- else:
40
- raise ValueError(f"IPEX quantization supports [awq], "
48
+ if self.method not in ["awq", "gptq"]:
49
+ raise ValueError(f"IPEX quantization supports [awq, gptq], "
50
f"but got {self.method}.")
51
52
def __repr__(self) -> str:
44
- return (f"IPEXConfig(method={self.method}"
53
+ return (f"IPEXConfig(method={self.method},"
54
f"weight_bits={self.weight_bits}, "
46
- f"group_size={self.group_size}")
47
-
48
- def get_ipex_quant_method_id(self) -> int:
49
- return IPEXConfig.IPEX_QUANT_METHOD_MAP[self.method]
55
+ f"group_size={self.group_size})")
56
57
@classmethod
58
def get_name(cls) -> str:
@@ -70,19 +76,32 @@ class IPEXConfig(QuantizationConfig):
76
@classmethod
77
def from_config(cls, config: Dict[str, Any]) -> "IPEXConfig":
78
method = cls.get_from_keys(config, ["quant_method"]).lower()
73
- weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
74
- group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
75
- return cls(method, weight_bits, group_size)
79
+ if method == "awq":
80
+ weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
81
+ group_size = cls.get_from_keys(config,
82
+ ["q_group_size", "group_size"])
83
+ modules_to_not_convert = cls.get_from_keys_or(
84
+ config, ["modules_to_not_convert"], None)
85
+ return cls(method, weight_bits, group_size, modules_to_not_convert,
86
+ False, False)
87
+ # otherwise for gptq
88
+ weight_bits = cls.get_from_keys(config, ["bits"])
89
+ group_size = cls.get_from_keys(config, ["group_size"])
90
+ lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
91
+ default=False)
92
+ desc_act = cls.get_from_keys_or(config, ["desc_act"], default=False)
93
+ return cls(method, weight_bits, group_size, [], desc_act,
94
+ lm_head_quantized)
95
96
@classmethod
97
def override_quantization_method(cls, hf_quant_cfg,
98
user_quant) -> Optional[str]:
80
- if not current_platform.is_cpu():
99
+ if not current_platform.is_cpu() and not current_platform.is_xpu():
100
return None
101
102
quant_method = hf_quant_cfg.get("quant_method", "").lower()
103
85
- if quant_method in ["awq"]:
104
+ if quant_method in ["awq", "gptq"]:
105
return cls.get_name()
106
107
return None
@@ -90,12 +109,81 @@ class IPEXConfig(QuantizationConfig):
109
def get_quant_method(self, layer: torch.nn.Module,
110
prefix: str) -> Optional["LinearMethodBase"]:
111
if isinstance(layer, LinearBase):
93
- return self.quant_method(self)
112
+ if self.method == "awq":
113
+ if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
114
+ return UnquantizedLinearMethod()
115
+ return IPEXAWQLinearMethod(self)
116
+ if self.method == "gptq":
117
+ return IPEXGPTQLinearMethod(self)
118
return None
119
120
121
+class IPEXGPTQLinearMethod(GPTQLinearMethod):
122
+ """GPTQ linear method using IPEX for the CPU/XPU backend.
123
+ """
124
+
125
+ def __init__(self, quant_config: IPEXConfig):
126
+ self.quant_config = quant_config # type: ignore
127
+
128
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
129
+ bias = layer.bias if not layer.skip_bias_add else None
130
+
131
+ try:
132
+ import intel_extension_for_pytorch as ipex
133
+ if ipex.__version__ < MIN_IPEX_VERSION:
134
+ raise ImportError(
135
+ "intel_extension_for_pytorch version is "
136
+ "wrong. Please install "
137
+ f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}.")
138
+ except ImportError as err:
139
+ raise ImportError(
140
+ "Please install "
141
+ f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
142
+ f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
143
+ " to use IPEX-AWQ linear method.") from err
144
+ # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
145
+ # with better performance.
146
+ lowp_mode = ipex.quantization.WoqLowpMode.INT8
147
+ # The weight will be de-packed from INT4 to INT8.
148
+ weight_dtype = ipex.quantization.WoqWeightDtype.INT4
149
+ # The float activation will be quantized (dynamic, per-token) to INT8.
150
+ act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH_IC_BLOCK
151
+
152
+ qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
153
+ weight_dtype=weight_dtype,
154
+ lowp_mode=lowp_mode,
155
+ act_quant_mode=act_quant_mode,
156
+ group_size=self.quant_config.group_size,
157
+ )
158
+ layer.ipex_output_size = layer.qweight.shape[-1]
159
+ g_idx = layer.g_idx if self.quant_config.desc_act else None
160
+ layer.ipex_qlinear = ipex.llm.quantization.woq_linear. \
161
+ IPEXWeightOnlyQuantizedLinear.from_weight(
162
+ layer.qweight,
163
+ layer.scales,
164
+ layer.qzeros,
165
+ layer.qweight.size(0),
166
+ layer.ipex_output_size,
167
+ qconfig=qconfig,
168
+ g_idx=g_idx,
169
+ bias=bias,
170
+ group_size=self.quant_config.group_size,
171
+ quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["gptq"]
172
+ )
173
+
174
+ def apply(self,
175
+ layer: torch.nn.Module,
176
+ x: torch.Tensor,
177
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
178
+ reshaped_x = x.reshape(-1, x.shape[-1])
179
+ out = layer.ipex_qlinear(reshaped_x)
180
+ if bias is not None:
181
+ out.add_(bias)
182
+ return out.reshape(x.shape[:-1] + (layer.ipex_output_size, ))
183
+
184
+
185
class IPEXAWQLinearMethod(AWQLinearMethod):
98
- """AWQ linear method using IPEX for the CPU backend.
186
+ """AWQ linear method using IPEX for the CPU/XPU backend.
187
"""
188
189
def __init__(self, quant_config: IPEXConfig):
@@ -108,15 +196,16 @@ class IPEXAWQLinearMethod(AWQLinearMethod):
196
197
try:
198
import intel_extension_for_pytorch as ipex
111
- if ipex.__version__ < "2.4.0":
112
- raise ImportError("intel_extension_for_pytorch version is "
113
- "wrong. Please install "
114
- "intel_extension_for_pytorch>=2.4.0.")
199
+ if ipex.__version__ < MIN_IPEX_VERSION:
200
+ raise ImportError(
201
+ "intel_extension_for_pytorch version is "
202
+ "wrong. Please install "
203
+ f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}.")
204
except ImportError as err:
205
raise ImportError(
206
"Please install "
118
- "intel_extension_for_pytorch>=2.4.0 via "
119
- "`pip install intel_extension_for_pytorch>=2.4.0`"
207
+ f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
208
+ f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
209
" to use IPEX-AWQ linear method.") from err
210
211
# Using the compute dtype (lowp_mode) as INT8 to leverage instructions
@@ -136,19 +225,18 @@ class IPEXAWQLinearMethod(AWQLinearMethod):
225
226
layer.ipex_output_size = layer.qweight.size(
227
1) * self.quant_config.pack_factor
139
- layer.ipex_qlinear = ipex.nn.modules.weight_only_quantization.\
140
- WeightOnlyQuantizedLinear.from_weight(
141
- layer.qweight,
142
- layer.scales,
143
- layer.qzeros,
144
- layer.qweight.size(0),
145
- layer.ipex_output_size,
146
- qconfig=qconfig,
147
- bias=bias,
148
- group_size=self.quant_config.group_size,
149
- quant_method=
150
- self.quant_config.get_ipex_quant_method_id() # type: ignore
151
- )
228
+ layer.ipex_qlinear = ipex.llm.quantization.woq_linear. \
229
+ IPEXWeightOnlyQuantizedLinear.from_weight(
230
+ layer.qweight,
231
+ layer.scales,
232
+ layer.qzeros,
233
+ layer.qweight.size(0),
234
+ layer.ipex_output_size,
235
+ qconfig=qconfig,
236
+ bias=bias,
237
+ group_size=self.quant_config.group_size,
238
+ quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["awq"] # type: ignore
239
+ )
240
241
def apply(self,
242
layer: torch.nn.Module,
@@ -156,5 +244,4 @@ class IPEXAWQLinearMethod(AWQLinearMethod):
244
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
245
reshaped_x = x.reshape(-1, x.shape[-1])
246
out = layer.ipex_qlinear(reshaped_x)
159
-
247
return out.reshape(x.shape[:-1] + (layer.ipex_output_size, ))
vllm/model_executor/model_loader/loader.py
+3
-1
@@ -29,6 +29,8 @@ from vllm.envs import VLLM_USE_MODELSCOPE
29
from vllm.logger import init_logger
30
from vllm.model_executor.layers.linear import (ReplicatedLinear,
31
RowParallelLinear)
32
+from vllm.model_executor.layers.quantization.base_config import (
33
+ QuantizeMethodBase)
34
from vllm.model_executor.model_loader.tensorizer import (
35
TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
36
serialize_vllm_model, tensorizer_weights_iterator)
@@ -348,7 +350,7 @@ class DefaultModelLoader(BaseModelLoader):
350
351
for _, module in model.named_modules():
352
quant_method = getattr(module, "quant_method", None)
351
- if quant_method is not None:
353
+ if isinstance(quant_method, QuantizeMethodBase):
354
# When quant methods need to process weights after loading
355
# (for repacking, quantizing, etc), they expect parameters
356
# to be on the global target device. This scope is for the