@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6b2d25efc7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Hardware][XPU] AWQ/GPTQ support for xpu backend (#10107)

Signed-off-by: yan ma <yan.ma@intel.com>

Yan Ma committed Nov 19, 2024 at 02:18 UTC 6b2d25efc78f21867ca37e3f707c5a94f906478f
7 files changed +146 -52
docs/source/quantization/supported_hardware.rst
+4 -4
@@ -27,7 +27,7 @@ The table below shows the compatibility of various quantization implementations
27 - ✅︎
28 - ✅︎
29 - ✗
30 - - ✗
30 + - ✅︎
31 - ✅︎
32 - ✗
33 - ✗
@@ -38,8 +38,8 @@ The table below shows the compatibility of various quantization implementations
38 - ✅︎
39 - ✅︎
40 - ✗
41 - - ✗
42 - - ✗
41 + - ✅︎
42 + - ✅︎
43 - ✗
44 - ✗
45 * - Marlin (GPTQ/AWQ/FP8)
@@ -129,4 +129,4 @@ Notes:
129
130 Please note that this compatibility chart may be subject to change as vLLM continues to evolve and expand its support for different hardware platforms and quantization methods.
131
132 -For the most up-to-date information on hardware support and quantization methods, please check the `quantization directory <https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization>`_ or consult with the vLLM development team.
\ No newline at end of file
132 +For the most up-to-date information on hardware support and quantization methods, please check the `quantization directory <https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization>`_ or consult with the vLLM development team.
tests/quantization/test_ipex_quant.py
+6 -4
@@ -1,5 +1,5 @@
1 """Test model set-up and inference for quantized HF models supported
2 - on the CPU backend using IPEX (including AWQ).
2 + on the CPU/GPU backend using IPEX (including AWQ/GPTQ).
3
4 Validating the configuration and printing results for manual checking.
5
@@ -11,13 +11,15 @@ import pytest
11 from vllm.platforms import current_platform
12
13 MODELS = [
14 - "casperhansen/llama-3-8b-instruct-awq",
14 + "AMead10/Llama-3.2-1B-Instruct-AWQ",
15 + "shuyuej/Llama-3.2-1B-Instruct-GPTQ", # with g_idx
16 ]
17 DTYPE = ["bfloat16"]
18
19
19 -@pytest.mark.skipif(not current_platform.is_cpu(),
20 - reason="only supports the CPU backend.")
20 +@pytest.mark.skipif(not current_platform.is_cpu()
21 + and not current_platform.is_xpu(),
22 + reason="only supports Intel CPU/XPU backend.")
23 @pytest.mark.parametrize("model", MODELS)
24 @pytest.mark.parametrize("dtype", DTYPE)
25 def test_ipex_quant(vllm_runner, model, dtype):
vllm/model_executor/layers/linear.py
+1 -1
@@ -27,7 +27,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
27 "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
28 "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
29 "TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod",
30 - "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod"
30 + "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod", "IPEXGPTQLinearMethod"
31 ]
32
33
vllm/model_executor/layers/quantization/gptq.py
-1
@@ -210,7 +210,6 @@ class GPTQLinearMethod(LinearMethodBase):
210
211 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
212 # for torch.compile
213 - layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
213 layer.qzeros = Parameter(layer.qzeros.data, requires_grad=False)
214 layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
215 layer.g_idx = Parameter(layer.g_idx.data, requires_grad=False)
vllm/model_executor/layers/quantization/gptq_marlin.py
+4
@@ -23,6 +23,7 @@ from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
23 PackedColumnParameter,
24 PackedvLLMParameter,
25 RowvLLMParameter)
26 +from vllm.platforms import current_platform
27 from vllm.scalar_type import scalar_types
28
29 logger = init_logger(__name__)
@@ -134,6 +135,9 @@ class GPTQMarlinConfig(QuantizationConfig):
135 sym = quant_config.get("sym")
136 desc_act = quant_config.get("desc_act")
137
138 + if not current_platform.is_cuda():
139 + return False
140 +
141 if quant_method != "gptq":
142 return False
143
vllm/model_executor/layers/quantization/ipex_quant.py
+128 -41
@@ -2,21 +2,26 @@ from typing import Any, Dict, List, Optional
2
3 import torch
4
5 -from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
6 -from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
5 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
6 + UnquantizedLinearMethod)
7 +from vllm.model_executor.layers.quantization.awq import (AWQLinearMethod,
8 + is_layer_skipped_awq)
9 from vllm.model_executor.layers.quantization.base_config import (
10 QuantizationConfig)
11 +from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
12 from vllm.platforms import current_platform
13
14 +MIN_IPEX_VERSION = "2.5.0"
15 +
16
17 class IPEXConfig(QuantizationConfig):
13 - """INT8 quantization config class using IPEX for the CPU backend,
14 - including AWQ.
18 + """INT8 quantization config class using IPEX for the CPU/XPU backend,
19 + including AWQ, GPTQ.
20 """
21
22 IPEX_QUANT_METHOD_MAP = {
23 "awq": 1,
19 - "gptq": 2,
24 + "gptq": 0,
25 }
26
27 def __init__(
@@ -24,29 +29,30 @@ class IPEXConfig(QuantizationConfig):
29 method: str,
30 weight_bits: int,
31 group_size: int,
32 + modules_to_not_convert: Optional[List[str]] = None,
33 + desc_act: Optional[bool] = None,
34 + lm_head_quantized: Optional[bool] = None,
35 ) -> None:
36 self.method = method
37 self.weight_bits = weight_bits
38 self.group_size = group_size
39 + self.modules_to_not_convert = modules_to_not_convert or []
40 + self.desc_act = desc_act
41 + self.lm_head_quantized = lm_head_quantized
42 self.pack_factor = 32 // self.weight_bits
43
44 if self.weight_bits not in [4]:
45 raise ValueError(f"IPEX quantization supports weight bits [4], "
46 f"but got {self.weight_bits}.")
47
37 - if self.method == "awq":
38 - self.quant_method = IPEXAWQLinearMethod
39 - else:
40 - raise ValueError(f"IPEX quantization supports [awq], "
48 + if self.method not in ["awq", "gptq"]:
49 + raise ValueError(f"IPEX quantization supports [awq, gptq], "
50 f"but got {self.method}.")
51
52 def __repr__(self) -> str:
44 - return (f"IPEXConfig(method={self.method}"
53 + return (f"IPEXConfig(method={self.method},"
54 f"weight_bits={self.weight_bits}, "
46 - f"group_size={self.group_size}")
47 -
48 - def get_ipex_quant_method_id(self) -> int:
49 - return IPEXConfig.IPEX_QUANT_METHOD_MAP[self.method]
55 + f"group_size={self.group_size})")
56
57 @classmethod
58 def get_name(cls) -> str:
@@ -70,19 +76,32 @@ class IPEXConfig(QuantizationConfig):
76 @classmethod
77 def from_config(cls, config: Dict[str, Any]) -> "IPEXConfig":
78 method = cls.get_from_keys(config, ["quant_method"]).lower()
73 - weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
74 - group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
75 - return cls(method, weight_bits, group_size)
79 + if method == "awq":
80 + weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
81 + group_size = cls.get_from_keys(config,
82 + ["q_group_size", "group_size"])
83 + modules_to_not_convert = cls.get_from_keys_or(
84 + config, ["modules_to_not_convert"], None)
85 + return cls(method, weight_bits, group_size, modules_to_not_convert,
86 + False, False)
87 + # otherwise for gptq
88 + weight_bits = cls.get_from_keys(config, ["bits"])
89 + group_size = cls.get_from_keys(config, ["group_size"])
90 + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
91 + default=False)
92 + desc_act = cls.get_from_keys_or(config, ["desc_act"], default=False)
93 + return cls(method, weight_bits, group_size, [], desc_act,
94 + lm_head_quantized)
95
96 @classmethod
97 def override_quantization_method(cls, hf_quant_cfg,
98 user_quant) -> Optional[str]:
80 - if not current_platform.is_cpu():
99 + if not current_platform.is_cpu() and not current_platform.is_xpu():
100 return None
101
102 quant_method = hf_quant_cfg.get("quant_method", "").lower()
103
85 - if quant_method in ["awq"]:
104 + if quant_method in ["awq", "gptq"]:
105 return cls.get_name()
106
107 return None
@@ -90,12 +109,81 @@ class IPEXConfig(QuantizationConfig):
109 def get_quant_method(self, layer: torch.nn.Module,
110 prefix: str) -> Optional["LinearMethodBase"]:
111 if isinstance(layer, LinearBase):
93 - return self.quant_method(self)
112 + if self.method == "awq":
113 + if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
114 + return UnquantizedLinearMethod()
115 + return IPEXAWQLinearMethod(self)
116 + if self.method == "gptq":
117 + return IPEXGPTQLinearMethod(self)
118 return None
119
120
121 +class IPEXGPTQLinearMethod(GPTQLinearMethod):
122 + """GPTQ linear method using IPEX for the CPU/XPU backend.
123 + """
124 +
125 + def __init__(self, quant_config: IPEXConfig):
126 + self.quant_config = quant_config # type: ignore
127 +
128 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
129 + bias = layer.bias if not layer.skip_bias_add else None
130 +
131 + try:
132 + import intel_extension_for_pytorch as ipex
133 + if ipex.__version__ < MIN_IPEX_VERSION:
134 + raise ImportError(
135 + "intel_extension_for_pytorch version is "
136 + "wrong. Please install "
137 + f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}.")
138 + except ImportError as err:
139 + raise ImportError(
140 + "Please install "
141 + f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
142 + f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
143 + " to use IPEX-AWQ linear method.") from err
144 + # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
145 + # with better performance.
146 + lowp_mode = ipex.quantization.WoqLowpMode.INT8
147 + # The weight will be de-packed from INT4 to INT8.
148 + weight_dtype = ipex.quantization.WoqWeightDtype.INT4
149 + # The float activation will be quantized (dynamic, per-token) to INT8.
150 + act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH_IC_BLOCK
151 +
152 + qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
153 + weight_dtype=weight_dtype,
154 + lowp_mode=lowp_mode,
155 + act_quant_mode=act_quant_mode,
156 + group_size=self.quant_config.group_size,
157 + )
158 + layer.ipex_output_size = layer.qweight.shape[-1]
159 + g_idx = layer.g_idx if self.quant_config.desc_act else None
160 + layer.ipex_qlinear = ipex.llm.quantization.woq_linear. \
161 + IPEXWeightOnlyQuantizedLinear.from_weight(
162 + layer.qweight,
163 + layer.scales,
164 + layer.qzeros,
165 + layer.qweight.size(0),
166 + layer.ipex_output_size,
167 + qconfig=qconfig,
168 + g_idx=g_idx,
169 + bias=bias,
170 + group_size=self.quant_config.group_size,
171 + quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["gptq"]
172 + )
173 +
174 + def apply(self,
175 + layer: torch.nn.Module,
176 + x: torch.Tensor,
177 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
178 + reshaped_x = x.reshape(-1, x.shape[-1])
179 + out = layer.ipex_qlinear(reshaped_x)
180 + if bias is not None:
181 + out.add_(bias)
182 + return out.reshape(x.shape[:-1] + (layer.ipex_output_size, ))
183 +
184 +
185 class IPEXAWQLinearMethod(AWQLinearMethod):
98 - """AWQ linear method using IPEX for the CPU backend.
186 + """AWQ linear method using IPEX for the CPU/XPU backend.
187 """
188
189 def __init__(self, quant_config: IPEXConfig):
@@ -108,15 +196,16 @@ class IPEXAWQLinearMethod(AWQLinearMethod):
196
197 try:
198 import intel_extension_for_pytorch as ipex
111 - if ipex.__version__ < "2.4.0":
112 - raise ImportError("intel_extension_for_pytorch version is "
113 - "wrong. Please install "
114 - "intel_extension_for_pytorch>=2.4.0.")
199 + if ipex.__version__ < MIN_IPEX_VERSION:
200 + raise ImportError(
201 + "intel_extension_for_pytorch version is "
202 + "wrong. Please install "
203 + f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}.")
204 except ImportError as err:
205 raise ImportError(
206 "Please install "
118 - "intel_extension_for_pytorch>=2.4.0 via "
119 - "`pip install intel_extension_for_pytorch>=2.4.0`"
207 + f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
208 + f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
209 " to use IPEX-AWQ linear method.") from err
210
211 # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
@@ -136,19 +225,18 @@ class IPEXAWQLinearMethod(AWQLinearMethod):
225
226 layer.ipex_output_size = layer.qweight.size(
227 1) * self.quant_config.pack_factor
139 - layer.ipex_qlinear = ipex.nn.modules.weight_only_quantization.\
140 - WeightOnlyQuantizedLinear.from_weight(
141 - layer.qweight,
142 - layer.scales,
143 - layer.qzeros,
144 - layer.qweight.size(0),
145 - layer.ipex_output_size,
146 - qconfig=qconfig,
147 - bias=bias,
148 - group_size=self.quant_config.group_size,
149 - quant_method=
150 - self.quant_config.get_ipex_quant_method_id() # type: ignore
151 - )
228 + layer.ipex_qlinear = ipex.llm.quantization.woq_linear. \
229 + IPEXWeightOnlyQuantizedLinear.from_weight(
230 + layer.qweight,
231 + layer.scales,
232 + layer.qzeros,
233 + layer.qweight.size(0),
234 + layer.ipex_output_size,
235 + qconfig=qconfig,
236 + bias=bias,
237 + group_size=self.quant_config.group_size,
238 + quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["awq"] # type: ignore
239 + )
240
241 def apply(self,
242 layer: torch.nn.Module,
@@ -156,5 +244,4 @@ class IPEXAWQLinearMethod(AWQLinearMethod):
244 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
245 reshaped_x = x.reshape(-1, x.shape[-1])
246 out = layer.ipex_qlinear(reshaped_x)
159 -
247 return out.reshape(x.shape[:-1] + (layer.ipex_output_size, ))
vllm/model_executor/model_loader/loader.py
+3 -1
@@ -29,6 +29,8 @@ from vllm.envs import VLLM_USE_MODELSCOPE
29 from vllm.logger import init_logger
30 from vllm.model_executor.layers.linear import (ReplicatedLinear,
31 RowParallelLinear)
32 +from vllm.model_executor.layers.quantization.base_config import (
33 + QuantizeMethodBase)
34 from vllm.model_executor.model_loader.tensorizer import (
35 TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
36 serialize_vllm_model, tensorizer_weights_iterator)
@@ -348,7 +350,7 @@ class DefaultModelLoader(BaseModelLoader):
350
351 for _, module in model.named_modules():
352 quant_method = getattr(module, "quant_method", None)
351 - if quant_method is not None:
353 + if isinstance(quant_method, QuantizeMethodBase):
354 # When quant methods need to process weights after loading
355 # (for repacking, quantizing, etc), they expect parameters
356 # to be on the global target device. This scope is for the