A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Hardware][CPU] Support AWQ for CPU backend (#7515)
Li, Jiang committed
Oct 10, 2024 at 00:28 UTC
ca77dd7a44f2bc103c668560818918ac0335835a
9 files changed
+214
-7
.buildkite/run-cpu-test.sh
+8
-2
@@ -27,13 +27,19 @@ docker exec cpu-test bash -c "
27
pytest -v -s tests/models/decoder_only/language \
28
--ignore=tests/models/test_fp8.py \
29
--ignore=tests/models/decoder_only/language/test_jamba.py \
30
+ --ignore=tests/models/decoder_only/language/test_granitemoe.py \
31
--ignore=tests/models/decoder_only/language/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
32
33
# Run compressed-tensor test
34
+# docker exec cpu-test bash -c "
35
+# pytest -s -v \
36
+# tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
37
+# tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynanmic_per_token"
38
+
39
+# Run AWQ test
40
docker exec cpu-test bash -c "
41
pytest -s -v \
35
- tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
36
- tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynanmic_per_token"
42
+ tests/quantization/test_ipex_quant.py"
43
44
# online inference
45
docker exec cpu-test bash -c "
Dockerfile.cpu
+1
-1
@@ -22,7 +22,7 @@ ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/li
22
23
RUN echo 'ulimit -c 0' >> ~/.bashrc
24
25
-RUN pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/cpu/intel_extension_for_pytorch-2.4.0%2Bgitfbaa4bc-cp310-cp310-linux_x86_64.whl
25
+RUN pip install intel_extension_for_pytorch==2.4.0
26
27
WORKDIR /workspace
28
docs/source/quantization/supported_hardware.rst
+2
-2
@@ -28,7 +28,7 @@ The table below shows the compatibility of various quantization implementations
28
- ✅︎
29
- ✗
30
- ✗
31
- - ✗
31
+ - ✅︎
32
- ✗
33
- ✗
34
* - GPTQ
@@ -61,7 +61,7 @@ The table below shows the compatibility of various quantization implementations
61
- ✅︎
62
- ✗
63
- ✗
64
- - ✗
64
+ - ✅︎
65
- ✗
66
- ✗
67
* - FP8 (W8A8)
tests/quantization/test_ipex_quant.py
new
+28
@@ -0,0 +1,28 @@
1
+"""Test model set-up and inference for quantized HF models supported
2
+ on the CPU backend using IPEX (including AWQ).
3
+
4
+ Validating the configuration and printing results for manual checking.
5
+
6
+ Run `pytest tests/quantization/test_ipex_quant.py`.
7
+"""
8
+
9
+import pytest
10
+
11
+from vllm.platforms import current_platform
12
+
13
+MODELS = [
14
+ "casperhansen/llama-3-8b-instruct-awq",
15
+]
16
+DTYPE = ["bfloat16"]
17
+
18
+
19
+@pytest.mark.skipif(not current_platform.is_cpu(),
20
+ reason="only supports the CPU backend.")
21
+@pytest.mark.parametrize("model", MODELS)
22
+@pytest.mark.parametrize("dtype", DTYPE)
23
+def test_ipex_quant(vllm_runner, model, dtype):
24
+ with vllm_runner(model, dtype=dtype) as llm:
25
+ output = llm.generate_greedy(["The capital of France is"],
26
+ max_tokens=32)
27
+ assert output
28
+ print(output)
vllm/model_executor/layers/linear.py
+1
-1
@@ -27,7 +27,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
27
"AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
28
"MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
29
"TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod",
30
- "ModelOptFp8LinearMethod"
30
+ "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod"
31
]
32
33
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
21
GPTQMarlinConfig)
22
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
23
GPTQMarlin24Config)
24
+from vllm.model_executor.layers.quantization.ipex_quant import IPEXConfig
25
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
26
from vllm.model_executor.layers.quantization.modelopt import ModelOptFp8Config
27
from vllm.model_executor.layers.quantization.neuron_quant import (
@@ -49,6 +50,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
50
"qqq": QQQConfig,
51
"experts_int8": ExpertsInt8Config,
52
"neuron_quant": NeuronQuantConfig,
53
+ "ipex": IPEXConfig,
54
}
55
56
vllm/model_executor/layers/quantization/awq_marlin.py
+4
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
20
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21
from vllm.model_executor.parameter import (GroupQuantScaleParameter,
22
PackedvLLMParameter)
23
+from vllm.platforms import current_platform
24
from vllm.scalar_type import scalar_types
25
26
logger = init_logger(__name__)
@@ -123,6 +124,9 @@ class AWQMarlinConfig(QuantizationConfig):
124
group_size = quant_config.get("group_size")
125
has_zp = quant_config.get("zero_point")
126
127
+ if not current_platform.is_cuda():
128
+ return False
129
+
130
if quant_method != "awq":
131
return False
132
vllm/model_executor/layers/quantization/ipex_quant.py
new
+166
@@ -0,0 +1,166 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+
5
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
6
+from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
7
+from vllm.model_executor.layers.quantization.base_config import (
8
+ QuantizationConfig)
9
+from vllm.platforms import current_platform
10
+
11
+
12
+class IPEXConfig(QuantizationConfig):
13
+ """INT8 quantization config class using IPEX for the CPU backend,
14
+ including AWQ.
15
+ """
16
+
17
+ IPEX_QUANT_METHOD_MAP = {
18
+ "awq": 1,
19
+ "gptq": 2,
20
+ }
21
+
22
+ def __init__(
23
+ self,
24
+ method: str,
25
+ weight_bits: int,
26
+ group_size: int,
27
+ ) -> None:
28
+ self.method = method
29
+ self.weight_bits = weight_bits
30
+ self.group_size = group_size
31
+ self.pack_factor = 32 // self.weight_bits
32
+
33
+ if self.weight_bits not in [4]:
34
+ raise ValueError(f"IPEX quantization supports weight bits [4], "
35
+ f"but got {self.weight_bits}.")
36
+
37
+ if self.method == "awq":
38
+ self.quant_method = IPEXAWQLinearMethod
39
+ else:
40
+ raise ValueError(f"IPEX quantization supports [awq], "
41
+ f"but got {self.method}.")
42
+
43
+ def __repr__(self) -> str:
44
+ return (f"IPEXConfig(method={self.method}"
45
+ f"weight_bits={self.weight_bits}, "
46
+ f"group_size={self.group_size}")
47
+
48
+ def get_ipex_quant_method_id(self) -> int:
49
+ return IPEXConfig.IPEX_QUANT_METHOD_MAP[self.method]
50
+
51
+ @classmethod
52
+ def get_name(cls) -> str:
53
+ return "ipex"
54
+
55
+ @classmethod
56
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
57
+ return [torch.bfloat16]
58
+
59
+ @classmethod
60
+ def get_min_capability(cls) -> int:
61
+ return -1
62
+
63
+ @staticmethod
64
+ def get_config_filenames() -> List[str]:
65
+ return [
66
+ "quant_config.json",
67
+ "quantize_config.json",
68
+ ]
69
+
70
+ @classmethod
71
+ def from_config(cls, config: Dict[str, Any]) -> "IPEXConfig":
72
+ method = cls.get_from_keys(config, ["quant_method"]).lower()
73
+ weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
74
+ group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
75
+ return cls(method, weight_bits, group_size)
76
+
77
+ @classmethod
78
+ def override_quantization_method(cls, hf_quant_cfg,
79
+ user_quant) -> Optional[str]:
80
+ if not current_platform.is_cpu():
81
+ return None
82
+
83
+ quant_method = hf_quant_cfg.get("quant_method", "").lower()
84
+
85
+ if quant_method in ["awq"]:
86
+ return cls.get_name()
87
+
88
+ return None
89
+
90
+ def get_quant_method(self, layer: torch.nn.Module,
91
+ prefix: str) -> Optional["LinearMethodBase"]:
92
+ if isinstance(layer, LinearBase):
93
+ return self.quant_method(self)
94
+ return None
95
+
96
+ def get_scaled_act_names(self) -> List[str]:
97
+ if self.method == "awq":
98
+ return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
99
+ else:
100
+ return []
101
+
102
+
103
+class IPEXAWQLinearMethod(AWQLinearMethod):
104
+ """AWQ linear method using IPEX for the CPU backend.
105
+ """
106
+
107
+ def __init__(self, quant_config: IPEXConfig):
108
+ self.quant_config = quant_config # type: ignore
109
+
110
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
111
+ super().process_weights_after_loading(layer=layer)
112
+
113
+ bias = layer.bias if not layer.skip_bias_add else None
114
+
115
+ try:
116
+ import intel_extension_for_pytorch as ipex
117
+ if ipex.__version__ < "2.4.0":
118
+ raise ImportError("intel_extension_for_pytorch version is "
119
+ "wrong. Please install "
120
+ "intel_extension_for_pytorch>=2.4.0.")
121
+ except ImportError as err:
122
+ raise ImportError(
123
+ "Please install "
124
+ "intel_extension_for_pytorch>=2.4.0 via "
125
+ "`pip install intel_extension_for_pytorch>=2.4.0`"
126
+ " to use IPEX-AWQ linear method.") from err
127
+
128
+ # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
129
+ # with better performance.
130
+ lowp_mode = ipex.quantization.WoqLowpMode.INT8
131
+ # The weight will be de-packed from INT4 to INT8.
132
+ weight_dtype = ipex.quantization.WoqWeightDtype.INT4
133
+ # The float activation will be quantized (dynamic, per-token) to INT8.
134
+ act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH
135
+
136
+ qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
137
+ weight_dtype=weight_dtype,
138
+ lowp_mode=lowp_mode,
139
+ act_quant_mode=act_quant_mode,
140
+ group_size=self.quant_config.group_size,
141
+ )
142
+
143
+ layer.ipex_output_size = layer.qweight.size(
144
+ 1) * self.quant_config.pack_factor
145
+ layer.ipex_qlinear = ipex.nn.modules.weight_only_quantization.\
146
+ WeightOnlyQuantizedLinear.from_weight(
147
+ layer.qweight,
148
+ layer.scales,
149
+ layer.qzeros,
150
+ layer.qweight.size(0),
151
+ layer.ipex_output_size,
152
+ qconfig=qconfig,
153
+ bias=bias,
154
+ group_size=self.quant_config.group_size,
155
+ quant_method=
156
+ self.quant_config.get_ipex_quant_method_id() # type: ignore
157
+ )
158
+
159
+ def apply(self,
160
+ layer: torch.nn.Module,
161
+ x: torch.Tensor,
162
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
163
+ reshaped_x = x.reshape(-1, x.shape[-1])
164
+ out = layer.ipex_qlinear(reshaped_x)
165
+
166
+ return out.reshape(x.shape[:-1] + (layer.ipex_output_size, ))
vllm/worker/cpu_worker.py
+2
-1
@@ -215,7 +215,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
215
def init_device(self) -> None:
216
if self.local_omp_cpuid != "all":
217
ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
218
- logger.info(ret)
218
+ if ret:
219
+ logger.info(ret)
220
221
self.init_distributed_environment()
222
# Set random seed.