A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
FIX MOE issue in AutoRound format (#18586)
Signed-off-by: wenhuach21 <wenhua.cheng@intel.com>
Wenhua Cheng committed
May 24, 2025 at 13:01 UTC
ec82c3e388b962a30a02fa376c222cef787b3c14
2 files changed
+30
-28
README.md
+1
-1
@@ -58,7 +58,7 @@ vLLM is fast with:
58
- Efficient management of attention key and value memory with [**PagedAttention**](https://blog.vllm.ai/2023/06/20/vllm.html)
59
- Continuous batching of incoming requests
60
- Fast model execution with CUDA/HIP graph
61
-- Quantizations: [GPTQ](https://arxiv.org/abs/2210.17323), [AWQ](https://arxiv.org/abs/2306.00978), INT4, INT8, and FP8.
61
+- Quantizations: [GPTQ](https://arxiv.org/abs/2210.17323), [AWQ](https://arxiv.org/abs/2306.00978), [AutoRound](https://arxiv.org/abs/2309.05516),INT4, INT8, and FP8.
62
- Optimized CUDA kernels, including integration with FlashAttention and FlashInfer.
63
- Speculative decoding
64
- Chunked prefill
vllm/model_executor/layers/quantization/auto_round.py
+29
-27
@@ -8,6 +8,7 @@ import torch
8
from vllm.logger import init_logger
9
from vllm.model_executor.layers.linear import (LinearBase,
10
UnquantizedLinearMethod)
11
+from vllm.model_executor.layers.quantization import QuantizationMethods
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
@@ -74,7 +75,7 @@ class AutoRoundConfig(QuantizationConfig):
75
f"group_size={self.group_size}, sym={self.sym})")
76
77
@classmethod
77
- def get_name(cls): ## use str will trigger preci issue
78
+ def get_name(cls) -> QuantizationMethods:
79
return "auto-round"
80
81
@classmethod
@@ -142,18 +143,18 @@ class AutoRoundConfig(QuantizationConfig):
143
prefix, layer.__class__.__name__, weight_bits, group_size,
144
sym)
145
if backend == "auto" or "marlin" in backend:
146
+ AWQ_TYPE_MAP = {
147
+ 4: scalar_types.uint4,
148
+ 8: scalar_types.uint8,
149
+ }
150
+ use_marlin = (weight_bits
151
+ in AWQ_TYPE_MAP) and check_marlin_supported(
152
+ AWQ_TYPE_MAP[weight_bits], group_size, not sym)
153
+
154
if isinstance(layer, FusedMoE):
146
- use_marlin = check_moe_marlin_supports_layer(layer, group_size)
147
- else:
155
+ use_marlin = use_marlin and check_moe_marlin_supports_layer(
156
+ layer, group_size)
157
149
- AWQ_TYPE_MAP = {
150
- 4: scalar_types.uint4,
151
- 8: scalar_types.uint8,
152
- }
153
- use_marlin = ((weight_bits, sym) in AWQ_TYPE_MAP
154
- and check_marlin_supported(
155
- AWQ_TYPE_MAP[(weight_bits)], group_size,
156
- not sym))
158
else:
159
use_marlin = False
160
if use_marlin:
@@ -180,10 +181,11 @@ class AutoRoundConfig(QuantizationConfig):
181
from vllm.model_executor.layers.quantization.moe_wna16 import (
182
MoeWNA16Config)
183
config = {
183
- "linear_quant_method": "awq",
184
- "weight_bits": weight_bits,
184
+ "quant_method": "awq",
185
+ "bits": weight_bits,
186
"group_size": group_size,
187
"zero_point": not sym,
188
+ "lm_head": False,
189
}
190
return MoeWNA16Config.from_config(config).get_quant_method(
191
layer, prefix)
@@ -213,18 +215,18 @@ class AutoRoundConfig(QuantizationConfig):
215
prefix, layer.__class__.__name__, weight_bits, group_size,
216
sym)
217
if backend == "auto" or "marlin" in backend:
218
+ GPTQ_TYPE_MAP = {
219
+ (4, True): scalar_types.uint4b8,
220
+ (8, True): scalar_types.uint8b128,
221
+ }
222
+ use_marlin = ((weight_bits, sym) in GPTQ_TYPE_MAP
223
+ and check_marlin_supported(
224
+ GPTQ_TYPE_MAP[(weight_bits, sym)],
225
+ group_size,
226
+ has_zp=not sym))
227
if isinstance(layer, FusedMoE):
217
- use_marlin = check_moe_marlin_supports_layer(layer, group_size)
218
- else:
219
- GPTQ_TYPE_MAP = {
220
- (4, True): scalar_types.uint4b8,
221
- (8, True): scalar_types.uint8b128,
222
- }
223
- use_marlin = ((weight_bits, sym) in GPTQ_TYPE_MAP
224
- and check_marlin_supported(
225
- GPTQ_TYPE_MAP[(weight_bits, sym)],
226
- group_size,
227
- has_zp=not sym))
228
+ use_marlin = use_marlin and check_moe_marlin_supports_layer(
229
+ layer, group_size)
230
else:
231
use_marlin = False
232
if use_marlin:
@@ -251,11 +253,11 @@ class AutoRoundConfig(QuantizationConfig):
253
from vllm.model_executor.layers.quantization.moe_wna16 import (
254
MoeWNA16Config)
255
config = {
254
- "linear_quant_method": "gptq",
255
- "weight_bits": weight_bits,
256
+ "quant_method": "gptq",
257
+ "bits": weight_bits,
258
"group_size": group_size,
259
"sym": sym,
258
- "lm_head_quantized": False,
260
+ "lm_head": False,
261
}
262
return MoeWNA16Config.from_config(config).get_quant_method(
263
layer, prefix)