@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / baaa2b24da
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix] fix moe_wna16 get_quant_method (#12648)

Fix https://github.com/vllm-project/vllm/issues/12647 The `get_quant_method` of `moe_wna16` always return moe method, GPTQ-based linear method or AWQ-based linear method, even when the target module is attention layer. https://github.com/vllm-project/vllm/blob/baeded25699f9f4851843306f27f685c4d4ee7c5/vllm/attention/layer.py#L86-L92 Signed-off-by: Jinzhen Lin <linjinzhen@hotmail.com>

Jinzhen Lin committed Feb 2, 2025 at 15:29 UTC baaa2b24da86d63965dbffc34c97c7c4b50288db
1 file changed +12 -15
vllm/model_executor/layers/quantization/moe_wna16.py
+12 -15
@@ -6,16 +6,13 @@ from vllm.distributed import get_tensor_model_parallel_rank, get_tp_group
6 from vllm.model_executor.layers.fused_moe.layer import (
7 FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
8 from vllm.model_executor.layers.linear import UnquantizedLinearMethod
9 -from vllm.model_executor.layers.quantization.awq import (AWQConfig,
10 - AWQLinearMethod)
11 -from vllm.model_executor.layers.quantization.awq_marlin import (
12 - AWQMarlinConfig, AWQMarlinLinearMethod)
9 +from vllm.model_executor.layers.quantization.awq import AWQConfig
10 +from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig, QuantizeMethodBase)
15 -from vllm.model_executor.layers.quantization.gptq import (GPTQConfig,
16 - GPTQLinearMethod)
13 +from vllm.model_executor.layers.quantization.gptq import GPTQConfig
14 from vllm.model_executor.layers.quantization.gptq_marlin import (
18 - GPTQMarlinConfig, GPTQMarlinLinearMethod)
15 + GPTQMarlinConfig)
16 from vllm.model_executor.utils import set_weight_attrs
17 from vllm.platforms import current_platform
18
@@ -131,18 +128,18 @@ class MoeWNA16Config(QuantizationConfig):
128 else:
129 if self.linear_quant_method == "gptq":
130 if self.use_marlin:
134 - return GPTQMarlinLinearMethod(
135 - GPTQMarlinConfig.from_config(self.full_config))
131 + return GPTQMarlinConfig.from_config(
132 + self.full_config).get_quant_method(layer, prefix)
133 else:
137 - return GPTQLinearMethod(
138 - GPTQConfig.from_config(self.full_config))
134 + return GPTQConfig.from_config(
135 + self.full_config).get_quant_method(layer, prefix)
136 elif self.linear_quant_method == "awq":
137 if self.use_marlin:
141 - return AWQMarlinLinearMethod(
142 - AWQMarlinConfig.from_config(self.full_config))
138 + return AWQMarlinConfig.from_config(
139 + self.full_config).get_quant_method(layer, prefix)
140 else:
144 - return AWQLinearMethod(
145 - AWQConfig.from_config(self.full_config))
141 + return AWQConfig.from_config(
142 + self.full_config).get_quant_method(layer, prefix)
143 else:
144 raise ValueError("moe_wna16 only support gptq and awq.")
145